From 9af54f4ff26db869f6527ab9a001e44d83b79c7d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 18 Aug 2026 13:30:42 -0400 Subject: [PATCH 1/2] deps: update simdutf to 9.1.0 Refresh deps/v8/third_party/simdutf from the v9.1.0 single-header release (upstream 2d4e249d). Refs: https://github.com/simdutf/simdutf/releases/tag/v9.1.0 Signed-off-by: Daniel Lemire --- deps/v8/third_party/simdutf/README.chromium | 4 +- deps/v8/third_party/simdutf/simdutf.cpp | 19116 ++++++++---------- deps/v8/third_party/simdutf/simdutf.h | 9025 ++++++++- 3 files changed, 16522 insertions(+), 11623 deletions(-) diff --git a/deps/v8/third_party/simdutf/README.chromium b/deps/v8/third_party/simdutf/README.chromium index 003c53476cc0..9a0724d10cea 100644 --- a/deps/v8/third_party/simdutf/README.chromium +++ b/deps/v8/third_party/simdutf/README.chromium @@ -2,8 +2,8 @@ Name: simdutf unicode transcoder Short Name: simdutf URL: https://github.com/simdutf/simdutf Update Mechanism: Manual -Revision: da645ece -Version: 7.7.0 +Revision: 2d4e249d +Version: 9.1.0 License: MIT License File: LICENSE Shipped: yes diff --git a/deps/v8/third_party/simdutf/simdutf.cpp b/deps/v8/third_party/simdutf/simdutf.cpp index ebada9ab4eb6..13fc34c401f7 100644 --- a/deps/v8/third_party/simdutf/simdutf.cpp +++ b/deps/v8/third_party/simdutf/simdutf.cpp @@ -1,905 +1,11 @@ -/* auto-generated on 2025-11-21 22:27:43 -0500. Do not edit! */ +/* auto-generated on 2026-08-18 10:18:07 -0400. Do not edit! */ /* begin file src/simdutf.cpp */ #include "simdutf.h" -#if SIMDUTF_FEATURE_BASE64 - // We include base64_tables once. -/* begin file src/tables/base64_tables.h */ -#ifndef SIMDUTF_BASE64_TABLES_H -#define SIMDUTF_BASE64_TABLES_H -#include -#include - -namespace simdutf { -namespace { -namespace tables { -namespace base64 { -namespace base64_default { - -const char e0[256] = { - 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', - 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', - 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', - 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', - 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', - 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', - 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', - 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', - 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', - 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', - 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', - 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', - 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', - 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', - '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', - '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', - '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', - '/'}; - -const char e1[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', - '/'}; - -const char e2[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', - '/'}; - -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; - -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_default - -namespace base64_url { - -const char e0[256] = { - 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', - 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', - 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', - 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', - 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', - 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', - 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', - 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', - 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', - 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', - 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', - 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', - 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', - 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', - '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', - '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', - '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', - '_'}; - -const char e1[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', - '_'}; - -const char e2[256] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', - 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', - 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', - 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', - 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', - 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', - 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', - 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', - 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', - '_'}; - -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_url - -namespace base64_default_or_url { -const uint32_t d0[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, - 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, - 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, - 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, - 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, - 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, - 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, - 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, - 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, - 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, - 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, - 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d1[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, - 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, - 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, - 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, - 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, - 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, - 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, - 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, - 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, - 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, - 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, - 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d2[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, - 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, - 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, - 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, - 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, - 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, - 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, - 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, - 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, - 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, - 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, - 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -const uint32_t d3[256] = { - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, - 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, - 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, - 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, - 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, - 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, - 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, - 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, - 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, - 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, - 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, - 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, - 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, - 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; -} // namespace base64_default_or_url -const uint64_t thintable_epi8[256] = { - 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, - 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, - 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, - 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, - 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, - 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, - 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, - 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, - 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, - 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, - 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, - 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, - 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, - 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, - 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, - 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, - 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, - 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, - 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, - 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, - 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, - 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, - 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, - 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, - 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, - 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, - 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, - 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, - 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, - 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, - 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, - 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, - 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, - 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, - 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, - 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, - 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, - 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, - 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, - 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, - 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, - 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, - 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, - 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, - 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, - 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, - 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, - 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, - 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, - 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, - 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, - 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, - 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, - 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, - 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, - 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, - 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, - 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, - 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, - 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, - 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, - 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, - 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, - 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, - 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, - 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, - 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, - 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, - 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, - 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, - 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, - 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, - 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, - 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, - 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, - 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, - 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, - 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, - 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, - 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, - 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, - 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, - 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, - 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, - 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, - 0x0000000000000000, -}; - -const uint8_t pshufb_combine_table[272] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - -const unsigned char BitsSetTable256mul2[256] = { - 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, - 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, - 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, - 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, - 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, - 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, - 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, - 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, - 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, - 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, - 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, - 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, - 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, - 14, 10, 12, 12, 14, 12, 14, 14, 16}; - -constexpr uint8_t to_base64_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, - 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -constexpr uint8_t to_base64_url_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -constexpr uint8_t to_base64_default_or_url_value[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, - 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - -static_assert(sizeof(to_base64_value) == 256, - "to_base64_value must have 256 elements"); -static_assert(sizeof(to_base64_url_value) == 256, - "to_base64_url_value must have 256 elements"); -static_assert(to_base64_value[uint8_t(' ')] == 64, - "space must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t(' ')] == 64, - "space must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\t')] == 64, - "tab must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\t')] == 64, - "tab must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\r')] == 64, - "cr must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\r')] == 64, - "cr must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\n')] == 64, - "lf must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\n')] == 64, - "lf must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('\f')] == 64, - "ff must be == 64 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('\f')] == 64, - "ff must be == 64 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('+')] == 62, - "+ must be == 62 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('-')] == 62, - "- must be == 62 in to_base64_url_value"); -static_assert(to_base64_value[uint8_t('/')] == 63, - "/ must be == 62 in to_base64_value"); -static_assert(to_base64_url_value[uint8_t('_')] == 63, - "_ must be == 62 in to_base64_url_value"); -} // namespace base64 -} // namespace tables -} // unnamed namespace -} // namespace simdutf - -#endif // SIMDUTF_BASE64_TABLES_H -/* end file src/tables/base64_tables.h */ -#endif // SIMDUTF_FEATURE_BASE64 - /* begin file src/encoding_types.cpp */ namespace simdutf { -std::string to_string(encoding_type bom) { +std::string_view to_string(encoding_type bom) { switch (bom) { case UTF16_LE: return "UTF16 little-endian"; @@ -2869,87 +1975,101 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; -}; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 +}; } // namespace arm64 } // namespace simdutf @@ -2960,7 +2080,6 @@ class implementation final : public simdutf::implementation { /* begin file src/simdutf/arm64/begin.h */ // redefining SIMDUTF_IMPLEMENTATION to "arm64" // #define SIMDUTF_IMPLEMENTATION arm64 -#define SIMDUTF_SIMD_HAS_BYTEMASK 1 /* end file src/simdutf/arm64/begin.h */ // Declarations @@ -2985,7 +2104,14 @@ namespace { /* result might be undefined when input_num is zero */ simdutf_really_inline int count_ones(uint64_t input_num) { +#ifdef SIMDUTF_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif } #if SIMDUTF_NEED_TRAILING_ZEROES @@ -3078,6 +2204,32 @@ namespace { } // namespace #endif // SIMDUTF_REGULAR_VISUAL_STUDIO +// Returns true if any lane of `mask` is set. The argument *must* be the result +// of a lane-wise comparison, i.e. each byte must be either 0x00 or 0xff. The +// lane width of the comparison is irrelevant: byte and 32-bit masks should be +// reinterpreted with vreinterpretq_u16_u8 / vreinterpretq_u16_u32 by the +// caller. +// +// This compiles to two instructions (shrn + fcmp) and, unlike a reduction such +// as vmaxvq_u8, it never moves the value to a general-purpose register. Such a +// transfer has a latency of about 3 cycles on Apple hardware, on top of the 3 +// cycles of the reduction itself. +// +// Both steps rely on the input being a comparison mask. The narrowing shift +// keeps bits 4..11 of each 16-bit lane, so it only preserves 'is non-zero' when +// every byte is 0x00 or 0xff. The floating-point comparison is safe for the +// same reason: the only non-zero bit pattern that compares equal to 0.0 is -0.0 +// (0x8000000000000000), and the most significant byte of the narrowed value can +// only be 0x00, 0x0f, 0xf0 or 0xff. +// +// There is deliberately a single overload: Visual Studio defines every 128-bit +// NEON type as the same union type, so overloading on uint8x16_t, uint16x8_t +// and uint32x4_t does not compile there. +simdutf_really_inline bool any_lane_set(const uint16x8_t mask) { + const uint8x8_t narrowed = vshrn_n_u16(mask, 4); + return vget_lane_f64(vreinterpret_f64_u8(narrowed), 0) != 0.0; +} + template struct simd8; // @@ -3322,9 +2474,10 @@ template <> struct simd8 { // ... template simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { - int8x16x2_t pair = match_system(big_endian) - ? int8x16x2_t{{this->value, vmovq_n_s8(0)}} - : int8x16x2_t{{vmovq_n_s8(0), this->value}}; + constexpr auto matches = match_system(big_endian); + const int8x16x2_t pair = matches + ? int8x16x2_t{{this->value, vmovq_n_s8(0)}} + : int8x16x2_t{{vmovq_n_s8(0), this->value}}; vst2q_s8(reinterpret_cast(p), pair); } @@ -3523,6 +2676,12 @@ template struct simd8x64 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); return simd8x64(simd8(uint8x16_t(this->chunks[0])) >= mask, @@ -3892,7 +3051,12 @@ template struct simd16x32 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } - + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, @@ -3992,7 +3156,11 @@ template <> struct simd32 { simdutf_really_inline simd32(const uint32x4_t v) : value(v) {} - simdutf_really_inline bool any() const { return vmaxvq_u32(value) != 0; } + // simd32 is only ever produced by lane-wise comparisons (and bitwise + // combinations thereof), so the cheap any_lane_set is always applicable. + simdutf_really_inline bool any() const { + return any_lane_set(vreinterpretq_u16_u32(value)); + } }; //---------------------------------------------------------------------- @@ -4111,7 +3279,6 @@ simdutf_really_inline simd64 sum_8bytes(const simd8 v) { /* end file src/simdutf/arm64/simd.h */ /* begin file src/simdutf/arm64/end.h */ -#undef SIMDUTF_SIMD_HAS_BYTEMASK /* end file src/simdutf/arm64/end.h */ #endif // SIMDUTF_IMPLEMENTATION_ARM64 @@ -4559,95 +3726,109 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 -#if SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 }; @@ -5172,10 +4353,10 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; void to_well_formed_utf16be(const char16_t *input, size_t len, char16_t *output) const noexcept final; void to_well_formed_utf16le(const char16_t *input, size_t len, @@ -5184,87 +4365,101 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 -#if SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 }; @@ -6461,10 +5656,10 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; void to_well_formed_utf16be(const char16_t *input, size_t len, char16_t *output) const noexcept final; void to_well_formed_utf16le(const char16_t *input, size_t len, @@ -6473,87 +5668,101 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 -#if SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 }; @@ -6997,7 +6206,12 @@ template struct simd8x64 { this->chunks[2] > mask, this->chunks[3] > mask) .to_bitmask(); } - + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } simdutf_really_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, @@ -7229,6 +6443,13 @@ template struct simd16x32 { .to_bitmask(); } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); + } + simdutf_really_inline uint64_t eq(const T m) const { const simd16 mask = simd16::splat(m); return simd16x32(this->chunks[0] == mask, this->chunks[1] == mask, @@ -7722,65 +6943,74 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 -#if SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 @@ -7789,32 +7019,33 @@ class implementation final : public simdutf::implementation { simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 #ifdef SIMDUTF_INTERNAL_TESTS @@ -9518,86 +8749,96 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; + base64_options options) const noexcept override; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 private: const bool _supports_zvbb; @@ -9771,42 +9012,47 @@ SIMDUTF_UNTARGET_REGION #endif // SIMDUTF_RVV_H /* end file src/simdutf/rvv.h */ -/* begin file src/simdutf/lsx.h */ -#ifndef SIMDUTF_LSX_H -#define SIMDUTF_LSX_H +/* begin file src/simdutf/lasx.h */ +#ifndef SIMDUTF_LASX_H +#define SIMDUTF_LASX_H #ifdef SIMDUTF_FALLBACK_H - #error "lsx.h must be included before fallback.h" + #error "lasx.h must be included before fallback.h" #endif -#ifndef SIMDUTF_IMPLEMENTATION_LSX - #define SIMDUTF_IMPLEMENTATION_LSX (SIMDUTF_IS_LSX) +#ifndef SIMDUTF_IMPLEMENTATION_LASX + #define SIMDUTF_IMPLEMENTATION_LASX (SIMDUTF_IS_LSX) #endif -#if SIMDUTF_IMPLEMENTATION_LSX && SIMDUTF_IS_LSX - #define SIMDUTF_CAN_ALWAYS_RUN_LSX 1 +#if SIMDUTF_IMPLEMENTATION_LASX && SIMDUTF_IS_LASX + #define SIMDUTF_CAN_ALWAYS_RUN_LASX 1 #else - #define SIMDUTF_CAN_ALWAYS_RUN_LSX 0 + #define SIMDUTF_CAN_ALWAYS_RUN_LASX 0 #endif #define SIMDUTF_CAN_ALWAYS_RUN_FALLBACK (SIMDUTF_IMPLEMENTATION_FALLBACK) -#if SIMDUTF_IMPLEMENTATION_LSX +#if SIMDUTF_IMPLEMENTATION_LASX + #define SIMDUTF_TARGET_LASX SIMDUTF_TARGET_REGION("lasx,lsx") + + // For runtime dispatching to work, we need the lsxintrin to appear + // before we call SIMDUTF_TARGET_LASX. It is unclear why. + #include namespace simdutf { /** - * Implementation for LoongArch SX. + * Implementation for LoongArch ASX. */ -namespace lsx {} // namespace lsx +namespace lasx {} // namespace lasx } // namespace simdutf -/* begin file src/simdutf/lsx/implementation.h */ -#ifndef SIMDUTF_LSX_IMPLEMENTATION_H -#define SIMDUTF_LSX_IMPLEMENTATION_H +/* begin file src/simdutf/lasx/implementation.h */ +#ifndef SIMDUTF_LASX_IMPLEMENTATION_H +#define SIMDUTF_LASX_IMPLEMENTATION_H namespace simdutf { -namespace lsx { +namespace lasx { namespace { using namespace simdutf; @@ -9815,8 +9061,9 @@ using namespace simdutf; class implementation final : public simdutf::implementation { public: simdutf_really_inline implementation() - : simdutf::implementation("lsx", "LOONGARCH SX", - internal::instruction_set::LSX) {} + : simdutf::implementation("lasx", "LOONGARCH ASX", + internal::instruction_set::LSX | + internal::instruction_set::LASX) {} #if SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused int detect_encodings(const char *input, size_t length) const noexcept final; @@ -10008,109 +9255,220 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 }; -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_IMPLEMENTATION_H -/* end file src/simdutf/lsx/implementation.h */ +#endif // SIMDUTF_LASX_IMPLEMENTATION_H +/* end file src/simdutf/lasx/implementation.h */ -/* begin file src/simdutf/lsx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lsx" -// #define SIMDUTF_IMPLEMENTATION lsx +/* begin file src/simdutf/lasx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lasx" +// #define SIMDUTF_IMPLEMENTATION lasx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lsx/begin.h */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_TARGET_LASX +#endif +/* end file src/simdutf/lasx/begin.h */ // Declarations -/* begin file src/simdutf/lsx/intrinsics.h */ -#ifndef SIMDUTF_LSX_INTRINSICS_H -#define SIMDUTF_LSX_INTRINSICS_H +/* begin file src/simdutf/lasx/intrinsics.h */ +#ifndef SIMDUTF_LASX_INTRINSICS_H +#define SIMDUTF_LASX_INTRINSICS_H // This should be the correct header whether // you use visual studio or other compilers. #include +#include + +#if defined(__loongarch_asx) + #ifdef __clang__ + #define VREGS_PREFIX "$vr" + #define XREGS_PREFIX "$xr" + #else // GCC + #define VREGS_PREFIX "$f" + #define XREGS_PREFIX "$f" + #endif + #define __ALL_REGS \ + "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26," \ + "27,28,29,30,31" +// Convert __m128i to __m256i +static inline __m256i ____m256i(__m128i in) { + __m256i out = __lasx_xvldi(0); + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "+f"(out) + : [in] "f"(in)); + return out; +} +// Convert two __m128i to __m256i +static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) { + __m256i out; + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[lo], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".ifnc %[out], %[hi] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\j \n\t" + " xvori.b $xr\\i, $xr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f"(out), [hi] "+f"(inhi) + : [lo] "f"(inlo)); + return out; +} +// Convert __m256i low part to __m128i +static inline __m128i lasx_extracti128_lo(__m256i in) { + __m128i out; + __asm__ volatile(".ifnc %[out], %[in] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " vori.b $vr\\i, $vr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f"(out) + : [in] "f"(in)); + return out; +} +// Convert __m256i high part to __m128i +static inline __m128i lasx_extracti128_hi(__m256i in) { + __m128i out; + __asm__ volatile(".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "=f"(out) + : [in] "f"(in)); + return out; +} +#endif /* Encoding of argument for LoongArch64 xvldi instruction. See: @@ -10140,7 +9498,7 @@ all lanes the result as 64-bit elements to all lanes */ -namespace vldi { +namespace lasx_vldi { template class const_u16 { constexpr static const uint8_t b0 = ((v >> 0 * 8) & 0xff); @@ -10257,7 +9615,8 @@ template class const_u64 { constexpr static int value = int((operation << 8) | byte) - 8192; constexpr static bool valid = operation != 0xffff; }; -} // namespace vldi + +} // namespace lasx_vldi // Uncomment when running under QEMU affected // by bug https://gitlab.com/qemu-project/qemu/-/issues/2865 @@ -10267,45 +9626,78 @@ template class const_u64 { #endif #ifdef QEMU_VLDI_BUG - #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) - #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #define lasx_splat_u16(v) __lasx_xvreplgr2vr_h(v) + #define lasx_splat_u32(v) __lasx_xvreplgr2vr_w(v) #else -template constexpr __m128i lsx_splat_u16_aux() { +template constexpr __m256i lasx_splat_u16_aux() { constexpr bool is_imm10 = (int16_t(x) < 512) && (int16_t(x) > -512); constexpr uint16_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = vldi::const_u16::valid; - constexpr int vldi_imm = is_vldi ? vldi::const_u16::value : 0; + constexpr bool is_vldi = lasx_vldi::const_u16::valid; + constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u16::value : 0; - return is_imm10 ? __lsx_vrepli_h(int16_t(imm10)) - : is_vldi ? __lsx_vldi(vldi_imm) - : __lsx_vreplgr2vr_h(x); + return is_imm10 ? __lasx_xvrepli_h(int16_t(imm10)) + : is_vldi ? __lasx_xvldi(vldi_imm) + : __lasx_xvreplgr2vr_h(x); } -template constexpr __m128i lsx_splat_u32_aux() { +template constexpr __m256i lasx_splat_u32_aux() { constexpr bool is_imm10 = (int32_t(x) < 512) && (int32_t(x) > -512); constexpr uint32_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = vldi::const_u32::valid; - constexpr int vldi_imm = is_vldi ? vldi::const_u32::value : 0; + constexpr bool is_vldi = lasx_vldi::const_u32::valid; + constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u32::value : 0; - return is_imm10 ? __lsx_vrepli_w(int32_t(imm10)) - : is_vldi ? __lsx_vldi(vldi_imm) - : __lsx_vreplgr2vr_w(x); + return is_imm10 ? __lasx_xvrepli_w(int32_t(imm10)) + : is_vldi ? __lasx_xvldi(vldi_imm) + : __lasx_xvreplgr2vr_w(x); } - #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() - #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #define lasx_splat_u16(v) lasx_splat_u16_aux<(v)>() + #define lasx_splat_u32(v) lasx_splat_u32_aux<(v)>() #endif // QEMU_VLDI_BUG -#endif // SIMDUTF_LSX_INTRINSICS_H -/* end file src/simdutf/lsx/intrinsics.h */ -/* begin file src/simdutf/lsx/bitmanipulation.h */ -#ifndef SIMDUTF_LSX_BITMANIPULATION_H -#define SIMDUTF_LSX_BITMANIPULATION_H +#ifndef lsx_splat_u16 + #ifdef QEMU_VLDI_BUG + #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) + #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #else +namespace { +template constexpr __m128i lsx_splat_u16_aux() { + return ((int16_t(x) < 512) && (int16_t(x) > -512)) + ? __lsx_vrepli_h( + ((int16_t(x) < 512) && (int16_t(x) > -512)) ? int16_t(x) : 0) + : (lasx_vldi::const_u16::valid + ? __lsx_vldi(lasx_vldi::const_u16::valid + ? lasx_vldi::const_u16::value + : 0) + : __lsx_vreplgr2vr_h(x)); +} + +template constexpr __m128i lsx_splat_u32_aux() { + return ((int32_t(x) < 512) && (int32_t(x) > -512)) + ? __lsx_vrepli_w( + ((int32_t(x) < 512) && (int32_t(x) > -512)) ? int32_t(x) : 0) + : (lasx_vldi::const_u32::valid + ? __lsx_vldi(lasx_vldi::const_u32::valid + ? lasx_vldi::const_u32::value + : 0) + : __lsx_vreplgr2vr_w(x)); +} +} // namespace + #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() + #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #endif // QEMU_VLDI_BUG +#endif // lsx_splat_u16 + +#endif // SIMDUTF_LASX_INTRINSICS_H +/* end file src/simdutf/lasx/intrinsics.h */ +/* begin file src/simdutf/lasx/bitmanipulation.h */ +#ifndef SIMDUTF_LASX_BITMANIPULATION_H +#define SIMDUTF_LASX_BITMANIPULATION_H #include namespace simdutf { -namespace lsx { +namespace lasx { namespace { simdutf_really_inline int count_ones(uint64_t input_num) { @@ -10319,171 +9711,349 @@ simdutf_really_inline int trailing_zeroes(uint64_t input_num) { #endif } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_BITMANIPULATION_H -/* end file src/simdutf/lsx/bitmanipulation.h */ -/* begin file src/simdutf/lsx/simd.h */ -#ifndef SIMDUTF_LSX_SIMD_H -#define SIMDUTF_LSX_SIMD_H +#endif // SIMDUTF_LASX_BITMANIPULATION_H +/* end file src/simdutf/lasx/bitmanipulation.h */ +/* begin file src/simdutf/lasx/simd.h */ +#ifndef SIMDUTF_LASX_SIMD_H +#define SIMDUTF_LASX_SIMD_H namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace simd { -template struct simd8; +__attribute__((aligned(32))) static const uint8_t prev_shuf_table[32][32] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + {0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + {0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + {0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, + 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0}, + {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0}, + {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0}, + {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0}, + {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0}, + {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0}, + {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0}, + {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, +}; -// -// Base class of simd8 and simd8, both of which use __m128i -// internally. -// -template > struct base_u8 { - __m128i value; - static const int SIZE = sizeof(value); +__attribute__((aligned(32))) static const uint8_t bitsel_mask_table[32][32] = { + {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0}, + {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0}}; - // Conversion from/to SIMD register - simdutf_really_inline base_u8(const __m128i _value) : value(_value) {} - simdutf_really_inline operator const __m128i &() const { return this->value; } - simdutf_really_inline operator __m128i &() { return this->value; } +// Forward-declared so they can be used by splat and friends. +template struct base { + __m256i value; + + // Zero constructor + simdutf_really_inline base() : value{__m256i()} {} + + // Conversion from SIMD register + simdutf_really_inline base(const __m256i _value) : value(_value) {} + // Conversion to SIMD register + simdutf_really_inline operator const __m256i &() const { return this->value; } + simdutf_really_inline operator __m256i &() { return this->value; } + template + simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { + if (big_endian) { + __m256i zero = __lasx_xvldi(0); + __m256i in8 = __lasx_xvpermi_d(this->value, 0b11011000); + __m256i inlow = __lasx_xvilvl_b(in8, zero); + __m256i inhigh = __lasx_xvilvh_b(in8, zero); + __lasx_xvst(inlow, reinterpret_cast(ptr), 0); + __lasx_xvst(inhigh, reinterpret_cast(ptr), 32); + } else { + __m256i inlow = __lasx_vext2xv_hu_bu(this->value); + __m256i inhigh = __lasx_vext2xv_hu_bu( + __lasx_xvpermi_q(this->value, this->value, 0b00000001)); + __lasx_xvst(inlow, reinterpret_cast<__m256i *>(ptr), 0); + __lasx_xvst(inhigh, reinterpret_cast<__m256i *>(ptr), 32); + } + } + simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { + __m256i in32_0 = __lasx_vext2xv_wu_bu(this->value); + __lasx_xvst(in32_0, reinterpret_cast(ptr), 0); + + __m256i in8_1 = __lasx_xvpermi_d(this->value, 0b00000001); + __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); + __lasx_xvst(in32_1, reinterpret_cast(ptr), 32); + + __m256i in8_2 = __lasx_xvpermi_d(this->value, 0b00000010); + __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); + __lasx_xvst(in32_2, reinterpret_cast(ptr), 64); + __m256i in8_3 = __lasx_xvpermi_d(this->value, 0b00000011); + __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); + __lasx_xvst(in32_3, reinterpret_cast(ptr), 96); + } // Bit operations - simdutf_really_inline simd8 operator|(const simd8 other) const { - return __lsx_vor_v(this->value, other); + simdutf_really_inline Child operator|(const Child other) const { + return __lasx_xvor_v(this->value, other); } - simdutf_really_inline simd8 operator&(const simd8 other) const { - return __lsx_vand_v(this->value, other); + simdutf_really_inline Child operator&(const Child other) const { + return __lasx_xvand_v(this->value, other); } - simdutf_really_inline simd8 operator^(const simd8 other) const { - return __lsx_vxor_v(this->value, other); + simdutf_really_inline Child operator^(const Child other) const { + return __lasx_xvxor_v(this->value, other); } - simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdutf_really_inline simd8 &operator|=(const simd8 other) { - auto this_cast = static_cast *>(this); + simdutf_really_inline Child &operator|=(const Child other) { + auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } +}; + +template struct simd8; +template > +struct base8 : base> { + simdutf_really_inline base8() : base>() {} + simdutf_really_inline base8(const __m256i _value) : base>(_value) {} friend simdutf_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return __lsx_vseq_b(lhs, rhs); + return __lasx_xvseq_b(lhs, rhs); } - template + static const int SIZE = sizeof(base::value); + + template simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(this->value, N), - __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + static_assert(N <= 16, "unsupported shift value"); + + if (!N) + return this->value; + + __m256i zero = __lasx_xvldi(0); + __m256i result, shuf; + if (N < 16) { + shuf = __lasx_xvld(prev_shuf_table[N], 0); + + result = __lasx_xvshuf_b( + __lasx_xvpermi_q(this->value, this->value, 0b00000001), this->value, + shuf); + __m256i srl_prev = __lasx_xvbsrl_v( + __lasx_xvpermi_q(zero, prev_chunk.value, 0b00110001), (16 - N)); + __m256i mask = __lasx_xvld(bitsel_mask_table[N], 0); + result = __lasx_xvbitsel_v(result, srl_prev, mask); + + return result; + } else if (N == 16) { + return __lasx_xvpermi_q(this->value, prev_chunk.value, 0b00100001); + } } }; // SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base_u8 { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - +template <> struct simd8 : base8 { static simdutf_really_inline simd8 splat(bool _value) { - return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } - simdutf_really_inline simd8(const __m128i _value) : base_u8(_value) {} - // False constructor - simdutf_really_inline simd8() : simd8(__lsx_vldi(0)) {} + simdutf_really_inline simd8() : base8() {} + simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} // Splat constructor - simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {} - simdutf_really_inline void store(uint8_t dst[16]) const { - return __lsx_vst(this->value, dst, 0); - } + simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} simdutf_really_inline uint32_t to_bitmask() const { - return __lsx_vpickve2gr_wu(__lsx_vmsknz_b(*this), 0); + __m256i mask = __lasx_xvmsknz_b(this->value); + uint32_t mask0 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(mask, 4); + return (mask0 | (mask1 << 16)); } + simdutf_really_inline bool any() const { + if (__lasx_xbz_b(this->value)) + return false; + return true; + } + simdutf_really_inline simd8 operator~() const { return *this ^ true; } }; -// Unsigned bytes -template <> struct simd8 : base_u8 { - static simdutf_really_inline simd8 splat(uint8_t _value) { - return __lsx_vreplgr2vr_b(_value); +template struct base8_numeric : base8 { + static simdutf_really_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); } - static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } - static simdutf_really_inline simd8 load(const uint8_t *values) { - return __lsx_vld(values, 0); + static simdutf_really_inline simd8 zero() { return __lasx_xvldi(0); } + static simdutf_really_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); } - simdutf_really_inline simd8(const __m128i _value) - : base_u8(_value) {} - // Zero constructor - simdutf_really_inline simd8() : simd8(zero()) {} - // Array constructor - simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Splat constructor - simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Member-by-member initialization - simdutf_really_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i)v16u8{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, - v12, v13, v14, v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdutf_really_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Store to array - simdutf_really_inline void store(uint8_t dst[16]) const { - return __lsx_vst(this->value, dst, 0); + static simdutf_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + T v5, T v6, T v7, T v8, T v9, + T v10, T v11, T v12, T v13, + T v14, T v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15); } - // Order-specific operations - simdutf_really_inline simd8 - operator>=(const simd8 other) const { - return __lsx_vsle_bu(other, *this); - } - simdutf_really_inline simd8 - operator>(const simd8 other) const { - return __lsx_vslt_bu(other, *this); - } - simdutf_really_inline simd8 &operator-=(const simd8 other) { - value = __lsx_vsub_b(value, other.value); - return *this; - } - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true - // = nonzero. For ARM, returns all 1's. - simdutf_really_inline simd8 - gt_bits(const simd8 other) const { - return simd8(*this > other); - } + simdutf_really_inline base8_numeric() : base8() {} + simdutf_really_inline base8_numeric(const __m256i _value) + : base8(_value) {} - // Bit-specific operations - simdutf_really_inline simd8 any_bits_set(simd8 bits) const { - return __lsx_vslt_bu(__lsx_vldi(0), __lsx_vand_v(this->value, bits)); - } - simdutf_really_inline bool is_ascii() const { - return __lsx_vpickve2gr_hu(__lsx_vmskgez_b(this->value), 0) == 0xFFFF; + // Store to array + simdutf_really_inline void store(T dst[32]) const { + return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); } - simdutf_really_inline bool any_bits_set_anywhere() const { - return __lsx_vpickve2gr_hu(__lsx_vmsknz_b(this->value), 0) > 0; - } - template simdutf_really_inline simd8 shr() const { - return __lsx_vsrli_b(this->value, N); - } - template simdutf_really_inline simd8 shl() const { - return __lsx_vslli_b(this->value, N); - } + // Override to distinguish from bool version + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) template simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); + __m256i origin = __lasx_xvand_v(this->value, __lasx_xvldi(0x1f)); + return __lasx_xvshuf_b(__lasx_xvldi(0), lookup_table, origin); } template @@ -10497,150 +10067,119 @@ template <> struct simd8 : base_u8 { replace7, replace8, replace9, replace10, replace11, replace12, replace13, replace14, replace15)); } - - template - simdutf_really_inline simd8 - apply_lookup_16_to(const simd8 original) const { - __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); - return __lsx_vshuf_b(__lsx_vldi(0), *this, simd8(original_tmp)); - } - - simdutf_really_inline uint64_t sum_bytes() const { - const auto sum_u16 = __lsx_vhaddw_hu_bu(value, value); - const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); - - return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + - uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); - } }; // Signed bytes -template <> struct simd8 { - __m128i value; - - static const int SIZE = sizeof(value); +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) + : base8_numeric(_value) {} - static simdutf_really_inline simd8 splat(int8_t _value) { - return __lsx_vreplgr2vr_b(_value); + // Splat constructor + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + simdutf_really_inline operator simd8() const; + simdutf_really_inline bool is_ascii() const { + __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); + if (__lasx_xbnz_v(ascii_mask)) + return false; + return true; } - static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } - static simdutf_really_inline simd8 load(const int8_t values[16]) { - return __lsx_vld(values, 0); + // Order-sensitive comparisons + simdutf_really_inline simd8 operator>(const simd8 other) const { + return __lasx_xvslt_b(other, this->value); } - - template - simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { - __m128i zero = __lsx_vldi(0); - if (match_system(big_endian)) { - __lsx_vst(__lsx_vilvl_b(zero, (__m128i)this->value), - reinterpret_cast(p), 0); - __lsx_vst(__lsx_vilvh_b(zero, (__m128i)this->value), - reinterpret_cast(p + 8), 0); - } else { - __lsx_vst(__lsx_vilvl_b((__m128i)this->value, zero), - reinterpret_cast(p), 0); - __lsx_vst(__lsx_vilvh_b((__m128i)this->value, zero), - reinterpret_cast(p + 8), 0); - } + simdutf_really_inline simd8 operator<(const simd8 other) const { + return __lasx_xvslt_b(this->value, other); } +}; - simdutf_really_inline void store_ascii_as_utf32(char32_t *p) const { - __m128i zero = __lsx_vldi(0); - __m128i in16low = __lsx_vilvl_b(zero, (__m128i)this->value); - __m128i in16high = __lsx_vilvh_b(zero, (__m128i)this->value); - __m128i in32_0 = __lsx_vilvl_h(zero, in16low); - __m128i in32_1 = __lsx_vilvh_h(zero, in16low); - __m128i in32_2 = __lsx_vilvl_h(zero, in16high); - __m128i in32_3 = __lsx_vilvh_h(zero, in16high); - __lsx_vst(in32_0, reinterpret_cast(p), 0); - __lsx_vst(in32_1, reinterpret_cast(p + 4), 0); - __lsx_vst(in32_2, reinterpret_cast(p + 8), 0); - __lsx_vst(in32_3, reinterpret_cast(p + 12), 0); - } - - // In places where the table can be reused, which is most uses in simdutf, it - // is worth it to do 4 table lookups, as there is no direct zero extension - // from u8 to u32. - simdutf_really_inline void store_ascii_as_utf32_tbl(char32_t *p) const { - const simd8 tb1{0, 255, 255, 255, 1, 255, 255, 255, - 2, 255, 255, 255, 3, 255, 255, 255}; - const simd8 tb2{4, 255, 255, 255, 5, 255, 255, 255, - 6, 255, 255, 255, 7, 255, 255, 255}; - const simd8 tb3{8, 255, 255, 255, 9, 255, 255, 255, - 10, 255, 255, 255, 11, 255, 255, 255}; - const simd8 tb4{12, 255, 255, 255, 13, 255, 255, 255, - 14, 255, 255, 255, 15, 255, 255, 255}; - - // encourage store pairing and interleaving - const auto shuf1 = this->apply_lookup_16_to(tb1); - const auto shuf2 = this->apply_lookup_16_to(tb2); - shuf1.store(reinterpret_cast(p)); - shuf2.store(reinterpret_cast(p + 4)); - - const auto shuf3 = this->apply_lookup_16_to(tb3); - const auto shuf4 = this->apply_lookup_16_to(tb4); - shuf3.store(reinterpret_cast(p + 8)); - shuf4.store(reinterpret_cast(p + 12)); - } - // Conversion from/to SIMD register - simdutf_really_inline simd8(const __m128i _value) : value(_value) {} - - // Zero constructor - simdutf_really_inline simd8() : simd8(zero()) {} +// Unsigned bytes +template <> struct simd8 : base8_numeric { + simdutf_really_inline simd8() : base8_numeric() {} + simdutf_really_inline simd8(const __m256i _value) + : base8_numeric(_value) {} // Splat constructor - simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {} + simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdutf_really_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, + uint8_t v21, uint8_t v22, uint8_t v23, uint8_t v24, uint8_t v25, + uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, + uint8_t v31) + : simd8((__m256i)v32u8{v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31}) {} - // Store to array - simdutf_really_inline void store(int8_t dst[16]) const { - return __lsx_vst(value, dst, 0); + // Saturated math + simdutf_really_inline simd8 + saturating_sub(const simd8 other) const { + return __lasx_xvssub_bu(this->value, other); } - simdutf_really_inline operator simd8() const { - return ((__m128i)this->value); + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdutf_really_inline simd8 + gt_bits(const simd8 other) const { + return this->saturating_sub(other); } - - simdutf_really_inline simd8 - operator|(const simd8 other) const { - return __lsx_vor_v((__m128i)value, (__m128i)other.value); + simdutf_really_inline simd8 + operator>=(const simd8 other) const { + return __lasx_xvsle_bu(other, *this); + } + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return __lasx_xvslt_bu(other, *this); + } + simdutf_really_inline simd8 &operator-=(const simd8 other) { + value = __lasx_xvsub_b(value, other.value); + return *this; } + // Bit-specific operations simdutf_really_inline bool is_ascii() const { - return (__lsx_vpickve2gr_hu(__lsx_vmskgez_b((__m128i)this->value), 0) == - 0xffff); + __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); + if (__lasx_xbnz_v(ascii_mask)) + return false; + return true; } - - // Order-sensitive comparisons - simdutf_really_inline simd8 operator>(const simd8 other) const { - return __lsx_vslt_b((__m128i)other.value, (__m128i)value); + simdutf_really_inline bool any_bits_set_anywhere() const { + if (__lasx_xbnz_v(this->value)) + return true; + return false; } - simdutf_really_inline simd8 operator<(const simd8 other) const { - return __lsx_vslt_b((__m128i)value, (__m128i)other.value); + template simdutf_really_inline simd8 shr() const { + return __lasx_xvsrli_b(this->value, N); } - - template - simdutf_really_inline simd8 - prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(this->value, N), - __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + template simdutf_really_inline simd8 shl() const { + return __lasx_xvslli_b(this->value, N); } - template - simdutf_really_inline simd8 - apply_lookup_16_to(const simd8 original) const { - __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); - return __lsx_vshuf_b(__lsx_vldi(0), (__m128i)this->value, - simd8(original_tmp)); + simdutf_really_inline uint64_t sum_bytes() const { + const auto sum_u16 = __lasx_xvhaddw_hu_bu(value, value); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); } }; +simdutf_really_inline simd8::operator simd8() const { + return this->value; +} template struct simd8x64 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert( - NUM_CHUNKS == 4, - "LoongArch kernel should use four registers per 64-byte block."); + static_assert(NUM_CHUNKS == 2, + "LASX kernel should use two registers per 64-byte block."); simd8 chunks[NUM_CHUNKS]; simd8x64(const simd8x64 &o) = delete; // no copy allowed @@ -10648,36 +10187,36 @@ template struct simd8x64 { operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) + : chunks{chunk0, chunk1} {} simdutf_really_inline simd8x64(const T *ptr) : chunks{simd8::load(ptr), - simd8::load(ptr + sizeof(simd8) / sizeof(T)), - simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), - simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} + simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); - this->chunks[2].store(ptr + sizeof(simd8) * 2 / sizeof(T)); - this->chunks[3].store(ptr + sizeof(simd8) * 3 / sizeof(T)); + } + + simdutf_really_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); } simdutf_really_inline simd8x64 &operator|=(const simd8x64 &other) { this->chunks[0] |= other.chunks[0]; this->chunks[1] |= other.chunks[1]; - this->chunks[2] |= other.chunks[2]; - this->chunks[3] |= other.chunks[3]; return *this; } simdutf_really_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); + return this->chunks[0] | this->chunks[1]; } - simdutf_really_inline bool is_ascii() const { return reduce_or().is_ascii(); } + simdutf_really_inline bool is_ascii() const { + return this->reduce_or().is_ascii(); + } template simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { @@ -10685,152 +10224,123 @@ template struct simd8x64 { sizeof(simd8) * 0); this->chunks[1].template store_ascii_as_utf16(ptr + sizeof(simd8) * 1); - this->chunks[2].template store_ascii_as_utf16(ptr + - sizeof(simd8) * 2); - this->chunks[3].template store_ascii_as_utf16(ptr + - sizeof(simd8) * 3); } simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - this->chunks[0].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 0); - this->chunks[1].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 1); - this->chunks[2].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 2); - this->chunks[3].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 3); + this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); + this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); } - simdutf_really_inline uint64_t to_bitmask() const { - __m128i mask = __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[3]), 6); - mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[2]), 4)); - mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[1]), 2)); - mask = __lsx_vor_v(mask, __lsx_vmsknz_b(this->chunks[0])); - return __lsx_vpickve2gr_du(mask, 0); + simdutf_really_inline uint64_t lt(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask) + .to_bitmask(); } - simdutf_really_inline uint64_t lt(const T m) const { + simdutf_really_inline uint64_t gteq(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, - this->chunks[2] < mask, this->chunks[3] < mask) + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask) .to_bitmask(); } + simdutf_really_inline uint64_t gt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask, - this->chunks[2] > mask, this->chunks[3] > mask) + return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask) .to_bitmask(); } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); - return simd8x64(simd8(this->chunks[0].value) >= mask, - simd8(this->chunks[1].value) >= mask, - simd8(this->chunks[2].value) >= mask, - simd8(this->chunks[3].value) >= mask) + return simd8x64((simd8(__m256i(this->chunks[0])) >= mask), + (simd8(__m256i(this->chunks[1])) >= mask)) .to_bitmask(); } }; // struct simd8x64 -/* begin file src/simdutf/lsx/simd16-inl.h */ +/* begin file src/simdutf/lasx/simd16-inl.h */ template struct simd16; -template > struct base_u16 { - __m128i value; - static const size_t SIZE = sizeof(value); - static const size_t ELEMENTS = sizeof(value) / sizeof(T); - - // Conversion from/to SIMD register - simdutf_really_inline base_u16() = default; - simdutf_really_inline base_u16(const __m128i _value) : value(_value) {} - // Bit operations - simdutf_really_inline simd16 operator|(const simd16 other) const { - return __lsx_vor_v(this->value, other.value); - } - simdutf_really_inline simd16 operator&(const simd16 other) const { - return __lsx_vand_v(this->value, other.value); - } - simdutf_really_inline simd16 operator~() const { - return __lsx_vxori_b(this->value, 0xFF); - } - - friend simdutf_really_inline Mask operator==(const simd16 lhs, - const simd16 rhs) { - return __lsx_vseq_h(lhs.value, rhs.value); - } - - template - simdutf_really_inline simd16 byte_right_shift() const { - return __lsx_vbsrl_v(this->value, N); - } - - simdutf_really_inline uint16_t first() const { - return uint16_t(__lsx_vpickve2gr_w(value, 0)); - } -}; - template > -struct base16 : base_u16 { - using bitmask_type = uint16_t; +struct base16 : base> { + using bitmask_type = uint32_t; - simdutf_really_inline base16() : base_u16() {} - simdutf_really_inline base16(const __m128i _value) : base_u16(_value) {} + simdutf_really_inline base16() : base>() {} + simdutf_really_inline base16(const __m256i _value) + : base>(_value) {} template simdutf_really_inline base16(const Pointer *ptr) - : base16(__lsx_vld(ptr, 0)) {} + : base16(__lasx_xvld(reinterpret_cast(ptr), 0)) {} - static const int SIZE = sizeof(base_u16::value); + /// the size of vector in bytes + static const int SIZE = sizeof(base>::value); - template - simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(*this, N * 2), - __lsx_vbsrl_v(prev_chunk, 16 - N * 2)); - } + /// the number of elements of type T a vector can hold + static const int ELEMENTS = SIZE / sizeof(T); }; // SIMD byte mask type (returned by things like eq and gt) template <> struct simd16 : base16 { static simdutf_really_inline simd16 splat(bool _value) { - return __lsx_vreplgr2vr_h(uint16_t(-(!!_value))); + return __lasx_xvreplgr2vr_h(uint16_t(-(!!_value))); } simdutf_really_inline simd16() : base16() {} - simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} + simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} + // Splat constructor + simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} simdutf_really_inline bitmask_type to_bitmask() const { - __m128i mask = __lsx_vmsknz_b(this->value); - bitmask_type mask0 = bitmask_type(__lsx_vpickve2gr_wu(mask, 0)); - return mask0; + __m256i mask = __lasx_xvmsknz_b(this->value); + bitmask_type mask0 = __lasx_xvpickve2gr_wu(mask, 0); + bitmask_type mask1 = __lasx_xvpickve2gr_wu(mask, 4); + return (mask0 | (mask1 << 16)); } + simdutf_really_inline simd16 operator~() const { return *this ^ true; } - simdutf_really_inline bool is_zero() const { return __lsx_bz_v(this->value); } + simdutf_really_inline bool is_zero() const { + return __lasx_xbz_v(this->value); + } + + template simdutf_really_inline simd16 byte_right_shift() const { + const auto t0 = __lasx_xvbsrl_v(this->value, N); + const auto t1 = __lasx_xvpermi_q(this->value, __lasx_xvldi(0), 0b00000011); + const auto t2 = __lasx_xvbsll_v(t1, 16 - N); + const auto t3 = __lasx_xvor_v(t0, t2); + return t3; + } + + simdutf_really_inline uint16_t first() const { + return uint16_t(__lasx_xvpickve2gr_w(value, 0)); + } }; template struct base16_numeric : base16 { static simdutf_really_inline simd16 splat(T _value) { - return __lsx_vreplgr2vr_h(_value); + return __lasx_xvreplgr2vr_h((uint16_t)_value); } - static simdutf_really_inline simd16 zero() { return __lsx_vldi(0); } - + static simdutf_really_inline simd16 zero() { return __lasx_xvldi(0); } template static simdutf_really_inline simd16 load(const Pointer values) { - return __lsx_vld(values, 0); + return __lasx_xvld(values, 0); } - simdutf_really_inline base16_numeric(const __m128i _value) + simdutf_really_inline base16_numeric() : base16() {} + simdutf_really_inline base16_numeric(const __m256i _value) : base16(_value) {} // Store to array simdutf_really_inline void store(T dst[8]) const { - return __lsx_vst(this->value, dst, 0); + return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); } // Override to distinguish from bool version - simdutf_really_inline simd16 operator~() const { - return __lsx_vxori_b(this->value, 0xFF); - } + simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFFFu; } }; -// Unsigned code unitstemplate<> +// Unsigned code units template <> struct simd16 : base16_numeric { - simdutf_really_inline simd16(const __m128i _value) - : base16_numeric((__m128i)_value) {} + simdutf_really_inline simd16() : base16_numeric() {} + simdutf_really_inline simd16(const __m256i _value) + : base16_numeric(_value) {} // Splat constructor simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} @@ -10840,67 +10350,71 @@ template <> struct simd16 : base16_numeric { simdutf_really_inline simd16(const char16_t *values) : simd16(load(reinterpret_cast(values))) {} - // Copy constructor - simdutf_really_inline simd16(const simd16 mask) : simd16(mask.value) {} - // Order-specific operations simdutf_really_inline simd16 &operator+=(const simd16 other) { - value = __lsx_vadd_h(value, other.value); + value = __lasx_xvadd_h(value, other.value); return *this; } + // Change the endianness + simdutf_really_inline simd16 swap_bytes() const { + return __lasx_xvshuf4i_b(this->value, 0b10110001); + } + template static simdutf_really_inline simd8 pack_shifted_right(const simd16 &v0, const simd16 &v1) { - return __lsx_vssrlni_bu_h(v1.value, v0.value, N); + return __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(v1.value, v0.value, N), + 0b11011000); } // Pack with the unsigned saturation of two uint16_t code units into single // uint8_t vector static simdutf_really_inline simd8 pack(const simd16 &v0, const simd16 &v1) { - return pack_shifted_right<0>(v0, v1); - } - // Change the endianness - simdutf_really_inline simd16 swap_bytes() const { - return __lsx_vshuf4i_b(this->value, 0b10110001); + return pack_shifted_right<0>(v0, v1); } simdutf_really_inline uint64_t sum() const { - const auto sum_u32 = __lsx_vhaddw_wu_hu(value, value); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(value, value); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); - return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + - uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); + return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + } + + template simdutf_really_inline simd16 byte_right_shift() const { + return __lasx_xvbsrl_v(this->value, N); } }; simdutf_really_inline simd16 operator<(const simd16 a, const simd16 b) { - return __lsx_vslt_hu(a.value, b.value); + return __lasx_xvslt_hu(a.value, b.value); } simdutf_really_inline simd16 operator>(const simd16 a, const simd16 b) { - return __lsx_vslt_hu(b.value, a.value); + return __lasx_xvslt_hu(b.value, a.value); } simdutf_really_inline simd16 operator<=(const simd16 a, const simd16 b) { - return __lsx_vsle_hu(a.value, b.value); + return __lasx_xvsle_hu(a.value, b.value); } simdutf_really_inline simd16 operator>=(const simd16 a, const simd16 b) { - return __lsx_vsle_hu(b.value, a.value); + return __lasx_xvsle_hu(b.value, a.value); } template struct simd16x32 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); - static_assert( - NUM_CHUNKS == 4, - "LOONGARCH kernel should use four registers per 64-byte block."); + static_assert(NUM_CHUNKS == 2, + "LASX kernel should use two registers per 64-byte block."); simd16 chunks[NUM_CHUNKS]; simd16x32(const simd16x32 &o) = delete; // no copy allowed @@ -10908,127 +10422,144 @@ template struct simd16x32 { operator=(const simd16 other) = delete; // no assignment allowed simd16x32() = delete; // no default constructor allowed - simdutf_really_inline - simd16x32(const simd16 chunk0, const simd16 chunk1, - const simd16 chunk2, const simd16 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdutf_really_inline simd16x32(const simd16 chunk0, + const simd16 chunk1) + : chunks{chunk0, chunk1} {} simdutf_really_inline simd16x32(const T *ptr) : chunks{simd16::load(ptr), - simd16::load(ptr + sizeof(simd16) / sizeof(T)), - simd16::load(ptr + 2 * sizeof(simd16) / sizeof(T)), - simd16::load(ptr + 3 * sizeof(simd16) / sizeof(T))} {} + simd16::load(ptr + sizeof(simd16) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd16) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd16) * 1 / sizeof(T)); - this->chunks[2].store(ptr + sizeof(simd16) * 2 / sizeof(T)); - this->chunks[3].store(ptr + sizeof(simd16) * 3 / sizeof(T)); } simdutf_really_inline void swap_bytes() { this->chunks[0] = this->chunks[0].swap_bytes(); this->chunks[1] = this->chunks[1].swap_bytes(); - this->chunks[2] = this->chunks[2].swap_bytes(); - this->chunks[3] = this->chunks[3].swap_bytes(); } simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask) + .to_bitmask(); } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); - return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) + return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask) .to_bitmask(); } }; // struct simd16x32 -simdutf_really_inline simd16 operator^(const simd16 a, +simdutf_really_inline simd16 min(const simd16 a, + const simd16 b) { + return __lasx_xvmin_hu(a.value, b.value); +} + +simdutf_really_inline simd16 operator==(const simd16 a, + uint16_t b) { + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvseq_h(a.value, bv); +} + +simdutf_really_inline simd16 as_vector_u16(const simd16 x) { + return x.value; +} + +simdutf_really_inline simd16 operator&(const simd16 a, uint16_t b) { - const auto bv = __lsx_vreplgr2vr_h(b); - return __lsx_vxor_v(a.value, bv); + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvand_v(a.value, bv); } -simdutf_really_inline simd16 operator^(const simd16 a, - const simd16 b) { - return __lsx_vxor_v(a.value, b.value); +simdutf_really_inline simd16 operator&(const simd16 a, + const simd16 b) { + return __lasx_xvand_v(a.value, b.value); } -simdutf_really_inline simd16 min(const simd16 a, - const simd16 b) { - return __lsx_vmin_hu(a.value, b.value); +simdutf_really_inline simd16 operator^(const simd16 a, + uint16_t b) { + const auto bv = __lasx_xvreplgr2vr_h(b); + return __lasx_xvxor_v(a.value, bv); } -simdutf_really_inline simd16 as_vector_u16(const simd16 x) { - return x.value; +simdutf_really_inline simd16 operator^(const simd16 a, + const simd16 b) { + return __lasx_xvxor_v(a.value, b.value); } -/* end file src/simdutf/lsx/simd16-inl.h */ -/* begin file src/simdutf/lsx/simd32-inl.h */ +/* end file src/simdutf/lasx/simd16-inl.h */ +/* begin file src/simdutf/lasx/simd32-inl.h */ template struct simd32; template <> struct simd32 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint32_t); // constructors - simdutf_really_inline simd32(__m128i v) : value(v) {} + simdutf_really_inline simd32(__m256i v) : value(v) {} template - simdutf_really_inline simd32(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} + simdutf_really_inline simd32(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} // in-place operators simdutf_really_inline simd32 &operator-=(const simd32 other) { - value = __lsx_vsub_w(value, other.value); + value = __lasx_xvsub_w(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lsx_vpickve2gr_wu(value, 0)) + - uint64_t(__lsx_vpickve2gr_wu(value, 1)) + - uint64_t(__lsx_vpickve2gr_wu(value, 2)) + - uint64_t(__lsx_vpickve2gr_wu(value, 3)); + const auto odd = __lasx_xvsrli_d(value, 32); + const auto even = __lasx_xvand_v(value, __lasx_xvreplgr2vr_d(0xffffffff)); + + const auto sum64 = __lasx_xvadd_d(odd, even); + + return uint64_t(__lasx_xvpickve2gr_du(sum64, 0)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 1)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 2)) + + uint64_t(__lasx_xvpickve2gr_du(sum64, 3)); } // static members static simdutf_really_inline simd32 splat(uint32_t x) { - return __lsx_vreplgr2vr_w(x); + return __lasx_xvreplgr2vr_w(x); } static simdutf_really_inline simd32 zero() { - return __lsx_vrepli_w(0); + return __lasx_xvrepli_w(0); } }; // ------------------------------------------------------------ template <> struct simd32 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd32(__m128i v) : value(v) {} + simdutf_really_inline simd32(__m256i v) : value(v) {} }; // ------------------------------------------------------------ simdutf_really_inline simd32 operator&(const simd32 a, const simd32 b) { - return __lsx_vor_v(a.value, b.value); + return __lasx_xvor_v(a.value, b.value); } simdutf_really_inline simd32 operator<(const simd32 a, const simd32 b) { - return __lsx_vslt_wu(a.value, b.value); + return __lasx_xvslt_wu(a.value, b.value); } simdutf_really_inline simd32 operator>(const simd32 a, const simd32 b) { - return __lsx_vslt_wu(b.value, a.value); + return __lasx_xvslt_wu(b.value, a.value); } // ------------------------------------------------------------ @@ -11036,112 +10567,128 @@ simdutf_really_inline simd32 operator>(const simd32 a, simdutf_really_inline simd32 as_vector_u32(const simd32 v) { return v.value; } -/* end file src/simdutf/lsx/simd32-inl.h */ -/* begin file src/simdutf/lsx/simd64-inl.h */ +/* end file src/simdutf/lasx/simd32-inl.h */ +/* begin file src/simdutf/lasx/simd64-inl.h */ template struct simd64; template <> struct simd64 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint64_t); // constructors - simdutf_really_inline simd64(__m128i v) : value(v) {} + simdutf_really_inline simd64(__m256i v) : value(v) {} template - simdutf_really_inline simd64(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} + simdutf_really_inline simd64(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} // in-place operators simdutf_really_inline simd64 &operator+=(const simd64 other) { - value = __lsx_vadd_d(value, other.value); + value = __lasx_xvadd_d(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lsx_vpickve2gr_du(value, 0)) + - uint64_t(__lsx_vpickve2gr_du(value, 1)); + return uint64_t(__lasx_xvpickve2gr_du(value, 0)) + + uint64_t(__lasx_xvpickve2gr_du(value, 1)) + + uint64_t(__lasx_xvpickve2gr_du(value, 2)) + + uint64_t(__lasx_xvpickve2gr_du(value, 3)); } // static members static simdutf_really_inline simd64 zero() { - return __lsx_vrepli_d(0); + return __lasx_xvrepli_d(0); } }; // ------------------------------------------------------------ template <> struct simd64 { - __m128i value; + __m256i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd64(__m128i v) : value(v) {} + simdutf_really_inline simd64(__m256i v) : value(v) {} }; // ------------------------------------------------------------ simd64 sum_8bytes(const simd8 v) { - const auto sum_u16 = __lsx_vhaddw_hu_bu(v, v); - const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u16 = __lasx_xvhaddw_hu_bu(v, v); + const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); return simd64(sum_u64); } -/* end file src/simdutf/lsx/simd64-inl.h */ +/* end file src/simdutf/lasx/simd64-inl.h */ } // namespace simd } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf -#endif // SIMDUTF_LSX_SIMD_H -/* end file src/simdutf/lsx/simd.h */ +#endif // SIMDUTF_LASX_SIMD_H +/* end file src/simdutf/lasx/simd.h */ -/* begin file src/simdutf/lsx/end.h */ +/* begin file src/simdutf/lasx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lsx/end.h */ -#endif // SIMDUTF_IMPLEMENTATION_LSX +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_UNTARGET_REGION +#endif +/* end file src/simdutf/lasx/end.h */ -#endif // SIMDUTF_LSX_H -/* end file src/simdutf/lsx.h */ -/* begin file src/simdutf/lasx.h */ -#ifndef SIMDUTF_LASX_H -#define SIMDUTF_LASX_H +#endif // SIMDUTF_IMPLEMENTATION_LASX + +#endif // SIMDUTF_LASX_H +/* end file src/simdutf/lasx.h */ +/* begin file src/simdutf/lsx.h */ +#ifndef SIMDUTF_LSX_H +#define SIMDUTF_LSX_H #ifdef SIMDUTF_FALLBACK_H - #error "lasx.h must be included before fallback.h" + #error "lsx.h must be included before fallback.h" +#endif + +#ifndef SIMDUTF_CAN_ALWAYS_RUN_LASX + #error "lsx.h must be included after lasx.h" #endif -#ifndef SIMDUTF_IMPLEMENTATION_LASX - #define SIMDUTF_IMPLEMENTATION_LASX (SIMDUTF_IS_LASX) +#ifndef SIMDUTF_IMPLEMENTATION_LSX + #if SIMDUTF_CAN_ALWAYS_RUN_LASX + #define SIMDUTF_IMPLEMENTATION_LSX 0 + #else + #define SIMDUTF_IMPLEMENTATION_LSX (SIMDUTF_IS_LSX) + #endif #endif -#if SIMDUTF_IMPLEMENTATION_LASX && SIMDUTF_IS_LASX - #define SIMDUTF_CAN_ALWAYS_RUN_LASX 1 +#if SIMDUTF_IMPLEMENTATION_LSX && SIMDUTF_IS_LSX + #define SIMDUTF_CAN_ALWAYS_RUN_LSX 1 #else - #define SIMDUTF_CAN_ALWAYS_RUN_LASX 0 + #define SIMDUTF_CAN_ALWAYS_RUN_LSX 0 #endif #define SIMDUTF_CAN_ALWAYS_RUN_FALLBACK (SIMDUTF_IMPLEMENTATION_FALLBACK) -#if SIMDUTF_IMPLEMENTATION_LASX +#if SIMDUTF_IMPLEMENTATION_LSX namespace simdutf { /** - * Implementation for LoongArch ASX. + * Implementation for LoongArch SX. */ -namespace lasx {} // namespace lasx +namespace lsx {} // namespace lsx } // namespace simdutf -/* begin file src/simdutf/lasx/implementation.h */ -#ifndef SIMDUTF_LASX_IMPLEMENTATION_H -#define SIMDUTF_LASX_IMPLEMENTATION_H +/* begin file src/simdutf/lsx/implementation.h */ +#ifndef SIMDUTF_LSX_IMPLEMENTATION_H +#define SIMDUTF_LSX_IMPLEMENTATION_H namespace simdutf { -namespace lasx { +namespace lsx { namespace { using namespace simdutf; @@ -11150,9 +10697,8 @@ using namespace simdutf; class implementation final : public simdutf::implementation { public: simdutf_really_inline implementation() - : simdutf::implementation("lasx", "LOONGARCH ASX", - internal::instruction_set::LSX | - internal::instruction_set::LASX) {} + : simdutf::implementation("lsx", "LOONGARCH SX", + internal::instruction_set::LSX) {} #if SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused int detect_encodings(const char *input, size_t length) const noexcept final; @@ -11344,200 +10890,123 @@ class implementation final : public simdutf::implementation { #if SIMDUTF_FEATURE_UTF16 void change_endianness_utf16(const char16_t *buf, size_t length, char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; + size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; + const char16_t *input, size_t length) const noexcept override; ; + + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_BASE64 simdutf_warn_unused result base64_to_binary( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; simdutf_warn_unused full_result base64_to_binary_details( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; + last_chunk_handling_options::loose) const noexcept override; size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; const char *find(const char *start, const char *end, - char character) const noexcept; + char character) const noexcept override; const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; + char16_t character) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override; #endif // SIMDUTF_FEATURE_BASE64 }; -} // namespace lasx +} // namespace lsx } // namespace simdutf -#endif // SIMDUTF_LASX_IMPLEMENTATION_H -/* end file src/simdutf/lasx/implementation.h */ +#endif // SIMDUTF_LSX_IMPLEMENTATION_H +/* end file src/simdutf/lsx/implementation.h */ -/* begin file src/simdutf/lasx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lasx" -// #define SIMDUTF_IMPLEMENTATION lasx +/* begin file src/simdutf/lsx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lsx" +// #define SIMDUTF_IMPLEMENTATION lsx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lasx/begin.h */ +/* end file src/simdutf/lsx/begin.h */ // Declarations -/* begin file src/simdutf/lasx/intrinsics.h */ -#ifndef SIMDUTF_LASX_INTRINSICS_H -#define SIMDUTF_LASX_INTRINSICS_H +/* begin file src/simdutf/lsx/intrinsics.h */ +#ifndef SIMDUTF_LSX_INTRINSICS_H +#define SIMDUTF_LSX_INTRINSICS_H // This should be the correct header whether // you use visual studio or other compilers. #include -#include - -#if defined(__loongarch_asx) - #ifdef __clang__ - #define VREGS_PREFIX "$vr" - #define XREGS_PREFIX "$xr" - #else // GCC - #define VREGS_PREFIX "$f" - #define XREGS_PREFIX "$f" - #endif - #define __ALL_REGS \ - "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26," \ - "27,28,29,30,31" -// Convert __m128i to __m256i -static inline __m256i ____m256i(__m128i in) { - __m256i out = __lasx_xvldi(0); - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " XREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " VREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - : [out] "+f"(out) - : [in] "f"(in)); - return out; -} -// Convert two __m128i to __m256i -static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) { - __m256i out; - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[hi], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[lo], " VREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".ifnc %[out], %[hi] \n\t" - ".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " XREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[hi], " VREGS_PREFIX "\\j \n\t" - " xvori.b $xr\\i, $xr\\j, 0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".endif \n\t" - : [out] "=f"(out), [hi] "+f"(inhi) - : [lo] "f"(inlo)); - return out; -} -// Convert __m256i low part to __m128i -static inline __m128i lasx_extracti128_lo(__m256i in) { - __m128i out; - __asm__ volatile(".ifnc %[out], %[in] \n\t" - ".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " XREGS_PREFIX "\\j \n\t" - " vori.b $vr\\i, $vr\\j, 0 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - ".endif \n\t" - : [out] "=f"(out) - : [in] "f"(in)); - return out; -} -// Convert __m256i high part to __m128i -static inline __m128i lasx_extracti128_hi(__m256i in) { - __m128i out; - __asm__ volatile(".irp i," __ALL_REGS "\n\t" - " .ifc %[out], " VREGS_PREFIX "\\i \n\t" - " .irp j," __ALL_REGS "\n\t" - " .ifc %[in], " XREGS_PREFIX "\\j \n\t" - " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t" - " .endif \n\t" - " .endr \n\t" - " .endif \n\t" - ".endr \n\t" - : [out] "=f"(out) - : [in] "f"(in)); - return out; -} -#endif /* Encoding of argument for LoongArch64 xvldi instruction. See: @@ -11567,7 +11036,7 @@ all lanes the result as 64-bit elements to all lanes */ -namespace lasx_vldi { +namespace vldi { template class const_u16 { constexpr static const uint8_t b0 = ((v >> 0 * 8) & 0xff); @@ -11684,8 +11153,7 @@ template class const_u64 { constexpr static int value = int((operation << 8) | byte) - 8192; constexpr static bool valid = operation != 0xffff; }; - -} // namespace lasx_vldi +} // namespace vldi // Uncomment when running under QEMU affected // by bug https://gitlab.com/qemu-project/qemu/-/issues/2865 @@ -11694,46 +11162,48 @@ template class const_u64 { // #define QEMU_VLDI_BUG 1 #endif -#ifdef QEMU_VLDI_BUG - #define lasx_splat_u16(v) __lasx_xvreplgr2vr_h(v) - #define lasx_splat_u32(v) __lasx_xvreplgr2vr_w(v) -#else -template constexpr __m256i lasx_splat_u16_aux() { - constexpr bool is_imm10 = (int16_t(x) < 512) && (int16_t(x) > -512); - constexpr uint16_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = lasx_vldi::const_u16::valid; - constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u16::value : 0; - - return is_imm10 ? __lasx_xvrepli_h(int16_t(imm10)) - : is_vldi ? __lasx_xvldi(vldi_imm) - : __lasx_xvreplgr2vr_h(x); +#ifndef lsx_splat_u16 + #ifdef QEMU_VLDI_BUG + #define lsx_splat_u16(v) __lsx_vreplgr2vr_h(v) + #define lsx_splat_u32(v) __lsx_vreplgr2vr_w(v) + #else +namespace { +template constexpr __m128i lsx_splat_u16_aux() { + return ((int16_t(x) < 512) && (int16_t(x) > -512)) + ? __lsx_vrepli_h( + ((int16_t(x) < 512) && (int16_t(x) > -512)) ? int16_t(x) : 0) + : (vldi::const_u16::valid + ? __lsx_vldi(vldi::const_u16::valid + ? vldi::const_u16::value + : 0) + : __lsx_vreplgr2vr_h(x)); } -template constexpr __m256i lasx_splat_u32_aux() { - constexpr bool is_imm10 = (int32_t(x) < 512) && (int32_t(x) > -512); - constexpr uint32_t imm10 = is_imm10 ? x : 0; - constexpr bool is_vldi = lasx_vldi::const_u32::valid; - constexpr int vldi_imm = is_vldi ? lasx_vldi::const_u32::value : 0; - - return is_imm10 ? __lasx_xvrepli_w(int32_t(imm10)) - : is_vldi ? __lasx_xvldi(vldi_imm) - : __lasx_xvreplgr2vr_w(x); +template constexpr __m128i lsx_splat_u32_aux() { + return ((int32_t(x) < 512) && (int32_t(x) > -512)) + ? __lsx_vrepli_w( + ((int32_t(x) < 512) && (int32_t(x) > -512)) ? int32_t(x) : 0) + : (vldi::const_u32::valid + ? __lsx_vldi(vldi::const_u32::valid + ? vldi::const_u32::value + : 0) + : __lsx_vreplgr2vr_w(x)); } - - #define lasx_splat_u16(v) lasx_splat_u16_aux<(v)>() - #define lasx_splat_u32(v) lasx_splat_u32_aux<(v)>() -#endif // QEMU_VLDI_BUG - -#endif // SIMDUTF_LASX_INTRINSICS_H -/* end file src/simdutf/lasx/intrinsics.h */ -/* begin file src/simdutf/lasx/bitmanipulation.h */ -#ifndef SIMDUTF_LASX_BITMANIPULATION_H -#define SIMDUTF_LASX_BITMANIPULATION_H +} // namespace + #define lsx_splat_u16(v) lsx_splat_u16_aux<(v)>() + #define lsx_splat_u32(v) lsx_splat_u32_aux<(v)>() + #endif // QEMU_VLDI_BUG +#endif // lsx_splat_u16 +#endif // SIMDUTF_LSX_INTRINSICS_H +/* end file src/simdutf/lsx/intrinsics.h */ +/* begin file src/simdutf/lsx/bitmanipulation.h */ +#ifndef SIMDUTF_LSX_BITMANIPULATION_H +#define SIMDUTF_LSX_BITMANIPULATION_H #include namespace simdutf { -namespace lasx { +namespace lsx { namespace { simdutf_really_inline int count_ones(uint64_t input_num) { @@ -11747,349 +11217,171 @@ simdutf_really_inline int trailing_zeroes(uint64_t input_num) { #endif } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf -#endif // SIMDUTF_LASX_BITMANIPULATION_H -/* end file src/simdutf/lasx/bitmanipulation.h */ -/* begin file src/simdutf/lasx/simd.h */ -#ifndef SIMDUTF_LASX_SIMD_H -#define SIMDUTF_LASX_SIMD_H +#endif // SIMDUTF_LSX_BITMANIPULATION_H +/* end file src/simdutf/lsx/bitmanipulation.h */ +/* begin file src/simdutf/lsx/simd.h */ +#ifndef SIMDUTF_LSX_SIMD_H +#define SIMDUTF_LSX_SIMD_H namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace simd { -__attribute__((aligned(32))) static const uint8_t prev_shuf_table[32][32] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, - {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, - 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, - {0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, - {0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - {0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, - {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, - 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, - 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0}, - {15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0}, - {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0}, - {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0}, - {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0}, - {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0}, - {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0}, - {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, -}; - -__attribute__((aligned(32))) static const uint8_t bitsel_mask_table[32][32] = { - {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x0}, - {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0}}; - -// Forward-declared so they can be used by splat and friends. -template struct base { - __m256i value; - - // Zero constructor - simdutf_really_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdutf_really_inline base(const __m256i _value) : value(_value) {} - // Conversion to SIMD register - simdutf_really_inline operator const __m256i &() const { return this->value; } - simdutf_really_inline operator __m256i &() { return this->value; } - template - simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { - if (big_endian) { - __m256i zero = __lasx_xvldi(0); - __m256i in8 = __lasx_xvpermi_d(this->value, 0b11011000); - __m256i inlow = __lasx_xvilvl_b(in8, zero); - __m256i inhigh = __lasx_xvilvh_b(in8, zero); - __lasx_xvst(inlow, reinterpret_cast(ptr), 0); - __lasx_xvst(inhigh, reinterpret_cast(ptr), 32); - } else { - __m256i inlow = __lasx_vext2xv_hu_bu(this->value); - __m256i inhigh = __lasx_vext2xv_hu_bu( - __lasx_xvpermi_q(this->value, this->value, 0b00000001)); - __lasx_xvst(inlow, reinterpret_cast<__m256i *>(ptr), 0); - __lasx_xvst(inhigh, reinterpret_cast<__m256i *>(ptr), 32); - } - } - simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - __m256i in32_0 = __lasx_vext2xv_wu_bu(this->value); - __lasx_xvst(in32_0, reinterpret_cast(ptr), 0); +template struct simd8; - __m256i in8_1 = __lasx_xvpermi_d(this->value, 0b00000001); - __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); - __lasx_xvst(in32_1, reinterpret_cast(ptr), 32); +// +// Base class of simd8 and simd8, both of which use __m128i +// internally. +// +template > struct base_u8 { + __m128i value; + static const int SIZE = sizeof(value); - __m256i in8_2 = __lasx_xvpermi_d(this->value, 0b00000010); - __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); - __lasx_xvst(in32_2, reinterpret_cast(ptr), 64); + // Conversion from/to SIMD register + simdutf_really_inline base_u8(const __m128i _value) : value(_value) {} + simdutf_really_inline operator const __m128i &() const { return this->value; } + simdutf_really_inline operator __m128i &() { return this->value; } - __m256i in8_3 = __lasx_xvpermi_d(this->value, 0b00000011); - __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); - __lasx_xvst(in32_3, reinterpret_cast(ptr), 96); - } // Bit operations - simdutf_really_inline Child operator|(const Child other) const { - return __lasx_xvor_v(this->value, other); + simdutf_really_inline simd8 operator|(const simd8 other) const { + return __lsx_vor_v(this->value, other); } - simdutf_really_inline Child operator&(const Child other) const { - return __lasx_xvand_v(this->value, other); + simdutf_really_inline simd8 operator&(const simd8 other) const { + return __lsx_vand_v(this->value, other); } - simdutf_really_inline Child operator^(const Child other) const { - return __lasx_xvxor_v(this->value, other); + simdutf_really_inline simd8 operator^(const simd8 other) const { + return __lsx_vxor_v(this->value, other); } - simdutf_really_inline Child &operator|=(const Child other) { - auto this_cast = static_cast(this); + simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline simd8 &operator|=(const simd8 other) { + auto this_cast = static_cast *>(this); *this_cast = *this_cast | other; return *this_cast; } -}; - -template struct simd8; -template > -struct base8 : base> { - simdutf_really_inline base8() : base>() {} - simdutf_really_inline base8(const __m256i _value) : base>(_value) {} friend simdutf_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return __lasx_xvseq_b(lhs, rhs); + return __lsx_vseq_b(lhs, rhs); } - static const int SIZE = sizeof(base::value); - - template + template simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { - static_assert(N <= 16, "unsupported shift value"); + return __lsx_vor_v(__lsx_vbsll_v(this->value, N), + __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + } +}; - if (!N) - return this->value; +// SIMD byte mask type (returned by things like eq and gt) +template <> struct simd8 : base_u8 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; - __m256i zero = __lasx_xvldi(0); - __m256i result, shuf; - if (N < 16) { - shuf = __lasx_xvld(prev_shuf_table[N], 0); + static simdutf_really_inline simd8 splat(bool _value) { + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + } - result = __lasx_xvshuf_b( - __lasx_xvpermi_q(this->value, this->value, 0b00000001), this->value, - shuf); - __m256i srl_prev = __lasx_xvbsrl_v( - __lasx_xvpermi_q(zero, prev_chunk.value, 0b00110001), (16 - N)); - __m256i mask = __lasx_xvld(bitsel_mask_table[N], 0); - result = __lasx_xvbitsel_v(result, srl_prev, mask); + simdutf_really_inline simd8(const __m128i _value) : base_u8(_value) {} + // False constructor + simdutf_really_inline simd8() : simd8(__lsx_vldi(0)) {} + // Splat constructor + simdutf_really_inline simd8(bool _value) : simd8(splat(_value)) {} + simdutf_really_inline void store(uint8_t dst[16]) const { + return __lsx_vst(this->value, dst, 0); + } - return result; - } else if (N == 16) { - return __lasx_xvpermi_q(this->value, prev_chunk.value, 0b00100001); - } + simdutf_really_inline uint32_t to_bitmask() const { + return __lsx_vpickve2gr_wu(__lsx_vmsknz_b(*this), 0); } }; -// SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base8 { - static simdutf_really_inline simd8 splat(bool _value) { - return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); +// Unsigned bytes +template <> struct simd8 : base_u8 { + static simdutf_really_inline simd8 splat(uint8_t _value) { + return __lsx_vreplgr2vr_b(_value); } - - simdutf_really_inline simd8() : base8() {} - simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} + static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } + static simdutf_really_inline simd8 load(const uint8_t *values) { + return __lsx_vld(values, 0); + } + simdutf_really_inline simd8(const __m128i _value) + : base_u8(_value) {} + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} + // Array constructor + simdutf_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} // Splat constructor - simdutf_really_inline simd8(bool _value) : base8(splat(_value)) {} + simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Member-by-member initialization + simdutf_really_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) + : simd8((__m128i)v16u8{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15}) {} - simdutf_really_inline uint32_t to_bitmask() const { - __m256i mask = __lasx_xvmsknz_b(this->value); - uint32_t mask0 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(mask, 4); - return (mask0 | (mask1 << 16)); + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdutf_really_inline static simd8 + repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, + uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, + uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, + uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); } - simdutf_really_inline bool any() const { - if (__lasx_xbz_b(this->value)) - return false; - return true; + + // Store to array + simdutf_really_inline void store(uint8_t dst[16]) const { + return __lsx_vst(this->value, dst, 0); } - simdutf_really_inline simd8 operator~() const { return *this ^ true; } -}; -template struct base8_numeric : base8 { - static simdutf_really_inline simd8 splat(T _value) { - return __lasx_xvreplgr2vr_b(_value); + // Order-specific operations + simdutf_really_inline simd8 + operator>=(const simd8 other) const { + return __lsx_vsle_bu(other, *this); } - static simdutf_really_inline simd8 zero() { return __lasx_xvldi(0); } - static simdutf_really_inline simd8 load(const T values[32]) { - return __lasx_xvld(reinterpret_cast(values), 0); + simdutf_really_inline simd8 + operator>(const simd8 other) const { + return __lsx_vslt_bu(other, *this); } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdutf_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, - v12, v13, v14, v15); + simdutf_really_inline simd8 &operator-=(const simd8 other) { + value = __lsx_vsub_b(value, other.value); + return *this; + } + // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true + // = nonzero. For ARM, returns all 1's. + simdutf_really_inline simd8 + gt_bits(const simd8 other) const { + return simd8(*this > other); } - simdutf_really_inline base8_numeric() : base8() {} - simdutf_really_inline base8_numeric(const __m256i _value) - : base8(_value) {} - - // Store to array - simdutf_really_inline void store(T dst[32]) const { - return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); + // Bit-specific operations + simdutf_really_inline simd8 any_bits_set(simd8 bits) const { + return __lsx_vslt_bu(__lsx_vldi(0), __lsx_vand_v(this->value, bits)); + } + simdutf_really_inline bool is_ascii() const { + return __lsx_vpickve2gr_hu(__lsx_vmskgez_b(this->value), 0) == 0xFFFF; } - // Override to distinguish from bool version - simdutf_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdutf_really_inline bool any_bits_set_anywhere() const { + return __lsx_vpickve2gr_hu(__lsx_vmsknz_b(this->value), 0) > 0; + } + template simdutf_really_inline simd8 shr() const { + return __lsx_vsrli_b(this->value, N); + } + template simdutf_really_inline simd8 shl() const { + return __lsx_vslli_b(this->value, N); + } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) template simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { - __m256i origin = __lasx_xvand_v(this->value, __lasx_xvldi(0x1f)); - return __lasx_xvshuf_b(__lasx_xvldi(0), lookup_table, origin); + return lookup_table.apply_lookup_16_to(*this); } template @@ -12103,115 +11395,150 @@ template struct base8_numeric : base8 { replace7, replace8, replace9, replace10, replace11, replace12, replace13, replace14, replace15)); } + + template + simdutf_really_inline simd8 + apply_lookup_16_to(const simd8 original) const { + __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); + return __lsx_vshuf_b(__lsx_vldi(0), *this, simd8(original_tmp)); + } + + simdutf_really_inline uint64_t sum_bytes() const { + const auto sum_u16 = __lsx_vhaddw_hu_bu(value, value); + const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + + uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); + } }; // Signed bytes -template <> struct simd8 : base8_numeric { - simdutf_really_inline simd8() : base8_numeric() {} - simdutf_really_inline simd8(const __m256i _value) - : base8_numeric(_value) {} +template <> struct simd8 { + __m128i value; - // Splat constructor - simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdutf_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - simdutf_really_inline operator simd8() const; - simdutf_really_inline bool is_ascii() const { - __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); - if (__lasx_xbnz_v(ascii_mask)) - return false; - return true; + static const int SIZE = sizeof(value); + + static simdutf_really_inline simd8 splat(int8_t _value) { + return __lsx_vreplgr2vr_b(_value); } - // Order-sensitive comparisons - simdutf_really_inline simd8 operator>(const simd8 other) const { - return __lasx_xvslt_b(other, this->value); + static simdutf_really_inline simd8 zero() { return __lsx_vldi(0); } + static simdutf_really_inline simd8 load(const int8_t values[16]) { + return __lsx_vld(values, 0); } - simdutf_really_inline simd8 operator<(const simd8 other) const { - return __lasx_xvslt_b(this->value, other); + + template + simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { + __m128i zero = __lsx_vldi(0); + if constexpr (match_system(big_endian)) { + __lsx_vst(__lsx_vilvl_b(zero, (__m128i)this->value), + reinterpret_cast(p), 0); + __lsx_vst(__lsx_vilvh_b(zero, (__m128i)this->value), + reinterpret_cast(p + 8), 0); + } else { + __lsx_vst(__lsx_vilvl_b((__m128i)this->value, zero), + reinterpret_cast(p), 0); + __lsx_vst(__lsx_vilvh_b((__m128i)this->value, zero), + reinterpret_cast(p + 8), 0); + } } -}; -// Unsigned bytes -template <> struct simd8 : base8_numeric { - simdutf_really_inline simd8() : base8_numeric() {} - simdutf_really_inline simd8(const __m256i _value) - : base8_numeric(_value) {} + simdutf_really_inline void store_ascii_as_utf32(char32_t *p) const { + __m128i zero = __lsx_vldi(0); + __m128i in16low = __lsx_vilvl_b(zero, (__m128i)this->value); + __m128i in16high = __lsx_vilvh_b(zero, (__m128i)this->value); + __m128i in32_0 = __lsx_vilvl_h(zero, in16low); + __m128i in32_1 = __lsx_vilvh_h(zero, in16low); + __m128i in32_2 = __lsx_vilvl_h(zero, in16high); + __m128i in32_3 = __lsx_vilvh_h(zero, in16high); + __lsx_vst(in32_0, reinterpret_cast(p), 0); + __lsx_vst(in32_1, reinterpret_cast(p + 4), 0); + __lsx_vst(in32_2, reinterpret_cast(p + 8), 0); + __lsx_vst(in32_3, reinterpret_cast(p + 12), 0); + } + + // In places where the table can be reused, which is most uses in simdutf, it + // is worth it to do 4 table lookups, as there is no direct zero extension + // from u8 to u32. + simdutf_really_inline void store_ascii_as_utf32_tbl(char32_t *p) const { + const simd8 tb1{0, 255, 255, 255, 1, 255, 255, 255, + 2, 255, 255, 255, 3, 255, 255, 255}; + const simd8 tb2{4, 255, 255, 255, 5, 255, 255, 255, + 6, 255, 255, 255, 7, 255, 255, 255}; + const simd8 tb3{8, 255, 255, 255, 9, 255, 255, 255, + 10, 255, 255, 255, 11, 255, 255, 255}; + const simd8 tb4{12, 255, 255, 255, 13, 255, 255, 255, + 14, 255, 255, 255, 15, 255, 255, 255}; + + // encourage store pairing and interleaving + const auto shuf1 = this->apply_lookup_16_to(tb1); + const auto shuf2 = this->apply_lookup_16_to(tb2); + shuf1.store(reinterpret_cast(p)); + shuf2.store(reinterpret_cast(p + 4)); + + const auto shuf3 = this->apply_lookup_16_to(tb3); + const auto shuf4 = this->apply_lookup_16_to(tb4); + shuf3.store(reinterpret_cast(p + 8)); + shuf4.store(reinterpret_cast(p + 12)); + } + // Conversion from/to SIMD register + simdutf_really_inline simd8(const __m128i _value) : value(_value) {} + + // Zero constructor + simdutf_really_inline simd8() : simd8(zero()) {} // Splat constructor - simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdutf_really_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, - uint8_t v21, uint8_t v22, uint8_t v23, uint8_t v24, uint8_t v25, - uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, - uint8_t v31) - : simd8((__m256i)v32u8{v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31}) {} + simdutf_really_inline simd8(const int8_t *values) : simd8(load(values)) {} - // Saturated math - simdutf_really_inline simd8 - saturating_sub(const simd8 other) const { - return __lasx_xvssub_bu(this->value, other); + // Store to array + simdutf_really_inline void store(int8_t dst[16]) const { + return __lsx_vst(value, dst, 0); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdutf_really_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - simdutf_really_inline simd8 - operator>=(const simd8 other) const { - return __lasx_xvsle_bu(other, *this); + simdutf_really_inline operator simd8() const { + return ((__m128i)this->value); } - simdutf_really_inline simd8 &operator-=(const simd8 other) { - value = __lasx_xvsub_b(value, other.value); - return *this; + + simdutf_really_inline simd8 + operator|(const simd8 other) const { + return __lsx_vor_v((__m128i)value, (__m128i)other.value); } - // Bit-specific operations simdutf_really_inline bool is_ascii() const { - __m256i ascii_mask = __lasx_xvslti_b(this->value, 0); - if (__lasx_xbnz_v(ascii_mask)) - return false; - return true; - } - simdutf_really_inline bool any_bits_set_anywhere() const { - if (__lasx_xbnz_v(this->value)) - return true; - return false; + return (__lsx_vpickve2gr_hu(__lsx_vmskgez_b((__m128i)this->value), 0) == + 0xffff); } - template simdutf_really_inline simd8 shr() const { - return __lasx_xvsrli_b(this->value, N); + + // Order-sensitive comparisons + simdutf_really_inline simd8 operator>(const simd8 other) const { + return __lsx_vslt_b((__m128i)other.value, (__m128i)value); } - template simdutf_really_inline simd8 shl() const { - return __lasx_xvslli_b(this->value, N); + simdutf_really_inline simd8 operator<(const simd8 other) const { + return __lsx_vslt_b((__m128i)value, (__m128i)other.value); } - simdutf_really_inline uint64_t sum_bytes() const { - const auto sum_u16 = __lasx_xvhaddw_hu_bu(value, value); - const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + template + simdutf_really_inline simd8 + prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(this->value, N), + __lsx_vbsrl_v(prev_chunk.value, 16 - N)); + } - return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + template + simdutf_really_inline simd8 + apply_lookup_16_to(const simd8 original) const { + __m128i original_tmp = __lsx_vand_v(original, __lsx_vldi(0x1f)); + return __lsx_vshuf_b(__lsx_vldi(0), (__m128i)this->value, + simd8(original_tmp)); } }; -simdutf_really_inline simd8::operator simd8() const { - return this->value; -} template struct simd8x64 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, - "LASX kernel should use two registers per 64-byte block."); + static_assert( + NUM_CHUNKS == 4, + "LoongArch kernel should use four registers per 64-byte block."); simd8 chunks[NUM_CHUNKS]; simd8x64(const simd8x64 &o) = delete; // no copy allowed @@ -12219,36 +11546,36 @@ template struct simd8x64 { operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) - : chunks{chunk0, chunk1} {} + simdutf_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} simdutf_really_inline simd8x64(const T *ptr) : chunks{simd8::load(ptr), - simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} + simd8::load(ptr + sizeof(simd8) / sizeof(T)), + simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), + simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); - } - - simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); + this->chunks[2].store(ptr + sizeof(simd8) * 2 / sizeof(T)); + this->chunks[3].store(ptr + sizeof(simd8) * 3 / sizeof(T)); } simdutf_really_inline simd8x64 &operator|=(const simd8x64 &other) { this->chunks[0] |= other.chunks[0]; this->chunks[1] |= other.chunks[1]; + this->chunks[2] |= other.chunks[2]; + this->chunks[3] |= other.chunks[3]; return *this; } simdutf_really_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); } - simdutf_really_inline bool is_ascii() const { - return this->reduce_or().is_ascii(); - } + simdutf_really_inline bool is_ascii() const { return reduce_or().is_ascii(); } template simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { @@ -12256,117 +11583,158 @@ template struct simd8x64 { sizeof(simd8) * 0); this->chunks[1].template store_ascii_as_utf16(ptr + sizeof(simd8) * 1); + this->chunks[2].template store_ascii_as_utf16(ptr + + sizeof(simd8) * 2); + this->chunks[3].template store_ascii_as_utf16(ptr + + sizeof(simd8) * 3); } simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { - this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); - this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); + this->chunks[0].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 0); + this->chunks[1].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 1); + this->chunks[2].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 2); + this->chunks[3].store_ascii_as_utf32_tbl(ptr + sizeof(simd8) * 3); + } + + simdutf_really_inline uint64_t to_bitmask() const { + __m128i mask = __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[3]), 6); + mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[2]), 4)); + mask = __lsx_vor_v(mask, __lsx_vbsll_v(__lsx_vmsknz_b(this->chunks[1]), 2)); + mask = __lsx_vor_v(mask, __lsx_vmsknz_b(this->chunks[0])); + return __lsx_vpickve2gr_du(mask, 0); } simdutf_really_inline uint64_t lt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask) + return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, + this->chunks[2] < mask, this->chunks[3] < mask) .to_bitmask(); } - simdutf_really_inline uint64_t gt(const T m) const { const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask) + return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask, + this->chunks[2] > mask, this->chunks[3] > mask) + .to_bitmask(); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) .to_bitmask(); } simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { const simd8 mask = simd8::splat(m); - return simd8x64((simd8(__m256i(this->chunks[0])) >= mask), - (simd8(__m256i(this->chunks[1])) >= mask)) + return simd8x64(simd8(this->chunks[0].value) >= mask, + simd8(this->chunks[1].value) >= mask, + simd8(this->chunks[2].value) >= mask, + simd8(this->chunks[3].value) >= mask) .to_bitmask(); } }; // struct simd8x64 -/* begin file src/simdutf/lasx/simd16-inl.h */ +/* begin file src/simdutf/lsx/simd16-inl.h */ template struct simd16; +template > struct base_u16 { + __m128i value; + static const size_t SIZE = sizeof(value); + static const size_t ELEMENTS = sizeof(value) / sizeof(T); + + // Conversion from/to SIMD register + simdutf_really_inline base_u16() = default; + simdutf_really_inline base_u16(const __m128i _value) : value(_value) {} + // Bit operations + simdutf_really_inline simd16 operator|(const simd16 other) const { + return __lsx_vor_v(this->value, other.value); + } + simdutf_really_inline simd16 operator&(const simd16 other) const { + return __lsx_vand_v(this->value, other.value); + } + simdutf_really_inline simd16 operator~() const { + return __lsx_vxori_b(this->value, 0xFF); + } + + friend simdutf_really_inline Mask operator==(const simd16 lhs, + const simd16 rhs) { + return __lsx_vseq_h(lhs.value, rhs.value); + } + + template + simdutf_really_inline simd16 byte_right_shift() const { + return __lsx_vbsrl_v(this->value, N); + } + + simdutf_really_inline uint16_t first() const { + return uint16_t(__lsx_vpickve2gr_w(value, 0)); + } +}; + template > -struct base16 : base> { - using bitmask_type = uint32_t; +struct base16 : base_u16 { + using bitmask_type = uint16_t; - simdutf_really_inline base16() : base>() {} - simdutf_really_inline base16(const __m256i _value) - : base>(_value) {} + simdutf_really_inline base16() : base_u16() {} + simdutf_really_inline base16(const __m128i _value) : base_u16(_value) {} template simdutf_really_inline base16(const Pointer *ptr) - : base16(__lasx_xvld(reinterpret_cast(ptr), 0)) {} + : base16(__lsx_vld(ptr, 0)) {} - /// the size of vector in bytes - static const int SIZE = sizeof(base>::value); + static const int SIZE = sizeof(base_u16::value); - /// the number of elements of type T a vector can hold - static const int ELEMENTS = SIZE / sizeof(T); + template + simdutf_really_inline simd16 prev(const simd16 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N * 2), + __lsx_vbsrl_v(prev_chunk, 16 - N * 2)); + } }; // SIMD byte mask type (returned by things like eq and gt) template <> struct simd16 : base16 { static simdutf_really_inline simd16 splat(bool _value) { - return __lasx_xvreplgr2vr_h(uint16_t(-(!!_value))); + return __lsx_vreplgr2vr_h(uint16_t(-(!!_value))); } simdutf_really_inline simd16() : base16() {} - simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} - // Splat constructor - simdutf_really_inline simd16(bool _value) : base16(splat(_value)) {} + simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} simdutf_really_inline bitmask_type to_bitmask() const { - __m256i mask = __lasx_xvmsknz_b(this->value); - bitmask_type mask0 = __lasx_xvpickve2gr_wu(mask, 0); - bitmask_type mask1 = __lasx_xvpickve2gr_wu(mask, 4); - return (mask0 | (mask1 << 16)); - } - simdutf_really_inline simd16 operator~() const { return *this ^ true; } - - simdutf_really_inline bool is_zero() const { - return __lasx_xbz_v(this->value); - } - - template simdutf_really_inline simd16 byte_right_shift() const { - const auto t0 = __lasx_xvbsrl_v(this->value, N); - const auto t1 = __lasx_xvpermi_q(this->value, __lasx_xvldi(0), 0b00000011); - const auto t2 = __lasx_xvbsll_v(t1, 16 - N); - const auto t3 = __lasx_xvor_v(t0, t2); - return t3; + __m128i mask = __lsx_vmsknz_b(this->value); + bitmask_type mask0 = bitmask_type(__lsx_vpickve2gr_wu(mask, 0)); + return mask0; } - simdutf_really_inline uint16_t first() const { - return uint16_t(__lasx_xvpickve2gr_w(value, 0)); - } + simdutf_really_inline bool is_zero() const { return __lsx_bz_v(this->value); } }; template struct base16_numeric : base16 { static simdutf_really_inline simd16 splat(T _value) { - return __lasx_xvreplgr2vr_h((uint16_t)_value); + return __lsx_vreplgr2vr_h(_value); } - static simdutf_really_inline simd16 zero() { return __lasx_xvldi(0); } + static simdutf_really_inline simd16 zero() { return __lsx_vldi(0); } + template static simdutf_really_inline simd16 load(const Pointer values) { - return __lasx_xvld(values, 0); + return __lsx_vld(values, 0); } - simdutf_really_inline base16_numeric() : base16() {} - simdutf_really_inline base16_numeric(const __m256i _value) + simdutf_really_inline base16_numeric(const __m128i _value) : base16(_value) {} // Store to array simdutf_really_inline void store(T dst[8]) const { - return __lasx_xvst(this->value, reinterpret_cast<__m256i *>(dst), 0); + return __lsx_vst(this->value, dst, 0); } // Override to distinguish from bool version - simdutf_really_inline simd16 operator~() const { return *this ^ 0xFFFFu; } + simdutf_really_inline simd16 operator~() const { + return __lsx_vxori_b(this->value, 0xFF); + } }; -// Unsigned code units +// Unsigned code unitstemplate<> template <> struct simd16 : base16_numeric { - simdutf_really_inline simd16() : base16_numeric() {} - simdutf_really_inline simd16(const __m256i _value) - : base16_numeric(_value) {} + simdutf_really_inline simd16(const __m128i _value) + : base16_numeric((__m128i)_value) {} // Splat constructor simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} @@ -12376,71 +11744,67 @@ template <> struct simd16 : base16_numeric { simdutf_really_inline simd16(const char16_t *values) : simd16(load(reinterpret_cast(values))) {} + // Copy constructor + simdutf_really_inline simd16(const simd16 mask) : simd16(mask.value) {} + // Order-specific operations simdutf_really_inline simd16 &operator+=(const simd16 other) { - value = __lasx_xvadd_h(value, other.value); + value = __lsx_vadd_h(value, other.value); return *this; } - // Change the endianness - simdutf_really_inline simd16 swap_bytes() const { - return __lasx_xvshuf4i_b(this->value, 0b10110001); - } - template static simdutf_really_inline simd8 pack_shifted_right(const simd16 &v0, const simd16 &v1) { - return __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(v1.value, v0.value, N), - 0b11011000); + return __lsx_vssrlni_bu_h(v1.value, v0.value, N); } // Pack with the unsigned saturation of two uint16_t code units into single // uint8_t vector static simdutf_really_inline simd8 pack(const simd16 &v0, const simd16 &v1) { - return pack_shifted_right<0>(v0, v1); } - simdutf_really_inline uint64_t sum() const { - const auto sum_u32 = __lasx_xvhaddw_wu_hu(value, value); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); - - return uint64_t(__lasx_xvpickve2gr_du(sum_u64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum_u64, 3)); + // Change the endianness + simdutf_really_inline simd16 swap_bytes() const { + return __lsx_vshuf4i_b(this->value, 0b10110001); } - template simdutf_really_inline simd16 byte_right_shift() const { - return __lasx_xvbsrl_v(this->value, N); + simdutf_really_inline uint64_t sum() const { + const auto sum_u32 = __lsx_vhaddw_wu_hu(value, value); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); + + return uint64_t(__lsx_vpickve2gr_du(sum_u64, 0)) + + uint64_t(__lsx_vpickve2gr_du(sum_u64, 1)); } }; simdutf_really_inline simd16 operator<(const simd16 a, const simd16 b) { - return __lasx_xvslt_hu(a.value, b.value); + return __lsx_vslt_hu(a.value, b.value); } simdutf_really_inline simd16 operator>(const simd16 a, const simd16 b) { - return __lasx_xvslt_hu(b.value, a.value); + return __lsx_vslt_hu(b.value, a.value); } simdutf_really_inline simd16 operator<=(const simd16 a, const simd16 b) { - return __lasx_xvsle_hu(a.value, b.value); + return __lsx_vsle_hu(a.value, b.value); } simdutf_really_inline simd16 operator>=(const simd16 a, const simd16 b) { - return __lasx_xvsle_hu(b.value, a.value); + return __lsx_vsle_hu(b.value, a.value); } template struct simd16x32 { static constexpr int NUM_CHUNKS = 64 / sizeof(simd16); - static_assert(NUM_CHUNKS == 2, - "LASX kernel should use two registers per 64-byte block."); + static_assert( + NUM_CHUNKS == 4, + "LOONGARCH kernel should use four registers per 64-byte block."); simd16 chunks[NUM_CHUNKS]; simd16x32(const simd16x32 &o) = delete; // no copy allowed @@ -12448,139 +11812,133 @@ template struct simd16x32 { operator=(const simd16 other) = delete; // no assignment allowed simd16x32() = delete; // no default constructor allowed - simdutf_really_inline simd16x32(const simd16 chunk0, - const simd16 chunk1) - : chunks{chunk0, chunk1} {} + simdutf_really_inline + simd16x32(const simd16 chunk0, const simd16 chunk1, + const simd16 chunk2, const simd16 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} simdutf_really_inline simd16x32(const T *ptr) : chunks{simd16::load(ptr), - simd16::load(ptr + sizeof(simd16) / sizeof(T))} {} + simd16::load(ptr + sizeof(simd16) / sizeof(T)), + simd16::load(ptr + 2 * sizeof(simd16) / sizeof(T)), + simd16::load(ptr + 3 * sizeof(simd16) / sizeof(T))} {} simdutf_really_inline void store(T *ptr) const { this->chunks[0].store(ptr + sizeof(simd16) * 0 / sizeof(T)); this->chunks[1].store(ptr + sizeof(simd16) * 1 / sizeof(T)); + this->chunks[2].store(ptr + sizeof(simd16) * 2 / sizeof(T)); + this->chunks[3].store(ptr + sizeof(simd16) * 3 / sizeof(T)); } simdutf_really_inline void swap_bytes() { this->chunks[0] = this->chunks[0].swap_bytes(); this->chunks[1] = this->chunks[1].swap_bytes(); + this->chunks[2] = this->chunks[2].swap_bytes(); + this->chunks[3] = this->chunks[3].swap_bytes(); } simdutf_really_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + simdutf_really_inline uint64_t gteq(const T m) const { + const simd16 mask = simd16::splat(m); + return simd16x32(this->chunks[0] >= mask, this->chunks[1] >= mask, + this->chunks[2] >= mask, this->chunks[3] >= mask) + .to_bitmask(); } simdutf_really_inline uint64_t lteq(const T m) const { const simd16 mask = simd16::splat(m); - return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask) + return simd16x32(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) .to_bitmask(); } }; // struct simd16x32 -simdutf_really_inline simd16 min(const simd16 a, - const simd16 b) { - return __lasx_xvmin_hu(a.value, b.value); -} - -simdutf_really_inline simd16 operator==(const simd16 a, - uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvseq_h(a.value, bv); -} - -simdutf_really_inline simd16 as_vector_u16(const simd16 x) { - return x.value; -} - -simdutf_really_inline simd16 operator&(const simd16 a, +simdutf_really_inline simd16 operator^(const simd16 a, uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvand_v(a.value, bv); + const auto bv = __lsx_vreplgr2vr_h(b); + return __lsx_vxor_v(a.value, bv); } -simdutf_really_inline simd16 operator&(const simd16 a, - const simd16 b) { - return __lasx_xvand_v(a.value, b.value); +simdutf_really_inline simd16 operator^(const simd16 a, + const simd16 b) { + return __lsx_vxor_v(a.value, b.value); } -simdutf_really_inline simd16 operator^(const simd16 a, - uint16_t b) { - const auto bv = __lasx_xvreplgr2vr_h(b); - return __lasx_xvxor_v(a.value, bv); +simdutf_really_inline simd16 min(const simd16 a, + const simd16 b) { + return __lsx_vmin_hu(a.value, b.value); } -simdutf_really_inline simd16 operator^(const simd16 a, - const simd16 b) { - return __lasx_xvxor_v(a.value, b.value); +simdutf_really_inline simd16 as_vector_u16(const simd16 x) { + return x.value; } -/* end file src/simdutf/lasx/simd16-inl.h */ -/* begin file src/simdutf/lasx/simd32-inl.h */ +/* end file src/simdutf/lsx/simd16-inl.h */ +/* begin file src/simdutf/lsx/simd32-inl.h */ template struct simd32; template <> struct simd32 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint32_t); // constructors - simdutf_really_inline simd32(__m256i v) : value(v) {} + simdutf_really_inline simd32(__m128i v) : value(v) {} template - simdutf_really_inline simd32(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} + simdutf_really_inline simd32(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} // in-place operators simdutf_really_inline simd32 &operator-=(const simd32 other) { - value = __lasx_xvsub_w(value, other.value); + value = __lsx_vsub_w(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - const auto odd = __lasx_xvsrli_d(value, 32); - const auto even = __lasx_xvand_v(value, __lasx_xvreplgr2vr_d(0xffffffff)); - - const auto sum64 = __lasx_xvadd_d(odd, even); - - return uint64_t(__lasx_xvpickve2gr_du(sum64, 0)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 1)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 2)) + - uint64_t(__lasx_xvpickve2gr_du(sum64, 3)); + return uint64_t(__lsx_vpickve2gr_wu(value, 0)) + + uint64_t(__lsx_vpickve2gr_wu(value, 1)) + + uint64_t(__lsx_vpickve2gr_wu(value, 2)) + + uint64_t(__lsx_vpickve2gr_wu(value, 3)); } // static members static simdutf_really_inline simd32 splat(uint32_t x) { - return __lasx_xvreplgr2vr_w(x); + return __lsx_vreplgr2vr_w(x); } static simdutf_really_inline simd32 zero() { - return __lasx_xvrepli_w(0); + return __lsx_vrepli_w(0); } }; // ------------------------------------------------------------ template <> struct simd32 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd32(__m256i v) : value(v) {} + simdutf_really_inline simd32(__m128i v) : value(v) {} }; // ------------------------------------------------------------ simdutf_really_inline simd32 operator&(const simd32 a, const simd32 b) { - return __lasx_xvor_v(a.value, b.value); + return __lsx_vor_v(a.value, b.value); } simdutf_really_inline simd32 operator<(const simd32 a, const simd32 b) { - return __lasx_xvslt_wu(a.value, b.value); + return __lsx_vslt_wu(a.value, b.value); } simdutf_really_inline simd32 operator>(const simd32 a, const simd32 b) { - return __lasx_xvslt_wu(b.value, a.value); + return __lsx_vslt_wu(b.value, a.value); } // ------------------------------------------------------------ @@ -12588,78 +11946,76 @@ simdutf_really_inline simd32 operator>(const simd32 a, simdutf_really_inline simd32 as_vector_u32(const simd32 v) { return v.value; } -/* end file src/simdutf/lasx/simd32-inl.h */ -/* begin file src/simdutf/lasx/simd64-inl.h */ +/* end file src/simdutf/lsx/simd32-inl.h */ +/* begin file src/simdutf/lsx/simd64-inl.h */ template struct simd64; template <> struct simd64 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); static const int ELEMENTS = SIZE / sizeof(uint64_t); // constructors - simdutf_really_inline simd64(__m256i v) : value(v) {} + simdutf_really_inline simd64(__m128i v) : value(v) {} template - simdutf_really_inline simd64(Ptr *ptr) : value(__lasx_xvld(ptr, 0)) {} + simdutf_really_inline simd64(Ptr *ptr) : value(__lsx_vld(ptr, 0)) {} // in-place operators simdutf_really_inline simd64 &operator+=(const simd64 other) { - value = __lasx_xvadd_d(value, other.value); + value = __lsx_vadd_d(value, other.value); return *this; } // members simdutf_really_inline uint64_t sum() const { - return uint64_t(__lasx_xvpickve2gr_du(value, 0)) + - uint64_t(__lasx_xvpickve2gr_du(value, 1)) + - uint64_t(__lasx_xvpickve2gr_du(value, 2)) + - uint64_t(__lasx_xvpickve2gr_du(value, 3)); + return uint64_t(__lsx_vpickve2gr_du(value, 0)) + + uint64_t(__lsx_vpickve2gr_du(value, 1)); } // static members static simdutf_really_inline simd64 zero() { - return __lasx_xvrepli_d(0); + return __lsx_vrepli_d(0); } }; // ------------------------------------------------------------ template <> struct simd64 { - __m256i value; + __m128i value; static const int SIZE = sizeof(value); // constructors - simdutf_really_inline simd64(__m256i v) : value(v) {} + simdutf_really_inline simd64(__m128i v) : value(v) {} }; // ------------------------------------------------------------ simd64 sum_8bytes(const simd8 v) { - const auto sum_u16 = __lasx_xvhaddw_hu_bu(v, v); - const auto sum_u32 = __lasx_xvhaddw_wu_hu(sum_u16, sum_u16); - const auto sum_u64 = __lasx_xvhaddw_du_wu(sum_u32, sum_u32); + const auto sum_u16 = __lsx_vhaddw_hu_bu(v, v); + const auto sum_u32 = __lsx_vhaddw_wu_hu(sum_u16, sum_u16); + const auto sum_u64 = __lsx_vhaddw_du_wu(sum_u32, sum_u32); return simd64(sum_u64); } -/* end file src/simdutf/lasx/simd64-inl.h */ +/* end file src/simdutf/lsx/simd64-inl.h */ } // namespace simd } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf -#endif // SIMDUTF_LASX_SIMD_H -/* end file src/simdutf/lasx/simd.h */ +#endif // SIMDUTF_LSX_SIMD_H +/* end file src/simdutf/lsx/simd.h */ -/* begin file src/simdutf/lasx/end.h */ +/* begin file src/simdutf/lsx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lasx/end.h */ +/* end file src/simdutf/lsx/end.h */ -#endif // SIMDUTF_IMPLEMENTATION_LASX +#endif // SIMDUTF_IMPLEMENTATION_LSX -#endif // SIMDUTF_LASX_H -/* end file src/simdutf/lasx.h */ +#endif // SIMDUTF_LSX_H +/* end file src/simdutf/lsx.h */ /* begin file src/simdutf/fallback.h */ #ifndef SIMDUTF_FALLBACK_H #define SIMDUTF_FALLBACK_H @@ -12767,4297 +12123,401 @@ class implementation final : public simdutf::implementation { const char32_t *buf, size_t len) const noexcept final; #endif // SIMDUTF_FEATURE_UTF32 -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t convert_latin1_to_utf8( - const char *buf, size_t len, char *utf8_output) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t convert_latin1_to_utf16le( - const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t convert_latin1_to_utf16be( - const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t convert_latin1_to_utf32( - const char *buf, size_t len, char32_t *utf32_output) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t convert_utf8_to_latin1( - const char *buf, size_t len, char *latin1_output) const noexcept final; - simdutf_warn_unused result convert_utf8_to_latin1_with_errors( - const char *buf, size_t len, char *latin1_buffer) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf8_to_latin1( - const char *buf, size_t len, char *latin1_output) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t convert_utf8_to_utf16le( - const char *buf, size_t len, char16_t *utf16_output) const noexcept final; - simdutf_warn_unused size_t convert_utf8_to_utf16be( - const char *buf, size_t len, char16_t *utf16_output) const noexcept final; - simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( - const char *buf, size_t len, char16_t *utf16_output) const noexcept final; - simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( - const char *buf, size_t len, char16_t *utf16_output) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( - const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( - const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t convert_utf8_to_utf32( - const char *buf, size_t len, char32_t *utf32_output) const noexcept final; - simdutf_warn_unused result convert_utf8_to_utf32_with_errors( - const char *buf, size_t len, char32_t *utf32_output) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf8_to_utf32( - const char *buf, size_t len, char32_t *utf32_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - convert_utf16le_to_latin1(const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_utf16be_to_latin1(const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( - const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( - const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf16le_to_latin1(const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf16be_to_latin1(const char16_t *buf, size_t len, - char *latin1_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t convert_utf16le_to_utf8( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused size_t convert_utf16be_to_utf8( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( - const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t convert_utf32_to_utf8( - const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused result convert_utf32_to_utf8_with_errors( - const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; - simdutf_warn_unused size_t convert_valid_utf32_to_utf8( - const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - convert_utf32_to_latin1(const char32_t *buf, size_t len, - char *latin1_output) const noexcept final; - simdutf_warn_unused result - convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, - char *latin1_output) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf32_to_latin1(const char32_t *buf, size_t len, - char *latin1_output) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - convert_utf32_to_utf16le(const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_utf32_to_utf16be(const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( - const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( - const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf32_to_utf16le(const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf32_to_utf16be(const char32_t *buf, size_t len, - char16_t *utf16_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_utf16le_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_utf16be_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( - const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; - simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( - const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf16le_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; - simdutf_warn_unused size_t - convert_valid_utf16be_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_buffer) const noexcept final; -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF16 - void change_endianness_utf16(const char16_t *buf, size_t length, - char16_t *output) const noexcept final; - simdutf_warn_unused size_t count_utf16le(const char16_t *buf, - size_t length) const noexcept; - simdutf_warn_unused size_t count_utf16be(const char16_t *buf, - size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF8 - simdutf_warn_unused size_t count_utf8(const char *buf, - size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF8 - -#if SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf8_length_from_utf16le(const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t - utf8_length_from_utf16be(const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t utf32_length_from_utf16le( - const char16_t *input, size_t length) const noexcept; - simdutf_warn_unused size_t utf32_length_from_utf16be( - const char16_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - simdutf_warn_unused size_t - utf16_length_from_utf8(const char *input, size_t length) const noexcept; - simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( - const char16_t *input, size_t length) const noexcept; - ; - simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( - const char16_t *input, size_t length) const noexcept; - ; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf8_length_from_utf32(const char32_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf16_length_from_utf32(const char32_t *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - simdutf_warn_unused size_t - utf32_length_from_utf8(const char *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - latin1_length_from_utf8(const char *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - simdutf_warn_unused size_t - utf8_length_from_latin1(const char *input, size_t length) const noexcept; -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - -#if SIMDUTF_FEATURE_BASE64 - simdutf_warn_unused result base64_to_binary( - const char *input, size_t length, char *output, base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused full_result base64_to_binary_details( - const char *input, size_t length, char *output, base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused result - base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - simdutf_warn_unused full_result base64_to_binary_details( - const char16_t *input, size_t length, char *output, - base64_options options, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) const noexcept; - size_t binary_to_base64(const char *input, size_t length, char *output, - base64_options options) const noexcept; - size_t binary_to_base64_with_lines(const char *input, size_t length, - char *output, size_t line_length, - base64_options options) const noexcept; - const char *find(const char *start, const char *end, - char character) const noexcept; - const char16_t *find(const char16_t *start, const char16_t *end, - char16_t character) const noexcept; - -#endif // SIMDUTF_FEATURE_BASE64 -}; -} // namespace fallback -} // namespace simdutf - -#endif // SIMDUTF_FALLBACK_IMPLEMENTATION_H -/* end file src/simdutf/fallback/implementation.h */ - -/* begin file src/simdutf/fallback/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "fallback" -// #define SIMDUTF_IMPLEMENTATION fallback -/* end file src/simdutf/fallback/begin.h */ - - // Declarations -/* begin file src/simdutf/fallback/bitmanipulation.h */ -#ifndef SIMDUTF_FALLBACK_BITMANIPULATION_H -#define SIMDUTF_FALLBACK_BITMANIPULATION_H - -#include - -namespace simdutf { -namespace fallback { -namespace {} // unnamed namespace -} // namespace fallback -} // namespace simdutf - -#endif // SIMDUTF_FALLBACK_BITMANIPULATION_H -/* end file src/simdutf/fallback/bitmanipulation.h */ - -/* begin file src/simdutf/fallback/end.h */ -/* end file src/simdutf/fallback/end.h */ - -#endif // SIMDUTF_IMPLEMENTATION_FALLBACK -#endif // SIMDUTF_FALLBACK_H -/* end file src/simdutf/fallback.h */ -#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO -SIMDUTF_POP_DISABLE_WARNINGS -#endif - -// The scalar routines should be included once. -/* begin file src/scalar/swap_bytes.h */ -#ifndef SIMDUTF_SWAP_BYTES_H -#define SIMDUTF_SWAP_BYTES_H - -namespace simdutf { -namespace scalar { - -inline simdutf_warn_unused uint16_t u16_swap_bytes(const uint16_t word) { - return uint16_t((word >> 8) | (word << 8)); -} - -inline simdutf_warn_unused uint32_t u32_swap_bytes(const uint32_t word) { - return ((word >> 24) & 0xff) | // move byte 3 to byte 0 - ((word << 8) & 0xff0000) | // move byte 1 to byte 2 - ((word >> 8) & 0xff00) | // move byte 2 to byte 1 - ((word << 24) & 0xff000000); // byte 0 to byte 3 -} - -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/swap_bytes.h */ -#if SIMDUTF_FEATURE_ASCII -/* begin file src/scalar/ascii.h */ -#ifndef SIMDUTF_ASCII_H -#define SIMDUTF_ASCII_H - -namespace simdutf { -namespace scalar { -namespace { -namespace ascii { -#if SIMDUTF_IMPLEMENTATION_FALLBACK -// Only used by the fallback kernel. -inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { - const uint8_t *data = reinterpret_cast(buf); - uint64_t pos = 0; - // process in blocks of 16 bytes when possible - for (; pos + 16 <= len; pos += 16) { - uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) != 0) { - return false; - } - } - // process the tail byte-by-byte - for (; pos < len; pos++) { - if (data[pos] >= 0b10000000) { - return false; - } - } - return true; -} -#endif -inline simdutf_warn_unused result validate_with_errors(const char *buf, - size_t len) noexcept { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - // process in blocks of 16 bytes when possible - for (; pos + 16 <= len; pos += 16) { - uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) != 0) { - for (; pos < len; pos++) { - if (data[pos] >= 0b10000000) { - return result(error_code::TOO_LARGE, pos); - } - } - } - } - // process the tail byte-by-byte - for (; pos < len; pos++) { - if (data[pos] >= 0b10000000) { - return result(error_code::TOO_LARGE, pos); - } - } - return result(error_code::SUCCESS, pos); -} - -} // namespace ascii -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/ascii.h */ -#endif // SIMDUTF_FEATURE_ASCII -#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/scalar/utf8.h */ -#ifndef SIMDUTF_UTF8_H -#define SIMDUTF_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf8 { -#if SIMDUTF_IMPLEMENTATION_FALLBACK || SIMDUTF_IMPLEMENTATION_RVV -// only used by the fallback kernel. -// credit: based on code from Google Fuchsia (Apache Licensed) -inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { - const uint8_t *data = reinterpret_cast(buf); - uint64_t pos = 0; - uint32_t code_point = 0; - while (pos < len) { - // check of the next 16 bytes are ascii. - uint64_t next_pos = pos + 16; - if (next_pos <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - pos = next_pos; - continue; - } - } - unsigned char byte = data[pos]; - - while (byte < 0b10000000) { - if (++pos == len) { - return true; - } - byte = data[pos]; - } - - if ((byte & 0b11100000) == 0b11000000) { - next_pos = pos + 2; - if (next_pos > len) { - return false; - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return false; - } - // range check - code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if ((code_point < 0x80) || (0x7ff < code_point)) { - return false; - } - } else if ((byte & 0b11110000) == 0b11100000) { - next_pos = pos + 3; - if (next_pos > len) { - return false; - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return false; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return false; - } - // range check - code_point = (byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if ((code_point < 0x800) || (0xffff < code_point) || - (0xd7ff < code_point && code_point < 0xe000)) { - return false; - } - } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 - next_pos = pos + 4; - if (next_pos > len) { - return false; - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return false; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return false; - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return false; - } - // range check - code_point = - (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff || 0x10ffff < code_point) { - return false; - } - } else { - // we may have a continuation - return false; - } - pos = next_pos; - } - return true; -} -#endif - -inline simdutf_warn_unused result validate_with_errors(const char *buf, - size_t len) noexcept { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - uint32_t code_point = 0; - while (pos < len) { - // check of the next 16 bytes are ascii. - size_t next_pos = pos + 16; - if (next_pos <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - pos = next_pos; - continue; - } - } - unsigned char byte = data[pos]; - - while (byte < 0b10000000) { - if (++pos == len) { - return result(error_code::SUCCESS, len); - } - byte = data[pos]; - } - - if ((byte & 0b11100000) == 0b11000000) { - next_pos = pos + 2; - if (next_pos > len) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if ((code_point < 0x80) || (0x7ff < code_point)) { - return result(error_code::OVERLONG, pos); - } - } else if ((byte & 0b11110000) == 0b11100000) { - next_pos = pos + 3; - if (next_pos > len) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - code_point = (byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if ((code_point < 0x800) || (0xffff < code_point)) { - return result(error_code::OVERLONG, pos); - } - if (0xd7ff < code_point && code_point < 0xe000) { - return result(error_code::SURROGATE, pos); - } - } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 - next_pos = pos + 4; - if (next_pos > len) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - code_point = - (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff) { - return result(error_code::OVERLONG, pos); - } - if (0x10ffff < code_point) { - return result(error_code::TOO_LARGE, pos); - } - } else { - // we either have too many continuation bytes or an invalid leading byte - if ((byte & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, pos); - } else { - return result(error_code::HEADER_BITS, pos); - } - } - pos = next_pos; - } - return result(error_code::SUCCESS, len); -} - -// Finds the previous leading byte starting backward from buf and validates with -// errors from there Used to pinpoint the location of an error when an invalid -// chunk is detected We assume that the stream starts with a leading byte, and -// to check that it is the case, we ask that you pass a pointer to the start of -// the stream (start). -inline simdutf_warn_unused result rewind_and_validate_with_errors( - const char *start, const char *buf, size_t len) noexcept { - // First check that we start with a leading byte - if ((*start & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, 0); - } - size_t extra_len{0}; - // A leading byte cannot be further than 4 bytes away - for (int i = 0; i < 5; i++) { - unsigned char byte = *buf; - if ((byte & 0b11000000) != 0b10000000) { - break; - } else { - buf--; - extra_len++; - } - } - - result res = validate_with_errors(buf, len + extra_len); - res.count -= extra_len; - return res; -} - -inline size_t count_code_points(const char *buf, size_t len) { - const int8_t *p = reinterpret_cast(buf); - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - // -65 is 0b10111111, anything larger in two-complement's should start a new - // code point. - if (p[i] > -65) { - counter++; - } - } - return counter; -} - -inline size_t utf16_length_from_utf8(const char *buf, size_t len) { - const int8_t *p = reinterpret_cast(buf); - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - if (p[i] > -65) { - counter++; - } - if (uint8_t(p[i]) >= 240) { - counter++; - } - } - return counter; -} - -simdutf_warn_unused inline size_t trim_partial_utf8(const char *input, - size_t length) { - if (length < 3) { - switch (length) { - case 2: - if (uint8_t(input[length - 1]) >= 0xc0) { - return length - 1; - } // 2-, 3- and 4-byte characters with only 1 byte left - if (uint8_t(input[length - 2]) >= 0xe0) { - return length - 2; - } // 3- and 4-byte characters with only 2 bytes left - return length; - case 1: - if (uint8_t(input[length - 1]) >= 0xc0) { - return length - 1; - } // 2-, 3- and 4-byte characters with only 1 byte left - return length; - case 0: - return length; - } - } - if (uint8_t(input[length - 1]) >= 0xc0) { - return length - 1; - } // 2-, 3- and 4-byte characters with only 1 byte left - if (uint8_t(input[length - 2]) >= 0xe0) { - return length - 2; - } // 3- and 4-byte characters with only 1 byte left - if (uint8_t(input[length - 3]) >= 0xf0) { - return length - 3; - } // 4-byte characters with only 3 bytes left - return length; -} - -} // namespace utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf8.h */ -#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING || \ - (SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1) -/* begin file src/scalar/utf16.h */ -#ifndef SIMDUTF_UTF16_H -#define SIMDUTF_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf16 { - -template -inline simdutf_warn_unused bool validate_as_ascii(const char16_t *data, - size_t len) noexcept { - for (size_t pos = 0; pos < len; pos++) { - char16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if (word >= 0x80) { - return false; - } - } - return true; -} - -template -inline simdutf_warn_unused bool validate(const char16_t *data, - size_t len) noexcept { - uint64_t pos = 0; - while (pos < len) { - char16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xF800) == 0xD800) { - if (pos + 1 >= len) { - return false; - } - char16_t diff = char16_t(word - 0xD800); - if (diff > 0x3FF) { - return false; - } - char16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - char16_t diff2 = char16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return false; - } - pos += 2; - } else { - pos++; - } - } - return true; -} - -template -inline simdutf_warn_unused result validate_with_errors(const char16_t *data, - size_t len) noexcept { - size_t pos = 0; - while (pos < len) { - char16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xF800) == 0xD800) { - if (pos + 1 >= len) { - return result(error_code::SURROGATE, pos); - } - char16_t diff = char16_t(word - 0xD800); - if (diff > 0x3FF) { - return result(error_code::SURROGATE, pos); - } - char16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - char16_t diff2 = uint16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return result(error_code::SURROGATE, pos); - } - pos += 2; - } else { - pos++; - } - } - return result(error_code::SUCCESS, pos); -} - -template -inline size_t count_code_points(const char16_t *p, size_t len) { - // We are not BOM aware. - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; - counter += ((word & 0xFC00) != 0xDC00); - } - return counter; -} - -template -inline size_t utf8_length_from_utf16(const char16_t *p, size_t len) { - // We are not BOM aware. - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; - counter++; // ASCII - counter += static_cast( - word > - 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes - counter += static_cast((word > 0x7FF && word <= 0xD7FF) || - (word >= 0xE000)); // three-byte - } - return counter; -} - -template -inline size_t utf32_length_from_utf16(const char16_t *p, size_t len) { - // We are not BOM aware. - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; - counter += ((word & 0xFC00) != 0xDC00); - } - return counter; -} - -simdutf_really_inline void -change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { - for (size_t i = 0; i < size; i++) { - *output++ = char16_t(input[i] >> 8 | input[i] << 8); - } -} - -template -simdutf_warn_unused inline size_t trim_partial_utf16(const char16_t *input, - size_t length) { - if (length == 0) { - return 0; - } - uint16_t last_word = uint16_t(input[length - 1]); - last_word = !match_system(big_endian) ? u16_swap_bytes(last_word) : last_word; - length -= ((last_word & 0xFC00) == 0xD800); - return length; -} - -template bool is_high_surrogate(char16_t c) { - c = !match_system(big_endian) ? u16_swap_bytes(c) : c; - return (0xd800 <= c && c <= 0xdbff); -} - -template bool is_low_surrogate(char16_t c) { - c = !match_system(big_endian) ? u16_swap_bytes(c) : c; - return (0xdc00 <= c && c <= 0xdfff); -} - -simdutf_really_inline bool high_surrogate(char16_t c) { - return (0xd800 <= c && c <= 0xdbff); -} - -simdutf_really_inline bool low_surrogate(char16_t c) { - return (0xdc00 <= c && c <= 0xdfff); -} - -template -inline result utf8_length_from_utf16_with_replacement(const char16_t *p, - size_t len) { - bool any_surrogates = false; - // We are not BOM aware. - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - if (is_high_surrogate(p[i])) { - any_surrogates = true; - // surrogate pair - if (i + 1 < len && is_low_surrogate(p[i + 1])) { - counter += 4; - i++; // skip low surrogate - } else { - counter += 3; // unpaired high surrogate replaced by U+FFFD - } - continue; - } else if (is_low_surrogate(p[i])) { - any_surrogates = true; - counter += 3; // unpaired low surrogate replaced by U+FFFD - continue; - } - char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; - counter++; // at least 1 byte - counter += - static_cast(word > 0x7F); // non-ASCII is at least 2 bytes - counter += static_cast(word > 0x7FF); // three-byte - } - return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, - counter}; -} - -// variable templates are a C++14 extension -template char16_t replacement() { - return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; -} - -template -void to_well_formed_utf16(const char16_t *input, size_t len, char16_t *output) { - const char16_t replacement = utf16::replacement(); - bool high_surrogate_prev = false, high_surrogate, low_surrogate; - size_t i = 0; - for (; i < len; i++) { - char16_t c = input[i]; - high_surrogate = is_high_surrogate(c); - low_surrogate = is_low_surrogate(c); - if (high_surrogate_prev && !low_surrogate) { - output[i - 1] = replacement; - } - - if (!high_surrogate_prev && low_surrogate) { - output[i] = replacement; - } else { - output[i] = input[i]; - } - high_surrogate_prev = high_surrogate; - } - - /* string may not end with high surrogate */ - if (high_surrogate_prev) { - output[i - 1] = replacement; - } -} - -} // namespace utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf16.h */ -#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING || - // (SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1) -#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/scalar/utf32.h */ -#ifndef SIMDUTF_UTF32_H -#define SIMDUTF_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32 { - -inline simdutf_warn_unused bool validate(const char32_t *buf, - size_t len) noexcept { - const uint32_t *data = reinterpret_cast(buf); - uint64_t pos = 0; - for (; pos < len; pos++) { - uint32_t word = data[pos]; - if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { - return false; - } - } - return true; -} - -inline simdutf_warn_unused result validate_with_errors(const char32_t *buf, - size_t len) noexcept { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - for (; pos < len; pos++) { - uint32_t word = data[pos]; - if (word > 0x10FFFF) { - return result(error_code::TOO_LARGE, pos); - } - if (word >= 0xD800 && word <= 0xDFFF) { - return result(error_code::SURROGATE, pos); - } - } - return result(error_code::SUCCESS, pos); -} - -inline size_t utf8_length_from_utf32(const char32_t *buf, size_t len) { - // We are not BOM aware. - const uint32_t *p = reinterpret_cast(buf); - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - // credit: @ttsugriy for the vectorizable approach - counter++; // ASCII - counter += static_cast(p[i] > 0x7F); // two-byte - counter += static_cast(p[i] > 0x7FF); // three-byte - counter += static_cast(p[i] > 0xFFFF); // four-bytes - } - return counter; -} - -inline size_t utf16_length_from_utf32(const char32_t *buf, size_t len) { - // We are not BOM aware. - const uint32_t *p = reinterpret_cast(buf); - size_t counter{0}; - for (size_t i = 0; i < len; i++) { - counter++; // non-surrogate word - counter += static_cast(p[i] > 0xFFFF); // surrogate pair - } - return counter; -} - -} // namespace utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf32.h */ -#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/latin1.h */ -#ifndef SIMDUTF_LATIN1_H -#define SIMDUTF_LATIN1_H - -namespace simdutf { -namespace scalar { -namespace { -namespace latin1 { - -simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, - size_t len) { - const uint8_t *c = reinterpret_cast(buf); - size_t answer = 0; - for (size_t i = 0; i < len; i++) { - if ((c[i] >> 7)) { - answer++; - } - } - return answer + len; -} - -} // namespace latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/latin1.h */ -#endif // SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_BASE64 -/* begin file src/scalar/base64.h */ -#ifndef SIMDUTF_BASE64_H -#define SIMDUTF_BASE64_H - -#include -#include -#include -#include -#include - -namespace simdutf { -namespace scalar { -namespace { -namespace base64 { - -// This function is not expected to be fast. Do not use in long loops. -// In most instances you should be using is_ignorable. -template bool is_ascii_white_space(char_type c) { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; -} - -template bool is_eight_byte(char_type c) { - if (sizeof(char_type) == 1) { - return true; - } - return uint8_t(c) == c; -} - -template -bool is_ignorable(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return false; - } - if (is_eight_byte(c) && code == 64) { - return true; - } - return ignore_garbage; -} -template -bool is_base64(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return true; - } - return false; -} - -template -bool is_base64_or_padding(char_type c, simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - if (c == '=') { - return true; - } - uint8_t code = to_base64[uint8_t(c)]; - if (is_eight_byte(c) && code <= 63) { - return true; - } - return false; -} - -template -bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { - return is_ignorable(c, options) || c == '='; -} - -struct reduced_input { - size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. - size_t equallocation; // location of the first padding character if any - size_t srclen; // length of the input buffer before padding - size_t full_input_length; // length of the input buffer with padding but - // without ignorable characters -}; - -// find the end of the base64 input buffer -// It returns the number of padding characters, the location of the first -// padding character if any, the length of the input buffer before padding -// and the length of the input buffer with padding. The input buffer is not -// modified. The function assumes that there are at most two padding characters. -template -reduced_input find_end(const char_type *src, size_t srclen, - simdutf::base64_options options) { - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - - size_t equalsigns = 0; - // We intentionally include trailing spaces in the full input length. - // See https://github.com/simdutf/simdutf/issues/824 - size_t full_input_length = srclen; - // skip trailing spaces - while (!ignore_garbage && srclen > 0 && - scalar::base64::is_eight_byte(src[srclen - 1]) && - to_base64[uint8_t(src[srclen - 1])] == 64) { - srclen--; - } - size_t equallocation = - srclen; // location of the first padding character if any - if (ignore_garbage) { - // Technically, we don't need to find the first padding character, we can - // just change our algorithms, but it adds substantial complexity. - auto it = simdutf::find(src, src + srclen, '='); - if (it != src + srclen) { - equallocation = it - src; - equalsigns = 1; - srclen = equallocation; - full_input_length = equallocation + 1; - } - return {equalsigns, equallocation, srclen, full_input_length}; - } - if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { - // This is the last '=' sign. - equallocation = srclen - 1; - srclen--; - equalsigns = 1; - // skip trailing spaces - while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && - to_base64[uint8_t(src[srclen - 1])] == 64) { - srclen--; - } - if (srclen > 0 && src[srclen - 1] == '=') { - // This is the second '=' sign. - equallocation = srclen - 1; - srclen--; - equalsigns = 2; - } - } - return {equalsigns, equallocation, srclen, full_input_length}; -} - -// Returns true upon success. The destination buffer must be large enough. -// This functions assumes that the padding (=) has been removed. -// if check_capacity is true, it will check that the destination buffer is -// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. -template -full_result base64_tail_decode_impl( - char *dst, size_t outlen, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, last_chunk_handling_options last_chunk_options) { - char *dstend = dst + outlen; - (void)dstend; - // This looks like 10 branches, but we expect the compiler to resolve this to - // two branches (easily predicted): - const uint8_t *to_base64 = - (options & base64_default_or_url) - ? tables::base64::to_base64_default_or_url_value - : ((options & base64_url) ? tables::base64::to_base64_url_value - : tables::base64::to_base64_value); - const uint32_t *d0 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d0 - : ((options & base64_url) ? tables::base64::base64_url::d0 - : tables::base64::base64_default::d0); - const uint32_t *d1 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d1 - : ((options & base64_url) ? tables::base64::base64_url::d1 - : tables::base64::base64_default::d1); - const uint32_t *d2 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d2 - : ((options & base64_url) ? tables::base64::base64_url::d2 - : tables::base64::base64_default::d2); - const uint32_t *d3 = - (options & base64_default_or_url) - ? tables::base64::base64_default_or_url::d3 - : ((options & base64_url) ? tables::base64::base64_url::d3 - : tables::base64::base64_default::d3); - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - - const char_type *srcend = src + length; - const char_type *srcinit = src; - const char *dstinit = dst; - - uint32_t x; - size_t idx; - uint8_t buffer[4]; - while (true) { - while (src + 4 <= srcend && is_eight_byte(src[0]) && - is_eight_byte(src[1]) && is_eight_byte(src[2]) && - is_eight_byte(src[3]) && - (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | - d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { - if (match_system(endianness::BIG)) { - x = scalar::u32_swap_bytes(x); - } - if (check_capacity && dstend - dst < 3) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - std::memcpy(dst, &x, 3); // optimization opportunity: copy 4 bytes - dst += 3; - src += 4; - } - const char_type *srccur = src; - idx = 0; - // we need at least four characters. -#ifdef __clang__ - // If possible, we read four characters at a time. (It is an optimization.) - if (ignore_garbage && src + 4 <= srcend) { - char_type c0 = src[0]; - char_type c1 = src[1]; - char_type c2 = src[2]; - char_type c3 = src[3]; - - uint8_t code0 = to_base64[uint8_t(c0)]; - uint8_t code1 = to_base64[uint8_t(c1)]; - uint8_t code2 = to_base64[uint8_t(c2)]; - uint8_t code3 = to_base64[uint8_t(c3)]; - - buffer[idx] = code0; - idx += (is_eight_byte(c0) && code0 <= 63); - buffer[idx] = code1; - idx += (is_eight_byte(c1) && code1 <= 63); - buffer[idx] = code2; - idx += (is_eight_byte(c2) && code2 <= 63); - buffer[idx] = code3; - idx += (is_eight_byte(c3) && code3 <= 63); - src += 4; - } -#endif - while ((idx < 4) && (src < srcend)) { - char_type c = *src; - - uint8_t code = to_base64[uint8_t(c)]; - buffer[idx] = uint8_t(code); - if (is_eight_byte(c) && code <= 63) { - idx++; - } else if (!ignore_garbage && - (code > 64 || !scalar::base64::is_eight_byte(c))) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; - } else { - // We have a space or a newline or garbage. We ignore it. - } - src++; - } - if (idx != 4) { - simdutf_log_assert(idx < 4, "idx should be less than 4"); - // We never should have that the number of base64 characters + the - // number of padding characters is more than 4. - if (!ignore_garbage && (idx + padding_characters > 4)) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } - - // The idea here is that in loose mode, - // if there is padding at all, it must be used - // to form 4-wise chunk. However, in loose mode, - // we do accept no padding at all. - if (!ignore_garbage && - last_chunk_options == last_chunk_handling_options::loose && - (idx >= 2) && padding_characters > 0 && - ((idx + padding_characters) & 3) != 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } else - - // The idea here is that in strict mode, we do not want to accept - // incomplete base64 chunks. So if the chunk was otherwise valid, we - // return BASE64_INPUT_REMAINDER. - if (!ignore_garbage && - last_chunk_options == last_chunk_handling_options::strict && - (idx >= 2) && ((idx + padding_characters) & 3) != 0) { - // The partial chunk was at src - idx - return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } else - // If there is a partial chunk with insufficient padding, with - // stop_before_partial, we need to just ignore it. In "only full" - // mode, skip the minute there are padding characters. - if ((last_chunk_options == - last_chunk_handling_options::stop_before_partial && - (padding_characters + idx < 4) && (idx != 0) && - (idx >= 2 || padding_characters == 0)) || - (last_chunk_options == - last_chunk_handling_options::only_full_chunks && - (idx >= 2 || padding_characters == 0))) { - // partial means that we are *not* going to consume the read - // characters. We need to rewind the src pointer. - src = srccur; - return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; - } else { - if (idx == 2) { - uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + - (uint32_t(buffer[1]) << 2 * 6); - if (!ignore_garbage && - (last_chunk_options == last_chunk_handling_options::strict) && - (triple & 0xffff)) { - return {BASE64_EXTRA_BITS, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - if (check_capacity && dstend - dst < 1) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 1); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 1); - } - dst += 1; - } else if (idx == 3) { - uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + - (uint32_t(buffer[1]) << 2 * 6) + - (uint32_t(buffer[2]) << 1 * 6); - if (!ignore_garbage && - (last_chunk_options == last_chunk_handling_options::strict) && - (triple & 0xff)) { - return {BASE64_EXTRA_BITS, size_t(src - srcinit), - size_t(dst - dstinit)}; - } - if (check_capacity && dstend - dst < 2) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 2); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 2); - } - dst += 2; - } else if (!ignore_garbage && idx == 1 && - (!is_partial(last_chunk_options) || - (is_partial(last_chunk_options) && - padding_characters > 0))) { - return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), - size_t(dst - dstinit)}; - } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; - } - return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; - } - } - if (check_capacity && dstend - dst < 3) { - return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), - size_t(dst - dstinit)}; - } - uint32_t triple = - (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + - (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); - if (match_system(endianness::BIG)) { - triple <<= 8; - std::memcpy(dst, &triple, 3); - } else { - triple = scalar::u32_swap_bytes(triple); - triple >>= 8; - std::memcpy(dst, &triple, 3); - } - dst += 3; - } -} - -template -full_result -base64_tail_decode(char *dst, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, - last_chunk_handling_options last_chunk_options) { - return base64_tail_decode_impl(dst, 0, src, length, padding_characters, - options, last_chunk_options); -} - -// like base64_tail_decode, but it will not write past the end of the output -// buffer. The outlen parameter is modified to reflect the number of bytes -// written. This functions assumes that the padding (=) has been removed. -// -template -full_result base64_tail_decode_safe( - char *dst, size_t outlen, const char_type *src, size_t length, - size_t padding_characters, // number of padding characters - // '=', typically 0, 1, 2. - base64_options options, last_chunk_handling_options last_chunk_options) { - return base64_tail_decode_impl(dst, outlen, src, length, - padding_characters, options, - last_chunk_options); -} - -inline full_result -patch_tail_result(full_result r, size_t previous_input, size_t previous_output, - size_t equallocation, size_t full_input_length, - last_chunk_handling_options last_chunk_options) { - r.input_count += previous_input; - r.output_count += previous_output; - if (r.padding_error) { - r.input_count = equallocation; - } - - if (r.error == error_code::SUCCESS) { - if (!is_partial(last_chunk_options)) { - // A success when we are not in stop_before_partial mode. - // means that we have consumed the whole input buffer. - r.input_count = full_input_length; - } else if (r.output_count % 3 != 0) { - r.input_count = full_input_length; - } - } - return r; -} - -// Returns the number of bytes written. The destination buffer must be large -// enough. It will add padding (=) if needed. -template -size_t tail_encode_base64_impl( - char *dst, const char *src, size_t srclen, base64_options options, - size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { - if (use_lines) { - // sanitize line_length and starting_line_offset. - // line_length must be greater than 3. - if (line_length < 4) { - line_length = 4; - } - simdutf_log_assert(line_offset <= line_length, - "line_offset should be less than line_length"); - } - // By default, we use padding if we are not using the URL variant. - // This is check with ((options & base64_url) == 0) which returns true if we - // are not using the URL variant. However, we also allow 'inversion' of the - // convention with the base64_reverse_padding option. If the - // base64_reverse_padding option is set, we use padding if we are using the - // URL variant, and we omit it if we are not using the URL variant. This is - // checked with - // ((options & base64_reverse_padding) == base64_reverse_padding). - bool use_padding = - ((options & base64_url) == 0) ^ - ((options & base64_reverse_padding) == base64_reverse_padding); - // This looks like 3 branches, but we expect the compiler to resolve this to - // a single branch: - const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 - : tables::base64::base64_default::e0; - const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 - : tables::base64::base64_default::e1; - const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 - : tables::base64::base64_default::e2; - char *out = dst; - size_t i = 0; - uint8_t t1, t2, t3; - for (; i + 2 < srclen; i += 3) { - t1 = uint8_t(src[i]); - t2 = uint8_t(src[i + 1]); - t3 = uint8_t(src[i + 2]); - if (use_lines) { - if (line_offset + 3 >= line_length) { - if (line_offset == line_length) { - *out++ = '\n'; - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - line_offset = 4; - } else if (line_offset + 1 == line_length) { - *out++ = e0[t1]; - *out++ = '\n'; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - line_offset = 3; - } else if (line_offset + 2 == line_length) { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = '\n'; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - line_offset = 2; - } else if (line_offset + 3 == line_length) { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = '\n'; - *out++ = e2[t3]; - line_offset = 1; - } - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - line_offset += 4; - } - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; - *out++ = e2[t3]; - } - } - switch (srclen - i) { - case 0: - break; - case 1: - t1 = uint8_t(src[i]); - if (use_lines) { - if (use_padding) { - if (line_offset + 3 >= line_length) { - if (line_offset == line_length) { - *out++ = '\n'; - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - *out++ = '='; - *out++ = '='; - } else if (line_offset + 1 == line_length) { - *out++ = e0[t1]; - *out++ = '\n'; - *out++ = e1[(t1 & 0x03) << 4]; - *out++ = '='; - *out++ = '='; - } else if (line_offset + 2 == line_length) { - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - *out++ = '\n'; - *out++ = '='; - *out++ = '='; - } else if (line_offset + 3 == line_length) { - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - *out++ = '='; - *out++ = '\n'; - *out++ = '='; - } - } else { - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - *out++ = '='; - *out++ = '='; - } - } else { - if (line_offset + 2 >= line_length) { - if (line_offset == line_length) { - *out++ = '\n'; - *out++ = e0[uint8_t(src[i])]; - *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; - } else if (line_offset + 1 == line_length) { - *out++ = e0[uint8_t(src[i])]; - *out++ = '\n'; - *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; - } else { - *out++ = e0[uint8_t(src[i])]; - *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; - // *out++ = '\n'; ==> no newline at the end of the output - } - } else { - *out++ = e0[uint8_t(src[i])]; - *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; - } - } - } else { - *out++ = e0[t1]; - *out++ = e1[(t1 & 0x03) << 4]; - if (use_padding) { - *out++ = '='; - *out++ = '='; - } - } - break; - default: /* case 2 */ - t1 = uint8_t(src[i]); - t2 = uint8_t(src[i + 1]); - if (use_lines) { - if (use_padding) { - if (line_offset + 3 >= line_length) { - if (line_offset == line_length) { - *out++ = '\n'; - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - *out++ = '='; - } else if (line_offset + 1 == line_length) { - *out++ = e0[t1]; - *out++ = '\n'; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - *out++ = '='; - } else if (line_offset + 2 == line_length) { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = '\n'; - *out++ = e2[(t2 & 0x0F) << 2]; - *out++ = '='; - } else if (line_offset + 3 == line_length) { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - *out++ = '\n'; - *out++ = '='; - } - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - *out++ = '='; - } - } else { - if (line_offset + 3 >= line_length) { - if (line_offset == line_length) { - *out++ = '\n'; - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - } else if (line_offset + 1 == line_length) { - *out++ = e0[t1]; - *out++ = '\n'; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - } else if (line_offset + 2 == line_length) { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = '\n'; - *out++ = e2[(t2 & 0x0F) << 2]; - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - // *out++ = '\n'; ==> no newline at the end of the output - } - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - } - } - } else { - *out++ = e0[t1]; - *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; - *out++ = e2[(t2 & 0x0F) << 2]; - if (use_padding) { - *out++ = '='; - } - } - } - return (size_t)(out - dst); -} - -// Returns the number of bytes written. The destination buffer must be large -// enough. It will add padding (=) if needed. -inline size_t tail_encode_base64(char *dst, const char *src, size_t srclen, - base64_options options) { - return tail_encode_base64_impl(dst, src, srclen, options); -} - -template -simdutf_warn_unused size_t maximal_binary_length_from_base64( - const char_type *input, size_t length) noexcept { - // We follow https://infra.spec.whatwg.org/#forgiving-base64-decode - size_t padding = 0; - if (length > 0) { - if (input[length - 1] == '=') { - padding++; - if (length > 1 && input[length - 2] == '=') { - padding++; - } - } - } - size_t actual_length = length - padding; - if (actual_length % 4 <= 1) { - return actual_length / 4 * 3; - } - // if we have a valid input, then the remainder must be 2 or 3 adding one or - // two extra bytes. - return actual_length / 4 * 3 + (actual_length % 4) - 1; -} - -template -simdutf_warn_unused full_result base64_to_binary_details_impl( - const char_type *input, size_t length, char *output, base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - if (length == 0) { - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; - } - return {SUCCESS, full_input_length, 0}; - } - full_result r = scalar::base64::base64_tail_decode( - output, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0 && !ignore_garbage) { - // additional checks - if ((r.output_count % 3 == 0) || - ((r.output_count % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; - } - } - // When is_partial(last_chunk_options) is true, we must either end with - // the end of the stream (beyond whitespace) or right after a non-ignorable - // character or at the very beginning of the stream. - // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - r.input_count < full_input_length) { - // First check if we can extend the input to the end of the stream - while (r.input_count < full_input_length && - base64_ignorable(*(input + r.input_count), options)) { - r.input_count++; - } - // If we are still not at the end of the stream, then we must backtrack - // to the last non-ignorable character. - if (r.input_count < full_input_length) { - while (r.input_count > 0 && - base64_ignorable(*(input + r.input_count - 1), options)) { - r.input_count--; - } - } - } - return r; -} - -template -simdutf_warn_unused full_result base64_to_binary_details_safe_impl( - const char_type *input, size_t length, char *output, size_t outlen, - base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = - (options == base64_options::base64_url_accept_garbage) || - (options == base64_options::base64_default_accept_garbage) || - (options == base64_options::base64_default_or_url_accept_garbage); - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - if (length == 0) { - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; - } - return {SUCCESS, full_input_length, 0}; - } - full_result r = scalar::base64::base64_tail_decode_safe( - output, outlen, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0 && !ignore_garbage) { - // additional checks - if ((r.output_count % 3 == 0) || - ((r.output_count % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; - } - } - - // When is_partial(last_chunk_options) is true, we must either end with - // the end of the stream (beyond whitespace) or right after a non-ignorable - // character or at the very beginning of the stream. - // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - r.input_count < full_input_length) { - // First check if we can extend the input to the end of the stream - while (r.input_count < full_input_length && - base64_ignorable(*(input + r.input_count), options)) { - r.input_count++; - } - // If we are still not at the end of the stream, then we must backtrack - // to the last non-ignorable character. - if (r.input_count < full_input_length) { - while (r.input_count > 0 && - base64_ignorable(*(input + r.input_count - 1), options)) { - r.input_count--; - } - } - } - return r; -} - -simdutf_warn_unused size_t -base64_length_from_binary(size_t length, base64_options options) noexcept { - // By default, we use padding if we are not using the URL variant. - // This is check with ((options & base64_url) == 0) which returns true if we - // are not using the URL variant. However, we also allow 'inversion' of the - // convention with the base64_reverse_padding option. If the - // base64_reverse_padding option is set, we use padding if we are using the - // URL variant, and we omit it if we are not using the URL variant. This is - // checked with - // ((options & base64_reverse_padding) == base64_reverse_padding). - bool use_padding = - ((options & base64_url) == 0) ^ - ((options & base64_reverse_padding) == base64_reverse_padding); - if (!use_padding) { - return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); - } - return (length + 2) / 3 * - 4; // We use padding to make the length a multiple of 4. -} - -simdutf_warn_unused size_t base64_length_from_binary_with_lines( - size_t length, base64_options options, size_t line_length) noexcept { - if (length == 0) { - return 0; - } - size_t base64_length = - scalar::base64::base64_length_from_binary(length, options); - if (line_length < 4) { - line_length = 4; - } - size_t lines = - (base64_length + line_length - 1) / line_length; // number of lines - return base64_length + lines - 1; -} - -// Return the length of the prefix that contains count base64 characters. -// Thus, if count is 3, the function returns the length of the prefix -// that contains 3 base64 characters. -// The function returns (size_t)-1 if there is not enough base64 characters in -// the input. -template -simdutf_warn_unused size_t prefix_length(size_t count, - simdutf::base64_options options, - const char_type *input, - size_t length) noexcept { - size_t i = 0; - while (i < length && is_ignorable(input[i], options)) { - i++; - } - if (count == 0) { - return i; // duh! - } - for (; i < length; i++) { - if (is_ignorable(input[i], options)) { - continue; - } - // We have a base64 character or a padding character. - count--; - if (count == 0) { - return i + 1; - } - } - simdutf_log_assert(false, "You never get here"); - - return -1; // should never happen -} - -} // namespace base64 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/base64.h */ -#endif // SIMDUTF_FEATURE_BASE64 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -/* begin file src/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ -#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H -#define SIMDUTF_VALID_UTF32_TO_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_utf8 { - -#if SIMDUTF_IMPLEMENTATION_FALLBACK || SIMDUTF_IMPLEMENTATION_PPC64 -// only used by the fallback and POWER kernel -inline size_t convert_valid(const char32_t *buf, size_t len, - char *utf8_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{utf8_output}; - while (pos < len) { - // try to convert the next block of 2 ASCII characters - if (pos + 2 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0xFFFFFF80FFFFFF80) == 0) { - *utf8_output++ = char(buf[pos]); - *utf8_output++ = char(buf[pos + 1]); - pos += 2; - continue; - } - } - uint32_t word = data[pos]; - if ((word & 0xFFFFFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - } else if ((word & 0xFFFFF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else if ((word & 0xFFFF0000) == 0) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 18) | 0b11110000); - *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } - } - return utf8_output - start; -} -#endif // SIMDUTF_IMPLEMENTATION_FALLBACK || SIMDUTF_IMPLEMENTATION_PPC64 - -} // namespace utf32_to_utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ -/* begin file src/scalar/utf32_to_utf8/utf32_to_utf8.h */ -#ifndef SIMDUTF_UTF32_TO_UTF8_H -#define SIMDUTF_UTF32_TO_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_utf8 { - -inline size_t convert(const char32_t *buf, size_t len, char *utf8_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{utf8_output}; - while (pos < len) { - // try to convert the next block of 2 ASCII characters - if (pos + 2 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0xFFFFFF80FFFFFF80) == 0) { - *utf8_output++ = char(buf[pos]); - *utf8_output++ = char(buf[pos + 1]); - pos += 2; - continue; - } - } - uint32_t word = data[pos]; - if ((word & 0xFFFFFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - } else if ((word & 0xFFFFF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else if ((word & 0xFFFF0000) == 0) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - if (word >= 0xD800 && word <= 0xDFFF) { - return 0; - } - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - if (word > 0x10FFFF) { - return 0; - } - *utf8_output++ = char((word >> 18) | 0b11110000); - *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } - } - return utf8_output - start; -} - -inline result convert_with_errors(const char32_t *buf, size_t len, - char *utf8_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{utf8_output}; - while (pos < len) { - // try to convert the next block of 2 ASCII characters - if (pos + 2 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0xFFFFFF80FFFFFF80) == 0) { - *utf8_output++ = char(buf[pos]); - *utf8_output++ = char(buf[pos + 1]); - pos += 2; - continue; - } - } - uint32_t word = data[pos]; - if ((word & 0xFFFFFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - } else if ((word & 0xFFFFF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else if ((word & 0xFFFF0000) == 0) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - if (word >= 0xD800 && word <= 0xDFFF) { - return result(error_code::SURROGATE, pos); - } - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - if (word > 0x10FFFF) { - return result(error_code::TOO_LARGE, pos); - } - *utf8_output++ = char((word >> 18) | 0b11110000); - *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } - } - return result(error_code::SUCCESS, utf8_output - start); -} - -} // namespace utf32_to_utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf32_to_utf8/utf32_to_utf8.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ -#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H -#define SIMDUTF_VALID_UTF32_TO_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_utf16 { - -template -inline size_t convert_valid(const char32_t *buf, size_t len, - char16_t *utf16_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - uint32_t word = data[pos]; - if ((word & 0xFFFF0000) == 0) { - // will not generate a surrogate pair - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(uint16_t(word))) - : char16_t(word); - pos++; - } else { - // will generate a surrogate pair - word -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - pos++; - } - } - return utf16_output - start; -} - -} // namespace utf32_to_utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ -/* begin file src/scalar/utf32_to_utf16/utf32_to_utf16.h */ -#ifndef SIMDUTF_UTF32_TO_UTF16_H -#define SIMDUTF_UTF32_TO_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_utf16 { - -template -inline size_t convert(const char32_t *buf, size_t len, char16_t *utf16_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - uint32_t word = data[pos]; - if ((word & 0xFFFF0000) == 0) { - if (word >= 0xD800 && word <= 0xDFFF) { - return 0; - } - // will not generate a surrogate pair - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(uint16_t(word))) - : char16_t(word); - } else { - // will generate a surrogate pair - if (word > 0x10FFFF) { - return 0; - } - word -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - } - pos++; - } - return utf16_output - start; -} - -template -inline result convert_with_errors(const char32_t *buf, size_t len, - char16_t *utf16_output) { - const uint32_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - uint32_t word = data[pos]; - if ((word & 0xFFFF0000) == 0) { - if (word >= 0xD800 && word <= 0xDFFF) { - return result(error_code::SURROGATE, pos); - } - // will not generate a surrogate pair - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(uint16_t(word))) - : char16_t(word); - } else { - // will generate a surrogate pair - if (word > 0x10FFFF) { - return result(error_code::TOO_LARGE, pos); - } - word -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - } - pos++; - } - return result(error_code::SUCCESS, utf16_output - start); -} - -} // namespace utf32_to_utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf32_to_utf16/utf32_to_utf16.h */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -/* begin file src/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ -#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H -#define SIMDUTF_VALID_UTF16_TO_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_utf8 { - -template -inline size_t convert_valid(const char16_t *buf, size_t len, - char *utf8_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{utf8_output}; - while (pos < len) { - // try to convert the next block of 4 ASCII characters - if (pos + 4 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (!match_system(big_endian)) { - v = (v >> 8) | (v << (64 - 8)); - } - if ((v & 0xFF80FF80FF80FF80) == 0) { - size_t final_pos = pos + 4; - while (pos < final_pos) { - *utf8_output++ = !match_system(big_endian) - ? char(u16_swap_bytes(buf[pos])) - : char(buf[pos]); - pos++; - } - continue; - } - } - - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - } else if ((word & 0xF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else if ((word & 0xF800) != 0xD800) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - uint32_t value = (diff << 10) + diff2 + 0x10000; - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((value >> 18) | 0b11110000); - *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((value & 0b111111) | 0b10000000); - pos += 2; - } - } - return utf8_output - start; -} - -} // namespace utf16_to_utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ -/* begin file src/scalar/utf16_to_utf8/utf16_to_utf8.h */ -#ifndef SIMDUTF_UTF16_TO_UTF8_H -#define SIMDUTF_UTF16_TO_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_utf8 { - -template -inline size_t convert(const char16_t *buf, size_t len, char *utf8_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{utf8_output}; - while (pos < len) { - // try to convert the next block of 8 bytes - if (pos + 4 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (!match_system(big_endian)) { - v = (v >> 8) | (v << (64 - 8)); - } - if ((v & 0xFF80FF80FF80FF80) == 0) { - size_t final_pos = pos + 4; - while (pos < final_pos) { - *utf8_output++ = !match_system(big_endian) - ? char(u16_swap_bytes(buf[pos])) - : char(buf[pos]); - pos++; - } - continue; - } - } - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - } else if ((word & 0xF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else if ((word & 0xF800) != 0xD800) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - // must be a surrogate pair - if (pos + 1 >= len) { - return 0; - } - uint16_t diff = uint16_t(word - 0xD800); - if (diff > 0x3FF) { - return 0; - } - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return 0; - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((value >> 18) | 0b11110000); - *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((value & 0b111111) | 0b10000000); - pos += 2; - } - } - return utf8_output - start; -} - -template -inline full_result convert_with_errors(const char16_t *buf, size_t len, - char *utf8_output, size_t utf8_len = 0) { - const uint16_t *data = reinterpret_cast(buf); - if (check_output && utf8_len == 0) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); - } - - size_t pos = 0; - char *start{utf8_output}; - char *end{utf8_output + utf8_len}; - - while (pos < len) { - // try to convert the next block of 8 bytes - if (pos + 4 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (!match_system(big_endian)) - v = (v >> 8) | (v << (64 - 8)); - if ((v & 0xFF80FF80FF80FF80) == 0) { - size_t final_pos = pos + 4; - while (pos < final_pos) { - *utf8_output++ = !match_system(big_endian) - ? char(u16_swap_bytes(buf[pos])) - : char(buf[pos]); - pos++; - if (check_output && size_t(end - utf8_output) == 0) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, - utf8_output - start); - } - } - continue; - } - } - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xFF80) == 0) { - // will generate one UTF-8 bytes - *utf8_output++ = char(word); - pos++; - if (check_output && size_t(end - utf8_output) == 0) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, - utf8_output - start); - } - } else if ((word & 0xF800) == 0) { - // will generate two UTF-8 bytes - // we have 0b110XXXXX 0b10XXXXXX - if (check_output && size_t(end - utf8_output) < 2) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, - utf8_output - start); - } - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - - } else if ((word & 0xF800) != 0xD800) { - // will generate three UTF-8 bytes - // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX - if (check_output && size_t(end - utf8_output) < 3) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, - utf8_output - start); - } - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - pos++; - } else { - - if (check_output && size_t(end - utf8_output) < 4) { - return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, - utf8_output - start); - } - // must be a surrogate pair - if (pos + 1 >= len) { - return full_result(error_code::SURROGATE, pos, utf8_output - start); - } - uint16_t diff = uint16_t(word - 0xD800); - if (diff > 0x3FF) { - return full_result(error_code::SURROGATE, pos, utf8_output - start); - } - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return full_result(error_code::SURROGATE, pos, utf8_output - start); - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - // will generate four UTF-8 bytes - // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX - *utf8_output++ = char((value >> 18) | 0b11110000); - *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((value & 0b111111) | 0b10000000); - pos += 2; - } - } - return full_result(error_code::SUCCESS, pos, utf8_output - start); -} - -template -inline result simple_convert_with_errors(const char16_t *buf, size_t len, - char *utf8_output) { - return convert_with_errors(buf, len, utf8_output, 0); -} - -} // namespace utf16_to_utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf16_to_utf8/utf16_to_utf8.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ -#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H -#define SIMDUTF_VALID_UTF16_TO_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_utf32 { - -template -inline size_t convert_valid(const char16_t *buf, size_t len, - char32_t *utf32_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xF800) != 0xD800) { - // No surrogate pair, extend 16-bit word to 32-bit word - *utf32_output++ = char32_t(word); - pos++; - } else { - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - uint32_t value = (diff << 10) + diff2 + 0x10000; - *utf32_output++ = char32_t(value); - pos += 2; - } - } - return utf32_output - start; -} - -} // namespace utf16_to_utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ -/* begin file src/scalar/utf16_to_utf32/utf16_to_utf32.h */ -#ifndef SIMDUTF_UTF16_TO_UTF32_H -#define SIMDUTF_UTF16_TO_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_utf32 { - -template -inline size_t convert(const char16_t *buf, size_t len, char32_t *utf32_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xF800) != 0xD800) { - // No surrogate pair, extend 16-bit word to 32-bit word - *utf32_output++ = char32_t(word); - pos++; - } else { - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - if (diff > 0x3FF) { - return 0; - } - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return 0; - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - *utf32_output++ = char32_t(value); - pos += 2; - } - } - return utf32_output - start; -} - -template -inline result convert_with_errors(const char16_t *buf, size_t len, - char32_t *utf32_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - uint16_t word = - !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xF800) != 0xD800) { - // No surrogate pair, extend 16-bit word to 32-bit word - *utf32_output++ = char32_t(word); - pos++; - } else { - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - if (diff > 0x3FF) { - return result(error_code::SURROGATE, pos); - } - if (pos + 1 >= len) { - return result(error_code::SURROGATE, pos); - } // minimal bound checking - uint16_t next_word = !match_system(big_endian) - ? u16_swap_bytes(data[pos + 1]) - : data[pos + 1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if (diff2 > 0x3FF) { - return result(error_code::SURROGATE, pos); - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - *utf32_output++ = char32_t(value); - pos += 2; - } - } - return result(error_code::SUCCESS, utf32_output - start); -} - -} // namespace utf16_to_utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf16_to_utf32/utf16_to_utf32.h */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && \ - (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_LATIN1) -/* begin file src/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ -#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H -#define SIMDUTF_VALID_UTF8_TO_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_utf16 { - -template -inline size_t convert_valid(const char *buf, size_t len, - char16_t *utf16_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - // try to convert the next block of 8 ASCII bytes - if (pos + 8 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 8; - while (pos < final_pos) { - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(buf[pos])) - : char16_t(buf[pos]); - pos++; - } - continue; - } - } - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(leading_byte)) - : char16_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 1 >= len) { - break; - } // minimal bound checking - uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | - (data[pos + 1] & 0b00111111)); - if (!match_system(big_endian)) { - code_point = u16_swap_bytes(uint16_t(code_point)); - } - *utf16_output++ = char16_t(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 2 >= len) { - break; - } // minimal bound checking - uint16_t code_point = uint16_t(((leading_byte & 0b00001111) << 12) | - ((data[pos + 1] & 0b00111111) << 6) | - (data[pos + 2] & 0b00111111)); - if (!match_system(big_endian)) { - code_point = u16_swap_bytes(uint16_t(code_point)); - } - *utf16_output++ = char16_t(code_point); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - break; - } // minimal bound checking - uint32_t code_point = ((leading_byte & 0b00000111) << 18) | - ((data[pos + 1] & 0b00111111) << 12) | - ((data[pos + 2] & 0b00111111) << 6) | - (data[pos + 3] & 0b00111111); - code_point -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - pos += 4; - } else { - // we may have a continuation but we do not do error checking - return 0; - } - } - return utf16_output - start; -} - -} // namespace utf8_to_utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ -/* begin file src/scalar/utf8_to_utf16/utf8_to_utf16.h */ -#ifndef SIMDUTF_UTF8_TO_UTF16_H -#define SIMDUTF_UTF8_TO_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_utf16 { - -template -inline size_t convert(const char *buf, size_t len, char16_t *utf16_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 16; - while (pos < final_pos) { - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(buf[pos])) - : char16_t(buf[pos]); - pos++; - } - continue; - } - } - - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(leading_byte)) - : char16_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - // range check - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if (code_point < 0x80 || 0x7ff < code_point) { - return 0; - } - if (!match_system(big_endian)) { - code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); - } - *utf16_output++ = char16_t(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 2 >= len) { - return 0; - } // minimal bound checking - - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return 0; - } - // range check - uint32_t code_point = (leading_byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if (code_point < 0x800 || 0xffff < code_point || - (0xd7ff < code_point && code_point < 0xe000)) { - return 0; - } - if (!match_system(big_endian)) { - code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); - } - *utf16_output++ = char16_t(code_point); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - return 0; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return 0; - } - - // range check - uint32_t code_point = (leading_byte & 0b00000111) << 18 | - (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | - (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff || 0x10ffff < code_point) { - return 0; - } - code_point -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - pos += 4; - } else { - return 0; - } - } - return utf16_output - start; -} - -template -inline result convert_with_errors(const char *buf, size_t len, - char16_t *utf16_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 16; - while (pos < final_pos) { - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(buf[pos])) - : char16_t(buf[pos]); - pos++; - } - continue; - } - } - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf16_output++ = !match_system(big_endian) - ? char16_t(u16_swap_bytes(leading_byte)) - : char16_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 1 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if (code_point < 0x80 || 0x7ff < code_point) { - return result(error_code::OVERLONG, pos); - } - if (!match_system(big_endian)) { - code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); - } - *utf16_output++ = char16_t(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8, it should become - // a single UTF-16 word. - if (pos + 2 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - uint32_t code_point = (leading_byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if ((code_point < 0x800) || (0xffff < code_point)) { - return result(error_code::OVERLONG, pos); - } - if (0xd7ff < code_point && code_point < 0xe000) { - return result(error_code::SURROGATE, pos); - } - if (!match_system(big_endian)) { - code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); - } - *utf16_output++ = char16_t(code_point); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - - // range check - uint32_t code_point = (leading_byte & 0b00000111) << 18 | - (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | - (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff) { - return result(error_code::OVERLONG, pos); - } - if (0x10ffff < code_point) { - return result(error_code::TOO_LARGE, pos); - } - code_point -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = u16_swap_bytes(high_surrogate); - low_surrogate = u16_swap_bytes(low_surrogate); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - pos += 4; - } else { - // we either have too many continuation bytes or an invalid leading byte - if ((leading_byte & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, pos); - } else { - return result(error_code::HEADER_BITS, pos); - } - } - } - return result(error_code::SUCCESS, utf16_output - start); -} - -/** - * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and - * we have up to len input bytes left, and we encountered some error. It is - * possible that the error is at 'buf' exactly, but it could also be in the - * previous bytes (up to 3 bytes back). - * - * prior_bytes indicates how many bytes, prior to 'buf' may belong to the - * current memory section and can be safely accessed. We prior_bytes to access - * safely up to three bytes before 'buf'. - * - * The caller is responsible to ensure that len > 0. - * - * If the error is believed to have occurred prior to 'buf', the count value - * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. - */ -template -inline result rewind_and_convert_with_errors(size_t prior_bytes, - const char *buf, size_t len, - char16_t *utf16_output) { - size_t extra_len{0}; - // We potentially need to go back in time and find a leading byte. - // In theory '3' would be sufficient, but sometimes the error can go back - // quite far. - size_t how_far_back = prior_bytes; - // size_t how_far_back = 3; // 3 bytes in the past + current position - // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } - bool found_leading_bytes{false}; - // important: it is i <= how_far_back and not 'i < how_far_back'. - for (size_t i = 0; i <= how_far_back; i++) { - unsigned char byte = buf[-static_cast(i)]; - found_leading_bytes = ((byte & 0b11000000) != 0b10000000); - if (found_leading_bytes) { - if (i > 0 && byte < 128) { - // If we had to go back and the leading byte is ascii - // then we can stop right away. - return result(error_code::TOO_LONG, 0 - i + 1); - } - buf -= i; - extra_len = i; - break; - } - } - // - // It is possible for this function to return a negative count in its result. - // C++ Standard Section 18.1 defines size_t is in which is described - // in C Standard as . C Standard Section 4.1.5 defines size_t as an - // unsigned integral type of the result of the sizeof operator - // - // An unsigned type will simply wrap round arithmetically (well defined). - // - if (!found_leading_bytes) { - // If how_far_back == 3, we may have four consecutive continuation bytes!!! - // [....] [continuation] [continuation] [continuation] | [buf is - // continuation] Or we possibly have a stream that does not start with a - // leading byte. - return result(error_code::TOO_LONG, 0 - how_far_back); - } - result res = convert_with_errors(buf, len + extra_len, utf16_output); - if (res.error) { - res.count -= extra_len; - } - return res; -} - -} // namespace utf8_to_utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf8_to_utf16/utf8_to_utf16.h */ -#endif // SIMDUTF_FEATURE_UTF8 && (SIMDUTF_FEATURE_UTF16 || - // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_LATIN1) - -#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 -/* begin file src/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ -#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H -#define SIMDUTF_VALID_UTF8_TO_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_utf32 { - -inline size_t convert_valid(const char *buf, size_t len, - char32_t *utf32_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - // try to convert the next block of 8 ASCII bytes - if (pos + 8 <= - len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 8; - while (pos < final_pos) { - *utf32_output++ = char32_t(buf[pos]); - pos++; - } - continue; - } - } - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf32_output++ = char32_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - break; - } // minimal bound checking - *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | - (data[pos + 1] & 0b00111111)); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8 - if (pos + 2 >= len) { - break; - } // minimal bound checking - *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | - ((data[pos + 1] & 0b00111111) << 6) | - (data[pos + 2] & 0b00111111)); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - break; - } // minimal bound checking - uint32_t code_word = ((leading_byte & 0b00000111) << 18) | - ((data[pos + 1] & 0b00111111) << 12) | - ((data[pos + 2] & 0b00111111) << 6) | - (data[pos + 3] & 0b00111111); - *utf32_output++ = char32_t(code_word); - pos += 4; - } else { - // we may have a continuation but we do not do error checking - return 0; - } - } - return utf32_output - start; -} - -} // namespace utf8_to_utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ -/* begin file src/scalar/utf8_to_utf32/utf8_to_utf32.h */ -#ifndef SIMDUTF_UTF8_TO_UTF32_H -#define SIMDUTF_UTF8_TO_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_utf32 { - -inline size_t convert(const char *buf, size_t len, char32_t *utf32_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 16; - while (pos < final_pos) { - *utf32_output++ = char32_t(buf[pos]); - pos++; - } - continue; - } - } - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf32_output++ = char32_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - // range check - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if (code_point < 0x80 || 0x7ff < code_point) { - return 0; - } - *utf32_output++ = char32_t(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8 - if (pos + 2 >= len) { - return 0; - } // minimal bound checking - - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return 0; - } - // range check - uint32_t code_point = (leading_byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if (code_point < 0x800 || 0xffff < code_point || - (0xd7ff < code_point && code_point < 0xe000)) { - return 0; - } - *utf32_output++ = char32_t(code_point); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - return 0; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return 0; - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return 0; - } - - // range check - uint32_t code_point = (leading_byte & 0b00000111) << 18 | - (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | - (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff || 0x10ffff < code_point) { - return 0; - } - *utf32_output++ = char32_t(code_point); - pos += 4; - } else { - return 0; - } - } - return utf32_output - start; -} - -inline result convert_with_errors(const char *buf, size_t len, - char32_t *utf32_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char32_t *start{utf32_output}; - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - size_t final_pos = pos + 16; - while (pos < final_pos) { - *utf32_output++ = char32_t(buf[pos]); - pos++; - } - continue; - } - } - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *utf32_output++ = char32_t(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == 0b11000000) { - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); - if (code_point < 0x80 || 0x7ff < code_point) { - return result(error_code::OVERLONG, pos); - } - *utf32_output++ = char32_t(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8 - if (pos + 2 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - // range check - uint32_t code_point = (leading_byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); - if (code_point < 0x800 || 0xffff < code_point) { - return result(error_code::OVERLONG, pos); - } - if (0xd7ff < code_point && code_point < 0xe000) { - return result(error_code::SURROGATE, pos); - } - *utf32_output++ = char32_t(code_point); - pos += 3; - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - if (pos + 3 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } - - // range check - uint32_t code_point = (leading_byte & 0b00000111) << 18 | - (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | - (data[pos + 3] & 0b00111111); - if (code_point <= 0xffff) { - return result(error_code::OVERLONG, pos); - } - if (0x10ffff < code_point) { - return result(error_code::TOO_LARGE, pos); - } - *utf32_output++ = char32_t(code_point); - pos += 4; - } else { - // we either have too many continuation bytes or an invalid leading byte - if ((leading_byte & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, pos); - } else { - return result(error_code::HEADER_BITS, pos); - } - } - } - return result(error_code::SUCCESS, utf32_output - start); -} - -/** - * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and - * we have up to len input bytes left, and we encountered some error. It is - * possible that the error is at 'buf' exactly, but it could also be in the - * previous bytes location (up to 3 bytes back). - * - * prior_bytes indicates how many bytes, prior to 'buf' may belong to the - * current memory section and can be safely accessed. We prior_bytes to access - * safely up to three bytes before 'buf'. - * - * The caller is responsible to ensure that len > 0. - * - * If the error is believed to have occurred prior to 'buf', the count value - * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. - */ -inline result rewind_and_convert_with_errors(size_t prior_bytes, - const char *buf, size_t len, - char32_t *utf32_output) { - size_t extra_len{0}; - // We potentially need to go back in time and find a leading byte. - size_t how_far_back = 3; // 3 bytes in the past + current position - if (how_far_back > prior_bytes) { - how_far_back = prior_bytes; - } - bool found_leading_bytes{false}; - // important: it is i <= how_far_back and not 'i < how_far_back'. - for (size_t i = 0; i <= how_far_back; i++) { - unsigned char byte = buf[-static_cast(i)]; - found_leading_bytes = ((byte & 0b11000000) != 0b10000000); - if (found_leading_bytes) { - if (i > 0 && byte < 128) { - // If we had to go back and the leading byte is ascii - // then we can stop right away. - return result(error_code::TOO_LONG, 0 - i + 1); - } - buf -= i; - extra_len = i; - break; - } - } - // - // It is possible for this function to return a negative count in its result. - // C++ Standard Section 18.1 defines size_t is in which is described - // in C Standard as . C Standard Section 4.1.5 defines size_t as an - // unsigned integral type of the result of the sizeof operator - // - // An unsigned type will simply wrap round arithmetically (well defined). - // - if (!found_leading_bytes) { - // If how_far_back == 3, we may have four consecutive continuation bytes!!! - // [....] [continuation] [continuation] [continuation] | [buf is - // continuation] Or we possibly have a stream that does not start with a - // leading byte. - return result(error_code::TOO_LONG, 0 - how_far_back); - } - - result res = convert_with_errors(buf, len + extra_len, utf32_output); - if (res.error) { - res.count -= extra_len; - } - return res; -} - -} // namespace utf8_to_utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/utf8_to_utf32/utf8_to_utf32.h */ -#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 - -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/latin1_to_utf8/latin1_to_utf8.h */ -#ifndef SIMDUTF_LATIN1_TO_UTF8_H -#define SIMDUTF_LATIN1_TO_UTF8_H - -namespace simdutf { -namespace scalar { -namespace { -namespace latin1_to_utf8 { - -inline size_t convert(const char *buf, size_t len, char *utf8_output) { - const unsigned char *data = reinterpret_cast(buf); - size_t pos = 0; - size_t utf8_pos = 0; - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | - v2}; // We are only interested in these bits: 1000 1000 1000 - // 1000, so it makes sense to concatenate everything - if ((v & 0x8080808080808080) == - 0) { // if NONE of these are set, e.g. all of them are zero, then - // everything is ASCII - size_t final_pos = pos + 16; - while (pos < final_pos) { - utf8_output[utf8_pos++] = char(buf[pos]); - pos++; - } - continue; - } - } - - unsigned char byte = data[pos]; - if ((byte & 0x80) == 0) { // if ASCII - // will generate one UTF-8 bytes - utf8_output[utf8_pos++] = char(byte); - pos++; - } else { - // will generate two UTF-8 bytes - utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); - utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); - pos++; - } - } - return utf8_pos; -} - -inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, - size_t utf8_len) { - const unsigned char *data = reinterpret_cast(buf); - size_t pos = 0; - size_t skip_pos = 0; - size_t utf8_pos = 0; - while (pos < len && utf8_pos < utf8_len) { - // try to convert the next block of 16 ASCII bytes - if (pos >= skip_pos && pos + 16 <= len && - utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, - // check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | - v2}; // We are only interested in these bits: 1000 1000 1000 - // 1000, so it makes sense to concatenate everything - if ((v & 0x8080808080808080) == - 0) { // if NONE of these are set, e.g. all of them are zero, then - // everything is ASCII - ::memcpy(utf8_output + utf8_pos, buf + pos, 16); - utf8_pos += 16; - pos += 16; - } else { - // At least one of the next 16 bytes are not ASCII, we will process them - // one by one - skip_pos = pos + 16; - } - } else { - const auto byte = data[pos]; - if ((byte & 0x80) == 0) { // if ASCII - // will generate one UTF-8 bytes - utf8_output[utf8_pos++] = char(byte); - pos++; - } else if (utf8_pos + 2 <= utf8_len) { - // will generate two UTF-8 bytes - utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); - utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); - pos++; - } else { - break; - } - } - } - return utf8_pos; -} - -} // namespace latin1_to_utf8 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/latin1_to_utf8/latin1_to_utf8.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/latin1_to_utf16/latin1_to_utf16.h */ -#ifndef SIMDUTF_LATIN1_TO_UTF16_H -#define SIMDUTF_LATIN1_TO_UTF16_H - -namespace simdutf { -namespace scalar { -namespace { -namespace latin1_to_utf16 { - -template -inline size_t convert(const char *buf, size_t len, char16_t *utf16_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - - while (pos < len) { - uint16_t word = - uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point - *utf16_output++ = - char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); - pos++; - } - - return utf16_output - start; -} - -template -inline result convert_with_errors(const char *buf, size_t len, - char16_t *utf16_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char16_t *start{utf16_output}; - - while (pos < len) { - uint16_t word = - uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point - *utf16_output++ = - char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); - pos++; - } - - return result(error_code::SUCCESS, utf16_output - start); -} - -} // namespace latin1_to_utf16 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf - -#endif -/* end file src/scalar/latin1_to_utf16/latin1_to_utf16.h */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/latin1_to_utf32/latin1_to_utf32.h */ -#ifndef SIMDUTF_LATIN1_TO_UTF32_H -#define SIMDUTF_LATIN1_TO_UTF32_H - -namespace simdutf { -namespace scalar { -namespace { -namespace latin1_to_utf32 { - -inline size_t convert(const char *buf, size_t len, char32_t *utf32_output) { - const unsigned char *data = reinterpret_cast(buf); - char32_t *start{utf32_output}; - for (size_t i = 0; i < len; i++) { - *utf32_output++ = (char32_t)data[i]; - } - return utf32_output - start; -} +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t convert_latin1_to_utf8( + const char *buf, size_t len, char *utf8_output) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -} // namespace latin1_to_utf32 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t convert_latin1_to_utf16le( + const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_latin1_to_utf16be( + const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#endif -/* end file src/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t convert_latin1_to_utf32( + const char *buf, size_t len, char32_t *utf32_output) const noexcept final; #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf8_to_latin1/utf8_to_latin1.h */ -#ifndef SIMDUTF_UTF8_TO_LATIN1_H -#define SIMDUTF_UTF8_TO_LATIN1_H + simdutf_warn_unused size_t convert_utf8_to_latin1( + const char *buf, size_t len, char *latin1_output) const noexcept final; + simdutf_warn_unused result convert_utf8_to_latin1_with_errors( + const char *buf, size_t len, char *latin1_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_latin1( + const char *buf, size_t len, char *latin1_output) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_latin1 { +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t convert_utf8_to_utf16le( + const char *buf, size_t len, char16_t *utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_utf8_to_utf16be( + const char *buf, size_t len, char16_t *utf16_output) const noexcept final; + simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( + const char *buf, size_t len, char16_t *utf16_output) const noexcept final; + simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( + const char *buf, size_t len, char16_t *utf16_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( + const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( + const char *buf, size_t len, char16_t *utf16_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -inline size_t convert(const char *buf, size_t len, char *latin_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{latin_output}; +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t convert_utf8_to_utf32( + const char *buf, size_t len, char32_t *utf32_output) const noexcept final; + simdutf_warn_unused result convert_utf8_to_utf32_with_errors( + const char *buf, size_t len, char32_t *utf32_output) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf8_to_utf32( + const char *buf, size_t len, char32_t *utf32_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 - // 1000 1000 .... etc - if ((v & 0x8080808080808080) == - 0) { // if NONE of these are set, e.g. all of them are zero, then - // everything is ASCII - size_t final_pos = pos + 16; - while (pos < final_pos) { - *latin_output++ = char(buf[pos]); - pos++; - } - continue; - } - } +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t + convert_utf16le_to_latin1(const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_utf16be_to_latin1(const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( + const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( + const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf16le_to_latin1(const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf16be_to_latin1(const char16_t *buf, size_t len, + char *latin1_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - // suppose it is not an all ASCII byte sequence - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *latin_output++ = char(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == - 0b11000000) { // the first three bits indicate: - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - return 0; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } // checks if the next byte is a valid continuation byte in UTF-8. A - // valid continuation byte starts with 10. - // range check - - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | - (data[pos + 1] & - 0b00111111); // assembles the Unicode code point from the two bytes. - // It does this by discarding the leading 110 and 10 - // bits from the two bytes, shifting the remaining bits - // of the first byte, and then combining the results - // with a bitwise OR operation. - if (code_point < 0x80 || 0xFF < code_point) { - return 0; // We only care about the range 129-255 which is Non-ASCII - // latin1 characters. A code_point beneath 0x80 is invalid as - // it is already covered by bytes whose leading bit is zero. - } - *latin_output++ = char(code_point); - pos += 2; - } else { - return 0; - } - } - return latin_output - start; -} +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t convert_utf16le_to_utf8( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_utf16be_to_utf8( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( + const char16_t *buf, size_t len, char *utf8_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -inline result convert_with_errors(const char *buf, size_t len, - char *latin_output) { - const uint8_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{latin_output}; +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t convert_utf32_to_utf8( + const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused result convert_utf32_to_utf8_with_errors( + const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; + simdutf_warn_unused size_t convert_valid_utf32_to_utf8( + const char32_t *buf, size_t len, char *utf8_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 - // 1000 1000...etc - if ((v & 0x8080808080808080) == - 0) { // if NONE of these are set, e.g. all of them are zero, then - // everything is ASCII - size_t final_pos = pos + 16; - while (pos < final_pos) { - *latin_output++ = char(buf[pos]); - pos++; - } - continue; - } - } - // suppose it is not an all ASCII byte sequence - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *latin_output++ = char(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == - 0b11000000) { // the first three bits indicate: - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - return result(error_code::TOO_SHORT, pos); - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return result(error_code::TOO_SHORT, pos); - } // checks if the next byte is a valid continuation byte in UTF-8. A - // valid continuation byte starts with 10. - // range check - - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | - (data[pos + 1] & - 0b00111111); // assembles the Unicode code point from the two bytes. - // It does this by discarding the leading 110 and 10 - // bits from the two bytes, shifting the remaining bits - // of the first byte, and then combining the results - // with a bitwise OR operation. - if (code_point < 0x80) { - return result(error_code::OVERLONG, pos); - } - if (0xFF < code_point) { - return result(error_code::TOO_LARGE, pos); - } // We only care about the range 129-255 which is Non-ASCII latin1 - // characters - *latin_output++ = char(code_point); - pos += 2; - } else if ((leading_byte & 0b11110000) == 0b11100000) { - // We have a three-byte UTF-8 - return result(error_code::TOO_LARGE, pos); - } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 - // we have a 4-byte UTF-8 word. - return result(error_code::TOO_LARGE, pos); - } else { - // we either have too many continuation bytes or an invalid leading byte - if ((leading_byte & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, pos); - } +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t + convert_utf32_to_latin1(const char32_t *buf, size_t len, + char *latin1_output) const noexcept final; + simdutf_warn_unused result + convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, + char *latin1_output) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf32_to_latin1(const char32_t *buf, size_t len, + char *latin1_output) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 - return result(error_code::HEADER_BITS, pos); - } - } - return result(error_code::SUCCESS, latin_output - start); -} - -inline result rewind_and_convert_with_errors(size_t prior_bytes, - const char *buf, size_t len, - char *latin1_output) { - size_t extra_len{0}; - // We potentially need to go back in time and find a leading byte. - // In theory '3' would be sufficient, but sometimes the error can go back - // quite far. - size_t how_far_back = prior_bytes; - // size_t how_far_back = 3; // 3 bytes in the past + current position - // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } - bool found_leading_bytes{false}; - // important: it is i <= how_far_back and not 'i < how_far_back'. - for (size_t i = 0; i <= how_far_back; i++) { - unsigned char byte = buf[-static_cast(i)]; - found_leading_bytes = ((byte & 0b11000000) != 0b10000000); - if (found_leading_bytes) { - if (i > 0 && byte < 128) { - // If we had to go back and the leading byte is ascii - // then we can stop right away. - return result(error_code::TOO_LONG, 0 - i + 1); - } - buf -= i; - extra_len = i; - break; - } - } - // - // It is possible for this function to return a negative count in its result. - // C++ Standard Section 18.1 defines size_t is in which is described - // in C Standard as . C Standard Section 4.1.5 defines size_t as an - // unsigned integral type of the result of the sizeof operator - // - // An unsigned type will simply wrap round arithmetically (well defined). - // - if (!found_leading_bytes) { - // If how_far_back == 3, we may have four consecutive continuation bytes!!! - // [....] [continuation] [continuation] [continuation] | [buf is - // continuation] Or we possibly have a stream that does not start with a - // leading byte. - return result(error_code::TOO_LONG, 0 - how_far_back); - } - result res = convert_with_errors(buf, len + extra_len, latin1_output); - if (res.error) { - res.count -= extra_len; - } - return res; -} +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t + convert_utf32_to_utf16le(const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_utf32_to_utf16be(const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( + const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( + const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf32_to_utf16le(const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf32_to_utf16be(const char32_t *buf, size_t len, + char16_t *utf16_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_utf16le_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_utf16be_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( + const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; + simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( + const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf16le_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; + simdutf_warn_unused size_t + convert_valid_utf16be_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_buffer) const noexcept final; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -} // namespace utf8_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +#if SIMDUTF_FEATURE_UTF16 + void change_endianness_utf16(const char16_t *buf, size_t length, + char16_t *output) const noexcept final; + simdutf_warn_unused size_t + count_utf16le(const char16_t *buf, size_t length) const noexcept override; + simdutf_warn_unused size_t + count_utf16be(const char16_t *buf, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF16 -#endif -/* end file src/scalar/utf8_to_latin1/utf8_to_latin1.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf16_to_latin1/utf16_to_latin1.h */ -#ifndef SIMDUTF_UTF16_TO_LATIN1_H -#define SIMDUTF_UTF16_TO_LATIN1_H +#if SIMDUTF_FEATURE_UTF8 + simdutf_warn_unused size_t count_utf8(const char *buf, + size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_latin1 { +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf8_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf8_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -#include // for std::memcpy +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t utf32_length_from_utf16le( + const char16_t *input, size_t length) const noexcept override; + simdutf_warn_unused size_t utf32_length_from_utf16be( + const char16_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -template -inline size_t convert(const char16_t *buf, size_t len, char *latin_output) { - if (len == 0) { - return 0; - } - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *current_write = latin_output; - uint16_t word = 0; - uint16_t too_large = 0; +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + simdutf_warn_unused size_t utf16_length_from_utf8( + const char *input, size_t length) const noexcept override; + simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( + const char16_t *input, size_t length) const noexcept override; + ; + simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( + const char16_t *input, size_t length) const noexcept override; + ; - while (pos < len) { - word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - too_large |= word; - *current_write++ = char(word & 0xFF); - pos++; - } - if ((too_large & 0xFF00) != 0) { - return 0; - } + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; - return current_write - latin_output; -} + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept override; -template -inline result convert_with_errors(const char16_t *buf, size_t len, - char *latin_output) { - if (len == 0) { - return result(error_code::SUCCESS, 0); - } - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{latin_output}; - uint16_t word; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - while (pos < len) { - if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that - // they are Latin1 - uint64_t v1, v2, v3, v4; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); - ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); - ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); - - if (!match_system(big_endian)) { - v1 = (v1 >> 8) | (v1 << (64 - 8)); - } - if (!match_system(big_endian)) { - v2 = (v2 >> 8) | (v2 << (64 - 8)); - } - if (!match_system(big_endian)) { - v3 = (v3 >> 8) | (v3 << (64 - 8)); - } - if (!match_system(big_endian)) { - v4 = (v4 >> 8) | (v4 << (64 - 8)); - } +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t utf8_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 - if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { - size_t final_pos = pos + 16; - while (pos < final_pos) { - *latin_output++ = !match_system(big_endian) - ? char(u16_swap_bytes(data[pos])) - : char(data[pos]); - pos++; - } - continue; - } - } - word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - if ((word & 0xFF00) == 0) { - *latin_output++ = char(word & 0xFF); - pos++; - } else { - return result(error_code::TOO_LARGE, pos); - } - } - return result(error_code::SUCCESS, latin_output - start); -} +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t utf16_length_from_utf32( + const char32_t *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -} // namespace utf16_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 + simdutf_warn_unused size_t utf32_length_from_utf8( + const char *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -#endif -/* end file src/scalar/utf16_to_latin1/utf16_to_latin1.h */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf32_to_latin1/utf32_to_latin1.h */ -#ifndef SIMDUTF_UTF32_TO_LATIN1_H -#define SIMDUTF_UTF32_TO_LATIN1_H +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t latin1_length_from_utf8( + const char *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_latin1 { +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + simdutf_warn_unused size_t utf8_length_from_latin1( + const char *input, size_t length) const noexcept override; +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -inline size_t convert(const char32_t *buf, size_t len, char *latin1_output) { - const uint32_t *data = reinterpret_cast(buf); - char *start = latin1_output; - uint32_t utf32_char; - size_t pos = 0; - uint32_t too_large = 0; +#if SIMDUTF_FEATURE_BASE64 + simdutf_warn_unused result base64_to_binary( + const char *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused full_result base64_to_binary_details( + const char *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused result base64_to_binary( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + simdutf_warn_unused full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, + base64_options options, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) const noexcept override; + size_t binary_to_base64(const char *input, size_t length, char *output, + base64_options options) const noexcept override; + size_t + binary_to_base64_with_lines(const char *input, size_t length, char *output, + size_t line_length, + base64_options options) const noexcept override; + const char *find(const char *start, const char *end, + char character) const noexcept override; + const char16_t *find(const char16_t *start, const char16_t *end, + char16_t character) const noexcept override; - while (pos < len) { - utf32_char = (uint32_t)data[pos]; - too_large |= utf32_char; - *latin1_output++ = (char)(utf32_char & 0xFF); - pos++; - } - if ((too_large & 0xFFFFFF00) != 0) { - return 0; - } - return latin1_output - start; -} +#endif // SIMDUTF_FEATURE_BASE64 +}; +} // namespace fallback +} // namespace simdutf -inline result convert_with_errors(const char32_t *buf, size_t len, - char *latin1_output) { - const uint32_t *data = reinterpret_cast(buf); - char *start{latin1_output}; - size_t pos = 0; - while (pos < len) { - if (pos + 2 <= - len) { // if it is safe to read 8 more bytes, check that they are Latin1 - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0xFFFFFF00FFFFFF00) == 0) { - *latin1_output++ = char(buf[pos]); - *latin1_output++ = char(buf[pos + 1]); - pos += 2; - continue; - } - } - uint32_t utf32_char = data[pos]; - if ((utf32_char & 0xFFFFFF00) == - 0) { // Check if the character can be represented in Latin-1 - *latin1_output++ = (char)(utf32_char & 0xFF); - pos++; - } else { - return result(error_code::TOO_LARGE, pos); - }; - } - return result(error_code::SUCCESS, latin1_output - start); -} +#endif // SIMDUTF_FALLBACK_IMPLEMENTATION_H +/* end file src/simdutf/fallback/implementation.h */ -} // namespace utf32_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +/* begin file src/simdutf/fallback/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "fallback" +// #define SIMDUTF_IMPLEMENTATION fallback +/* end file src/simdutf/fallback/begin.h */ -#endif -/* end file src/scalar/utf32_to_latin1/utf32_to_latin1.h */ -#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 + // Declarations +/* begin file src/simdutf/fallback/bitmanipulation.h */ +#ifndef SIMDUTF_FALLBACK_BITMANIPULATION_H +#define SIMDUTF_FALLBACK_BITMANIPULATION_H -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ -#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H -#define SIMDUTF_VALID_UTF8_TO_LATIN1_H +#include namespace simdutf { -namespace scalar { -namespace { -namespace utf8_to_latin1 { - -inline size_t convert_valid(const char *buf, size_t len, char *latin_output) { - const uint8_t *data = reinterpret_cast(buf); +namespace fallback { +namespace {} // unnamed namespace +} // namespace fallback +} // namespace simdutf - size_t pos = 0; - char *start{latin_output}; +#endif // SIMDUTF_FALLBACK_BITMANIPULATION_H +/* end file src/simdutf/fallback/bitmanipulation.h */ - while (pos < len) { - // try to convert the next block of 16 ASCII bytes - if (pos + 16 <= - len) { // if it is safe to read 16 more bytes, check that they are ascii - uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | - v2}; // We are only interested in these bits: 1000 1000 1000 - // 1000, so it makes sense to concatenate everything - if ((v & 0x8080808080808080) == - 0) { // if NONE of these are set, e.g. all of them are zero, then - // everything is ASCII - size_t final_pos = pos + 16; - while (pos < final_pos) { - *latin_output++ = char(buf[pos]); - pos++; - } - continue; - } - } +/* begin file src/simdutf/fallback/end.h */ +/* end file src/simdutf/fallback/end.h */ - // suppose it is not an all ASCII byte sequence - uint8_t leading_byte = data[pos]; // leading byte - if (leading_byte < 0b10000000) { - // converting one ASCII byte !!! - *latin_output++ = char(leading_byte); - pos++; - } else if ((leading_byte & 0b11100000) == - 0b11000000) { // the first three bits indicate: - // We have a two-byte UTF-8 - if (pos + 1 >= len) { - break; - } // minimal bound checking - if ((data[pos + 1] & 0b11000000) != 0b10000000) { - return 0; - } // checks if the next byte is a valid continuation byte in UTF-8. A - // valid continuation byte starts with 10. - // range check - - uint32_t code_point = - (leading_byte & 0b00011111) << 6 | - (data[pos + 1] & - 0b00111111); // assembles the Unicode code point from the two bytes. - // It does this by discarding the leading 110 and 10 - // bits from the two bytes, shifting the remaining bits - // of the first byte, and then combining the results - // with a bitwise OR operation. - *latin_output++ = char(code_point); - pos += 2; - } else { - // we may have a continuation but we do not do error checking - return 0; - } - } - return latin_output - start; -} +#endif // SIMDUTF_IMPLEMENTATION_FALLBACK +#endif // SIMDUTF_FALLBACK_H +/* end file src/simdutf/fallback.h */ +#ifndef SIMDUTF_REGULAR_VISUAL_STUDIO +SIMDUTF_POP_DISABLE_WARNINGS +#endif -} // namespace utf8_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +// The scalar routines should be included once. +#if SIMDUTF_FEATURE_ASCII +#endif // SIMDUTF_FEATURE_ASCII +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING || \ + (SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1) +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING || + // (SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1) +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING +#if SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_BASE64 +#endif // SIMDUTF_FEATURE_BASE64 -#endif -/* end file src/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ -#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H -#define SIMDUTF_VALID_UTF16_TO_LATIN1_H +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -namespace simdutf { -namespace scalar { -namespace { -namespace utf16_to_latin1 { +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -template -inline size_t convert_valid(const char16_t *buf, size_t len, - char *latin_output) { - const uint16_t *data = reinterpret_cast(buf); - size_t pos = 0; - char *start{latin_output}; - uint16_t word = 0; +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 - while (pos < len) { - word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; - *latin_output++ = char(word); - pos++; - } +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 - return latin_output - start; -} +#if SIMDUTF_FEATURE_UTF8 && \ + (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_LATIN1) +#endif // SIMDUTF_FEATURE_UTF8 && (SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_LATIN1) -} // namespace utf16_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 -#endif -/* end file src/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ -#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H -#define SIMDUTF_VALID_UTF32_TO_LATIN1_H - -namespace simdutf { -namespace scalar { -namespace { -namespace utf32_to_latin1 { - -inline size_t convert_valid(const char32_t *buf, size_t len, - char *latin1_output) { - const uint32_t *data = reinterpret_cast(buf); - char *start = latin1_output; - uint32_t utf32_char; - size_t pos = 0; - - while (pos < len) { - utf32_char = (uint32_t)data[pos]; - - if (pos + 2 <= - len) { // if it is safe to read 8 more bytes, check that they are Latin1 - uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); - if ((v & 0xFFFFFF00FFFFFF00) == 0) { - *latin1_output++ = char(buf[pos]); - *latin1_output++ = char(buf[pos + 1]); - pos += 2; - continue; - } else { - // output can not be represented in latin1 - return 0; - } - } - if ((utf32_char & 0xFFFFFF00) == 0) { - *latin1_output++ = char(utf32_char); - } else { - // output can not be represented in latin1 - return 0; - } - pos++; - } - return latin1_output - start; -} +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -} // namespace utf32_to_latin1 -} // unnamed namespace -} // namespace scalar -} // namespace simdutf +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -#endif -/* end file src/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 /* begin file src/implementation.cpp */ -#include #include +#include #include #if SIMDUTF_ATOMIC_REF -/* begin file src/scalar/atomic_util.h */ -#ifndef SIMDUTF_ATOMIC_UTIL_H -#define SIMDUTF_ATOMIC_UTIL_H -#if SIMDUTF_ATOMIC_REF - #include -namespace simdutf { -namespace scalar { - -// This function is a memcpy that uses atomic operations to read from the -// source. -inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { - static_assert(std::atomic_ref::required_alignment == sizeof(char), - "std::atomic_ref requires the same alignment as char_type"); - // We expect all 64-bit systems to be able to read 64-bit words from an - // aligned memory region atomically. You might be able to do better on - // specific systems, e.g., x64 systems can read 128-bit words atomically. - constexpr size_t alignment = sizeof(uint64_t); - - // Lambda for atomic byte-by-byte copy - auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, - size_t bytelen) noexcept { - char *mutable_src = const_cast(bytesrc); - for (size_t j = 0; j < bytelen; ++j) { - bytedst[j] = - std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); - } - }; - - // Handle unaligned start - size_t offset = reinterpret_cast(src) % alignment; - if (offset) { - size_t to_align = std::min(len, alignment - offset); - bbb_memcpy_atomic_read(dst, src, to_align); - src += to_align; - dst += to_align; - len -= to_align; - } - - // Process aligned 64-bit chunks - while (len >= alignment) { - auto *src_aligned = reinterpret_cast(const_cast(src)); - const auto dst_value = - std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); - std::memcpy(dst, &dst_value, sizeof(uint64_t)); - src += alignment; - dst += alignment; - len -= alignment; - } - - // Handle remaining bytes - if (len) { - bbb_memcpy_atomic_read(dst, src, len); - } -} - -// This function is a memcpy that uses atomic operations to write to the -// destination. -inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { - static_assert(std::atomic_ref::required_alignment == sizeof(char), - "std::atomic_ref requires the same alignment as char"); - // We expect all 64-bit systems to be able to write 64-bit words to an aligned - // memory region atomically. - // You might be able to do better on specific systems, e.g., x64 systems can - // write 128-bit words atomically. - constexpr size_t alignment = sizeof(uint64_t); - - // Lambda for atomic byte-by-byte write - auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, - size_t bytelen) noexcept { - for (size_t j = 0; j < bytelen; ++j) { - std::atomic_ref(bytedst[j]) - .store(bytesrc[j], std::memory_order_relaxed); - } - }; - - // Handle unaligned start - size_t offset = reinterpret_cast(dst) % alignment; - if (offset) { - size_t to_align = std::min(len, alignment - offset); - bbb_memcpy_atomic_write(dst, src, to_align); - dst += to_align; - src += to_align; - len -= to_align; - } - - // Process aligned 64-bit chunks - while (len >= alignment) { - auto *dst_aligned = reinterpret_cast(dst); - uint64_t src_val; - std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src - std::atomic_ref(*dst_aligned) - .store(src_val, std::memory_order_relaxed); - dst += alignment; - src += alignment; - len -= alignment; - } + #include +#endif - // Handle remaining bytes - if (len) { - bbb_memcpy_atomic_write(dst, src, len); - } -} -} // namespace scalar -} // namespace simdutf -#endif // SIMDUTF_ATOMIC_REF -#endif // SIMDUTF_ATOMIC_UTIL_H -/* end file src/scalar/atomic_util.h */ +// The macro SIMDUTF_USE_STATIC_INITIALIZATION, when set to 1, means that we +// will use translation-unit-scope variables to hold our implementations. +// +// The downside of a translation-unit-scope variable is that the initialization +// order is not well defined, thus if someone uses simdutf before main() starts, +// they might get a crash. Thus setting SIMDUTF_USE_STATIC_INITIALIZATION to 1 +// is not recommended if you are using simdutf in a library that might be used +// by other code before main() starts. However, the upside is that there is no +// synchronization overhead on every call to get_active_implementation(). When +// compiling without the c++ standard library, we use static initialization, +// because C++ relies on the standard library for thread-safe initialization of +// function-scope static variables. +// +// By default, we avoid translation-unit-scope static initialization, so we set +// SIMDUTF_USE_STATIC_INITIALIZATION to 0. It comes with a small performance +// cost on the first call to get_active_implementation(), and a smaller cost on +// subsequent calls but it is then safe to use the simdutf library in static +// initialization. +// +// Further reading: https://en.cppreference.com/cpp/language/siof +#ifndef SIMDUTF_USE_STATIC_INITIALIZATION + #if SIMDUTF_NO_LIBCXX + #define SIMDUTF_USE_STATIC_INITIALIZATION 1 + #else // SIMDUTF_NO_LIBCXX + #define SIMDUTF_USE_STATIC_INITIALIZATION 0 + #endif // SIMDUTF_NO_LIBCXX +#endif // SIMDUTF_USE_STATIC_INITIALIZATION + +// When building without libc++abi (SIMDUTF_NO_LIBCXX=1) on GCC/Clang, provide +// a weak stub for __cxa_pure_virtual so the abstract implementation vtable +// does not drag in libc++abi just for this unreachable hook. Kept weak so a +// real libc++abi definition wins if one is linked in anyway. +#if SIMDUTF_NO_LIBCXX +extern "C" __attribute__((weak, noreturn)) void __cxa_pure_virtual() { + __builtin_trap(); +} +namespace std { +__attribute__((weak, noreturn)) void +__glibcxx_assert_fail(const char *, int, const char *, const char *) noexcept { + __builtin_trap(); +} +} // namespace std #endif static_assert(sizeof(uint8_t) == sizeof(char), @@ -17069,22 +12529,6 @@ static_assert(sizeof(uint32_t) == sizeof(char32_t), // next line is redundant, but it is kept to catch defective systems. static_assert(CHAR_BIT == 8, "simdutf requires 8-bit bytes"); -// Useful for debugging purposes -namespace simdutf { -namespace { - -template std::string toBinaryString(T b) { - std::string binary = ""; - T mask = T(1) << (sizeof(T) * CHAR_BIT - 1); - while (mask > 0) { - binary += ((b & mask) == 0) ? '0' : '1'; - mask >>= 1; - } - return binary; -} -} // namespace -} // namespace simdutf - namespace simdutf { bool implementation::supported_by_runtime_system() const { uint32_t required_instruction_sets = this->required_instruction_sets(); @@ -17147,6 +12591,17 @@ simdutf_warn_unused size_t implementation::maximal_binary_length_from_base64( const char16_t *input, size_t length) const noexcept { return scalar::base64::maximal_binary_length_from_base64(input, length); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return scalar::base64::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return scalar::base64::binary_length_from_base64(input, length); +} + simdutf_warn_unused size_t implementation::base64_length_from_binary( size_t length, base64_options options) const noexcept { return scalar::base64::base64_length_from_binary(length, options); @@ -17164,66 +12619,108 @@ namespace internal { SIMDUTF_IMPLEMENTATION_LASX + SIMDUTF_IMPLEMENTATION_FALLBACK == \ 1) -// Static array of known implementations. We are hoping these get baked into the -// executable without requiring a static initializer. - #if SIMDUTF_IMPLEMENTATION_ICELAKE + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const icelake::implementation icelake_singleton{}; + #endif static const icelake::implementation *get_icelake_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const icelake::implementation icelake_singleton{}; + #endif return &icelake_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_HASWELL + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const haswell::implementation haswell_singleton{}; + #endif static const haswell::implementation *get_haswell_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const haswell::implementation haswell_singleton{}; + #endif return &haswell_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_WESTMERE + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const westmere::implementation westmere_singleton{}; + #endif static const westmere::implementation *get_westmere_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const westmere::implementation westmere_singleton{}; + #endif return &westmere_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_ARM64 + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const arm64::implementation arm64_singleton{}; + #endif static const arm64::implementation *get_arm64_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const arm64::implementation arm64_singleton{}; + #endif return &arm64_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_PPC64 + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const ppc64::implementation ppc64_singleton{}; + #endif static const ppc64::implementation *get_ppc64_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const ppc64::implementation ppc64_singleton{}; + #endif return &ppc64_singleton; } #endif #if SIMDUTF_IMPLEMENTATION_RVV + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const rvv::implementation rvv_singleton{}; + #endif static const rvv::implementation *get_rvv_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const rvv::implementation rvv_singleton{}; + #endif return &rvv_singleton; } #endif -#if SIMDUTF_IMPLEMENTATION_LSX -static const lsx::implementation *get_lsx_singleton() { - static const lsx::implementation lsx_singleton{}; - return &lsx_singleton; -} -#endif #if SIMDUTF_IMPLEMENTATION_LASX + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const lasx::implementation lasx_singleton{}; + #endif static const lasx::implementation *get_lasx_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const lasx::implementation lasx_singleton{}; + #endif return &lasx_singleton; } #endif +#if SIMDUTF_IMPLEMENTATION_LSX + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const lsx::implementation lsx_singleton{}; + #endif +static const lsx::implementation *get_lsx_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION + static const lsx::implementation lsx_singleton{}; + #endif + return &lsx_singleton; +} +#endif #if SIMDUTF_IMPLEMENTATION_FALLBACK + #if SIMDUTF_USE_STATIC_INITIALIZATION +static const fallback::implementation fallback_singleton{}; + #endif static const fallback::implementation *get_fallback_singleton() { + #if !SIMDUTF_USE_STATIC_INITIALIZATION static const fallback::implementation fallback_singleton{}; + #endif return &fallback_singleton; } #endif #if SIMDUTF_SINGLE_IMPLEMENTATION -static const implementation *get_single_implementation() { +simdutf_really_inline static const implementation *get_single_implementation() { return #if SIMDUTF_IMPLEMENTATION_ICELAKE get_icelake_singleton(); @@ -17240,12 +12737,12 @@ static const implementation *get_single_implementation() { #if SIMDUTF_IMPLEMENTATION_PPC64 get_ppc64_singleton(); #endif - #if SIMDUTF_IMPLEMENTATION_LSX - get_lsx_singleton(); - #endif #if SIMDUTF_IMPLEMENTATION_LASX get_lasx_singleton(); #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(); + #endif #if SIMDUTF_IMPLEMENTATION_FALLBACK get_fallback_singleton(); #endif @@ -17258,8 +12755,8 @@ static const implementation *get_single_implementation() { class detect_best_supported_implementation_on_first_use final : public implementation { public: - std::string name() const noexcept final { return set_best()->name(); } - std::string description() const noexcept final { + std::string_view name() const noexcept final { return set_best()->name(); } + std::string_view description() const noexcept final { return set_best()->description(); } uint32_t required_instruction_sets() const noexcept final { @@ -17459,6 +12956,20 @@ class detect_best_supported_implementation_on_first_use final return set_best()->utf8_length_from_utf16be_with_replacement(input, length); } + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept final override { + return set_best()->convert_utf16le_to_utf8_with_replacement(input, length, + utf8_buffer); + } + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept final override { + return set_best()->convert_utf16be_to_utf8_with_replacement(input, length, + utf8_buffer); + } + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -17828,6 +13339,16 @@ class detect_best_supported_implementation_on_first_use final char16_t character) const noexcept override { return set_best()->find(start, end, character); } + + simdutf_warn_unused size_t binary_length_from_base64( + const char *input, size_t length) const noexcept override { + return set_best()->binary_length_from_base64(input, length); + } + + simdutf_warn_unused size_t binary_length_from_base64( + const char16_t *input, size_t length) const noexcept override { + return set_best()->binary_length_from_base64(input, length); + } #endif // SIMDUTF_FEATURE_BASE64 simdutf_really_inline @@ -17845,38 +13366,72 @@ static_assert(std::is_trivially_destructible< "detect_best_supported_implementation_on_first_use should be " "trivially destructible"); +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const std::initializer_list + available_implementation_pointers{ + #if SIMDUTF_IMPLEMENTATION_ICELAKE + get_icelake_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_HASWELL + get_haswell_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_WESTMERE + get_westmere_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_ARM64 + get_arm64_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_PPC64 + get_ppc64_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_RVV + get_rvv_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_LASX + get_lasx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_FALLBACK + get_fallback_singleton(), + #endif + }; +#endif static const std::initializer_list & get_available_implementation_pointers() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const std::initializer_list available_implementation_pointers{ -#if SIMDUTF_IMPLEMENTATION_ICELAKE + #if SIMDUTF_IMPLEMENTATION_ICELAKE get_icelake_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_HASWELL + #endif + #if SIMDUTF_IMPLEMENTATION_HASWELL get_haswell_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_WESTMERE + #endif + #if SIMDUTF_IMPLEMENTATION_WESTMERE get_westmere_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_ARM64 + #endif + #if SIMDUTF_IMPLEMENTATION_ARM64 get_arm64_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_PPC64 + #endif + #if SIMDUTF_IMPLEMENTATION_PPC64 get_ppc64_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_RVV + #endif + #if SIMDUTF_IMPLEMENTATION_RVV get_rvv_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_LSX - get_lsx_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_LASX + #endif + #if SIMDUTF_IMPLEMENTATION_LASX get_lasx_singleton(), -#endif -#if SIMDUTF_IMPLEMENTATION_FALLBACK + #endif + #if SIMDUTF_IMPLEMENTATION_LSX + get_lsx_singleton(), + #endif + #if SIMDUTF_IMPLEMENTATION_FALLBACK get_fallback_singleton(), + #endif + }; #endif - }; // available_implementation_pointers return available_implementation_pointers; } @@ -17927,7 +13482,7 @@ class unsupported_implementation final : public implementation { } #endif // SIMDUTF_FEATURE_ASCII -#if SIMDUTF_FEATURE_ASCII +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *, size_t) const noexcept final override { @@ -17939,7 +13494,7 @@ class unsupported_implementation final : public implementation { size_t) const noexcept final override { return false; } -#endif // SIMDUTF_FEATURE_ASCII +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused bool @@ -18066,6 +13621,16 @@ class unsupported_implementation final : public implementation { return {OTHER, 0}; // Not supported } + simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *, size_t, char *) const noexcept final override { + return 0; // Not supported + } + + simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *, size_t, char *) const noexcept final override { + return 0; // Not supported + } + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -18373,6 +13938,14 @@ class unsupported_implementation final : public implementation { char16_t) const noexcept override { return nullptr; } + simdutf_warn_unused size_t + binary_length_from_base64(const char *, size_t) const noexcept override { + return 0; + } + simdutf_warn_unused size_t + binary_length_from_base64(const char16_t *, size_t) const noexcept override { + return 0; + } #endif // SIMDUTF_FEATURE_BASE64 unsupported_implementation() @@ -18380,8 +13953,13 @@ class unsupported_implementation final : public implementation { "Unsupported CPU (no detected SIMD instructions)", 0) {} }; +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const unsupported_implementation unsupported_singleton{}; +#endif const unsupported_implementation *get_unsupported_singleton() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const unsupported_implementation unsupported_singleton{}; +#endif return &unsupported_singleton; } static_assert(std::is_trivially_destructible::value, @@ -18441,42 +14019,68 @@ detect_best_supported_implementation_on_first_use::set_best() const noexcept { /** * The list of available implementations compiled into simdutf. */ +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const internal::available_implementation_list + available_implementations_instance{}; +#endif SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list & get_available_implementations() { +#if !SIMDUTF_USE_STATIC_INITIALIZATION static const internal::available_implementation_list - available_implementations{}; - return available_implementations; + available_implementations_instance{}; +#endif + return available_implementations_instance; } +#if SIMDUTF_USE_STATIC_INITIALIZATION && !SIMDUTF_SINGLE_IMPLEMENTATION +static const internal::detect_best_supported_implementation_on_first_use + detect_best_supported_implementation_on_first_use_singleton; +#endif + +#if SIMDUTF_USE_STATIC_INITIALIZATION +static internal::atomic_ptr + active_implementation_instance{ + #if SIMDUTF_SINGLE_IMPLEMENTATION + internal::get_single_implementation() + #else + &detect_best_supported_implementation_on_first_use_singleton + #endif + }; +#endif + /** * The active implementation. */ SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr & get_active_implementation() { -#if SIMDUTF_SINGLE_IMPLEMENTATION - // skip runtime detection - static internal::atomic_ptr active_implementation{ - internal::get_single_implementation()}; - return active_implementation; -#else +#if !SIMDUTF_USE_STATIC_INITIALIZATION + #if !SIMDUTF_SINGLE_IMPLEMENTATION static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; - static internal::atomic_ptr active_implementation{ - &detect_best_supported_implementation_on_first_use_singleton}; - return active_implementation; + #endif + static internal::atomic_ptr + active_implementation_instance{ + #if SIMDUTF_SINGLE_IMPLEMENTATION + internal::get_single_implementation() + #else + &detect_best_supported_implementation_on_first_use_singleton + #endif + }; #endif + return active_implementation_instance; } #if SIMDUTF_SINGLE_IMPLEMENTATION -const implementation *get_default_implementation() { +simdutf_really_inline const implementation *get_default_implementation() { return internal::get_single_implementation(); } #else -internal::atomic_ptr &get_default_implementation() { +simdutf_really_inline internal::atomic_ptr & +get_default_implementation() { return get_active_implementation(); } #endif -#define SIMDUTF_GET_CURRENT_IMPLEMENTION +#define SIMDUTF_GET_CURRENT_IMPLEMENTATION #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { @@ -18555,12 +14159,9 @@ simdutf_warn_unused size_t convert_latin1_to_utf32( return get_default_implementation()->convert_latin1_to_utf32(buf, len, latin1_output); } -simdutf_warn_unused size_t latin1_length_from_utf32(size_t length) noexcept { - return length; -} -simdutf_warn_unused size_t utf32_length_from_latin1(size_t length) noexcept { - return length; -} +// moved to the header file +// simdutf_warn_unused size_t latin1_length_from_utf32(size_t length) noexcept +// simdutf_warn_unused size_t utf32_length_from_latin1(size_t length) noexcept #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -18659,7 +14260,9 @@ simdutf_warn_unused bool validate_utf16le(const char16_t *buf, size_t len) noexcept { return get_default_implementation()->validate_utf16le(buf, len); } +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +#if SIMDUTF_FEATURE_BASE64 #if SIMDUTF_ATOMIC_REF template simdutf_warn_unused result atomic_base64_to_binary_safe_impl( @@ -18682,7 +14285,7 @@ simdutf_warn_unused result atomic_base64_to_binary_safe_impl( result r; while (!last_chunk) { last_chunk |= (temp_buffer.size() >= outlen - actual_out); - size_t temp_outlen = (std::min)(temp_buffer.size(), outlen - actual_out); + size_t temp_outlen = (detail::min)(temp_buffer.size(), outlen - actual_out); r = base64_to_binary_safe(input, length, temp_buffer.data(), temp_outlen, options, last_chunk_handling_options, decode_up_to_bad_char); @@ -18749,7 +14352,7 @@ simdutf_warn_unused result atomic_base64_to_binary_safe( } #endif // SIMDUTF_ATOMIC_REF -#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING +#endif // SIMDUTF_FEATURE_BASE64 #if SIMDUTF_FEATURE_UTF16 simdutf_warn_unused bool validate_utf16be(const char16_t *buf, @@ -18837,7 +14440,7 @@ convert_utf16_to_utf8_safe(const char16_t *buf, size_t len, char *utf8_output, // The worst case for convert_utf16_to_utf8 is when you go from 1 char16_t // to 3 characters of UTF-8. So we can read at most utf8_len / 3 char16_t // characters. - auto read_len = std::min(len, utf8_len / 3); + auto read_len = detail::min(len, utf8_len / 3); if (read_len <= 16) { break; } @@ -18929,12 +14532,9 @@ simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( return get_default_implementation()->convert_utf16be_to_latin1_with_errors( buf, len, latin1_buffer); } -simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept { - return length; -} -simdutf_warn_unused size_t utf16_length_from_latin1(size_t length) noexcept { - return length; -} +// moved to header file +// simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept +// simdutf_warn_unused size_t utf16_length_from_latin1(size_t length) noexcept #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -19190,12 +14790,12 @@ simdutf_warn_unused size_t count_utf16be(const char16_t *input, } #endif // SIMDUTF_FEATURE_UTF16 -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t count_utf8(const char *input, size_t length) noexcept { return get_default_implementation()->count_utf8(input, length); } -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 simdutf_warn_unused size_t latin1_length_from_utf8(const char *buf, @@ -19215,8 +14815,7 @@ simdutf_warn_unused size_t utf8_length_from_latin1(const char *buf, simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, size_t length) noexcept { #if SIMDUTF_IS_BIG_ENDIAN - result r = utf8_length_from_utf16be_with_replacement(input, length); - return r.count; + return utf8_length_from_utf16be(input, length); #else return utf8_length_from_utf16le(input, length); #endif @@ -19275,6 +14874,27 @@ simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( ->utf8_length_from_utf16be_with_replacement(input, length); } +simdutf_warn_unused size_t convert_utf16_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf16be_to_utf8_with_replacement(input, length, utf8_buffer); + #else + return convert_utf16le_to_utf8_with_replacement(input, length, utf8_buffer); + #endif +} + +simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept { + return get_default_implementation()->convert_utf16le_to_utf8_with_replacement( + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept { + return get_default_implementation()->convert_utf16be_to_utf8_with_replacement( + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -19300,23 +14920,22 @@ simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, #if SIMDUTF_FEATURE_BASE64 -simdutf_warn_unused size_t -base64_length_from_binary(size_t length, base64_options option) noexcept { - return scalar::base64::base64_length_from_binary(length, option); -} +// this has been moved to implementation.h +// simdutf_warn_unused size_t +// base64_length_from_binary(size_t length, base64_options option) noexcept; -simdutf_warn_unused size_t base64_length_from_binary_with_lines( - size_t length, base64_options options, size_t line_length) noexcept { - return scalar::base64::base64_length_from_binary_with_lines(length, options, - line_length); -} +// this has been moved to implementation.h +// simdutf_warn_unused size_t base64_length_from_binary_with_lines( +// size_t length, base64_options options, size_t line_length) noexcept; +// } -simdutf_warn_unused const char *find(const char *start, const char *end, - char character) noexcept { +simdutf_warn_unused const char *detail::find(const char *start, const char *end, + char character) noexcept { return get_default_implementation()->find(start, end, character); } -simdutf_warn_unused const char16_t * -find(const char16_t *start, const char16_t *end, char16_t character) noexcept { +simdutf_warn_unused const char16_t *detail::find(const char16_t *start, + const char16_t *end, + char16_t character) noexcept { return get_default_implementation()->find(start, end, character); } @@ -19339,6 +14958,16 @@ simdutf_warn_unused size_t maximal_binary_length_from_base64( input, length); } +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) noexcept { + return get_default_implementation()->binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) noexcept { + return get_default_implementation()->binary_length_from_base64(input, length); +} + simdutf_warn_unused result base64_to_binary( const char16_t *input, size_t length, char *output, base64_options options, last_chunk_handling_options last_chunk_handling_options) noexcept { @@ -19346,167 +14975,37 @@ simdutf_warn_unused result base64_to_binary( input, length, output, options, last_chunk_handling_options); } -template -simdutf_warn_unused result slow_base64_to_binary_safe_impl( - const chartype *input, size_t length, char *output, size_t &outlen, - base64_options options, - last_chunk_handling_options last_chunk_options) noexcept { - const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; - auto ri = simdutf::scalar::base64::find_end(input, length, options); - size_t equallocation = ri.equallocation; - size_t equalsigns = ri.equalsigns; - length = ri.srclen; - size_t full_input_length = ri.full_input_length; - (void)full_input_length; - if (length == 0) { - outlen = 0; - if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation}; - } - return {SUCCESS, 0}; - } - - // The parameters of base64_tail_decode_safe are: - // - dst: the output buffer - // - outlen: the size of the output buffer - // - srcr: the input buffer - // - length: the size of the input buffer - // - padded_characters: the number of padding characters - // - options: the options for the base64 decoder - // - last_chunk_options: the options for the last chunk - // The function will return the number of bytes written to the output buffer - // and the number of bytes read from the input buffer. - // The function will also return an error code if the input buffer is not - // valid base64. - full_result r = scalar::base64::base64_tail_decode_safe( - output, outlen, input, length, equalsigns, options, last_chunk_options); - r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, - full_input_length, last_chunk_options); - outlen = r.output_count; - if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && - equalsigns > 0) { - // additional checks - if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { - r.error = error_code::INVALID_BASE64_CHARACTER; - } - } - return {r.error, r.input_count}; // we cannot return r itself because it gets - // converted to error/output_count -} -simdutf_warn_unused bool base64_ignorable(char input, - base64_options options) noexcept { - return scalar::base64::is_ignorable(input, options); -} -simdutf_warn_unused bool base64_ignorable(char16_t input, - base64_options options) noexcept { - return scalar::base64::is_ignorable(input, options); -} -simdutf_warn_unused bool base64_valid(char input, - base64_options options) noexcept { - return scalar::base64::is_base64(input, options); -} -simdutf_warn_unused bool base64_valid(char16_t input, - base64_options options) noexcept { - return scalar::base64::is_base64(input, options); -} -simdutf_warn_unused bool -base64_valid_or_padding(char input, base64_options options) noexcept { - return scalar::base64::is_base64_or_padding(input, options); -} -simdutf_warn_unused bool -base64_valid_or_padding(char16_t input, base64_options options) noexcept { - return scalar::base64::is_base64_or_padding(input, options); +simdutf_warn_unused full_result base64_to_binary_details( + const char *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_handling_options) noexcept { + return get_default_implementation()->base64_to_binary_details( + input, length, output, options, last_chunk_handling_options); } -template -simdutf_warn_unused result base64_to_binary_safe_impl( - const chartype *input, size_t length, char *output, size_t &outlen, - base64_options options, - last_chunk_handling_options last_chunk_handling_options, - bool decode_up_to_bad_char) noexcept { - static_assert(std::is_same::value || - std::is_same::value, - "Only char and char16_t are supported."); - size_t remaining_input_length = length; - size_t remaining_output_length = outlen; - size_t input_position = 0; - size_t output_position = 0; - - // We also do a first pass using the fast path to decode as much as possible - size_t safe_input = (std::min)( - remaining_input_length, - base64_length_from_binary(remaining_output_length / 3 * 3, options)); - bool done_with_partial = (safe_input == remaining_input_length); - simdutf::full_result r = - get_default_implementation()->base64_to_binary_details( - input + input_position, safe_input, output + output_position, options, - done_with_partial - ? last_chunk_handling_options - : simdutf::last_chunk_handling_options::only_full_chunks); - simdutf_log_assert(r.input_count <= safe_input, - "You should not read more than safe_input"); - simdutf_log_assert(r.output_count <= remaining_output_length, - "You should not write more than remaining_output_length"); - // Technically redundant, but we want to be explicit about it. - input_position += r.input_count; - output_position += r.output_count; - remaining_input_length -= r.input_count; - remaining_output_length -= r.output_count; - if (r.error != simdutf::error_code::SUCCESS) { - // There is an error. We return. - if (decode_up_to_bad_char && - r.error == error_code::INVALID_BASE64_CHARACTER) { - return slow_base64_to_binary_safe_impl( - input, length, output, outlen, options, last_chunk_handling_options); - } - outlen = output_position; - return {r.error, input_position}; - } - - if (done_with_partial) { - // We are done. We have decoded everything. - outlen = output_position; - return {simdutf::error_code::SUCCESS, input_position}; - } - // We have decoded some data, but we still have some data to decode. - // We need to decode the rest of the input buffer. - r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( - input + input_position, remaining_input_length, output + output_position, - remaining_output_length, options, last_chunk_handling_options); - input_position += r.input_count; - output_position += r.output_count; - remaining_input_length -= r.input_count; - remaining_output_length -= r.output_count; - - if (r.error != simdutf::error_code::SUCCESS) { - // There is an error. We return. - if (decode_up_to_bad_char && - r.error == error_code::INVALID_BASE64_CHARACTER) { - return slow_base64_to_binary_safe_impl( - input, length, output, outlen, options, last_chunk_handling_options); - } - outlen = output_position; - return {r.error, input_position}; - } - if (input_position < length) { - // We cannot process the entire input in one go, so we need to - // process it in two steps: first the fast path, then the slow path. - // In some cases, the processing might 'eat up' trailing ignorable - // characters in the fast path, but that can be a problem. - // suppose we have just white space followed by a single base64 character. - // If we first process the white space with the fast path, it will - // eat all of it. But, by the JavaScript standard, we should consume - // no character. See - // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 - while (input_position > 0 && - base64_ignorable(input[input_position - 1], options)) { - input_position--; - } - } - outlen = output_position; - return {simdutf::error_code::SUCCESS, input_position}; +simdutf_warn_unused full_result base64_to_binary_details( + const char16_t *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_handling_options) noexcept { + return get_default_implementation()->base64_to_binary_details( + input, length, output, options, last_chunk_handling_options); } +// moved to implementation.h +// simdutf_warn_unused bool base64_ignorable(char input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_ignorable(char16_t input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_valid(char input, +// base64_options options) noexcept +// simdutf_warn_unused bool base64_valid(char16_t input, +// base64_options options) noexcept +// simdutf_warn_unused bool +// base64_valid_or_padding(char input, base64_options options) noexcept +// simdutf_warn_unused bool +// base64_valid_or_padding(char16_t input, base64_options options) noexcept + +// base64_to_binary_safe_impl is moved to +// include/simdutf/base64_implementation.h + #if SIMDUTF_ATOMIC_REF size_t atomic_binary_to_base64(const char *input, size_t length, char *output, base64_options options) noexcept { @@ -19520,7 +15019,7 @@ size_t atomic_binary_to_base64(const char *input, size_t length, char *output, #endif std::array inbuf; for (size_t i = 0; i < length; i += input_block_size) { - const size_t current_block_size = std::min(input_block_size, length - i); + const size_t current_block_size = detail::min(input_block_size, length - i); simdutf::scalar::memcpy_atomic_read(inbuf.data(), input + i, current_block_size); const size_t written = binary_to_base64(inbuf.data(), current_block_size, @@ -19540,7 +15039,7 @@ simdutf_warn_unused size_t convert_latin1_to_utf8_safe( while (true) { // convert_latin1_to_utf8 will never write more than input length * 2 - auto read_len = std::min(len, utf8_len >> 1); + auto read_len = detail::min(len, utf8_len >> 1); if (read_len <= 16) { break; } @@ -19607,11 +15106,18 @@ simdutf_warn_unused int detect_encodings(const char *buf, } #endif // SIMDUTF_FEATURE_DETECT_ENCODING +#if SIMDUTF_USE_STATIC_INITIALIZATION +static const implementation *const builtin_impl_instance = + get_available_implementations()[SIMDUTF_STRINGIFY( + SIMDUTF_BUILTIN_IMPLEMENTATION)]; +#endif const implementation *builtin_implementation() { - static const implementation *builtin_impl = +#if !SIMDUTF_USE_STATIC_INITIALIZATION + static const implementation *const builtin_impl_instance = get_available_implementations()[SIMDUTF_STRINGIFY( SIMDUTF_BUILTIN_IMPLEMENTATION)]; - return builtin_impl; +#endif + return builtin_impl_instance; } #if SIMDUTF_FEATURE_UTF8 @@ -19652,7 +15158,6 @@ SIMDUTF_DISABLE_UNDESIRED_WARNINGS /* begin file src/simdutf/arm64/begin.h */ // redefining SIMDUTF_IMPLEMENTATION to "arm64" // #define SIMDUTF_IMPLEMENTATION arm64 -#define SIMDUTF_SIMD_HAS_BYTEMASK 1 /* end file src/simdutf/arm64/begin.h */ namespace simdutf { namespace arm64 { @@ -19765,16 +15270,13 @@ convert_utf8_1_to_2_byte_to_utf16(uint8x16_t in, size_t shufutf8_idx) { /* begin file src/arm64/arm_utf16fix.cpp */ /* - * Returns if a vector of type uint8x16_t is all zero. + * Returns whether a vector of type uint8x16_t is not all zero. The input is + * always a combination of comparison masks (bytes equal to 0x00 or 0xff), so we + * can use the two-instruction test (shrn + fcmp) instead of a reduction + * followed by a costly move to a general-purpose register. */ -simdutf_really_inline int veq_non_zero(uint8x16_t v) { - // might compile to two instructions: - // umaxv s0, v0.4s - // fmov w0, s0 - // On Apple hardware, they both have a latency of 3 cycles, with a throughput - // of four instructions per cycle. So that's 6 cycles of latency (!!!) for the - // two instructions. A narrowing shift has the same latency and throughput. - return vmaxvq_u32(vreinterpretq_u32_u8(v)); +simdutf_really_inline bool veq_non_zero(uint8x16_t v) { + return any_lane_set(vreinterpretq_u16_u8(v)); } /* @@ -19792,7 +15294,7 @@ void utf16fix_block(char16_t *out, const char16_t *in) { uint8x16_t lb_masked, block_masked, lb_is_high, block_is_low; uint8x16_t illseq; - const int idx = !match_system(big_endian) ? 0 : 1; + constexpr int idx = !match_system(big_endian) ? 0 : 1; /* TODO: compute lookback using shifts */ lb = vld2q_u8((const uint8_t *)(in - 1)); @@ -19819,7 +15321,7 @@ void utf16fix_block(char16_t *out, const char16_t *in) { out[-1] = ill ? replacement : lbc; /* fix illegal sequencing in the main block */ - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { block.val[1] = vbslq_u8(block_illseq, vdupq_n_u8(0xfd), block.val[1]); block.val[0] = vorrq_u8(block_illseq, block.val[0]); } else { @@ -19835,7 +15337,7 @@ void utf16fix_block(char16_t *out, const char16_t *in) { template uint8x16_t get_mismatch_copy(const char16_t *in, char16_t *out) { - const int idx = !match_system(big_endian) ? 0 : 1; + constexpr int idx = !match_system(big_endian) ? 0 : 1; uint8x16x2_t lb = vld2q_u8((const uint8_t *)(in - 1)); uint8x16x2_t block = vld2q_u8((const uint8_t *)in); uint8x16_t lb_masked = vandq_u8(lb.val[idx], vdupq_n_u8(0xfc)); @@ -19966,7 +15468,7 @@ const char16_t *arm_validate_utf16(const char16_t *input, size_t size) { auto in0 = simd16(input); auto in1 = simd16(input + simd16::SIZE / sizeof(char16_t)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in0 = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in0))); in1 = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in1))); } @@ -20030,7 +15532,7 @@ const char16_t *arm_validate_utf16_as_ascii(const char16_t *input, uint16x8_t in1 = vld1q_u16(reinterpret_cast(input)); uint16x8_t in2 = vld1q_u16(reinterpret_cast(input + 8)); uint16x8_t inor = vorrq_u16(in1, in2); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { inor = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(inor))); } // next we compute inor > 0x7f @@ -20062,7 +15564,7 @@ const result arm_validate_utf16_with_errors(const char16_t *input, auto in1 = simd16(input + simd16::SIZE / sizeof(char16_t)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in0 = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in0))); in1 = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in1))); } @@ -20138,15 +15640,9 @@ const char32_t *arm_validate_utf32le(const char32_t *input, size_t size) { input += 4; } - uint32x4_t is_zero = - veorq_u32(vmaxq_u32(currentmax, standardmax), standardmax); - if (vmaxvq_u32(is_zero) != 0) { - return nullptr; - } - - is_zero = veorq_u32(vmaxq_u32(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (vmaxvq_u32(is_zero) != 0) { + const uint32x4_t too_large = vcgtq_u32(currentmax, standardmax); + const uint32x4_t surrogate = vcgtq_u32(currentoffsetmax, standardoffsetmax); + if (any_lane_set(vreinterpretq_u16_u32(vorrq_u32(too_large, surrogate)))) { return nullptr; } @@ -20169,15 +15665,16 @@ const result arm_validate_utf32le_with_errors(const char32_t *input, currentmax = vmaxq_u32(in, currentmax); currentoffsetmax = vmaxq_u32(vaddq_u32(in, offset), currentoffsetmax); - uint32x4_t is_zero = - veorq_u32(vmaxq_u32(currentmax, standardmax), standardmax); - if (vmaxvq_u32(is_zero) != 0) { - return result(error_code::TOO_LARGE, input - start); - } - - is_zero = veorq_u32(vmaxq_u32(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (vmaxvq_u32(is_zero) != 0) { + // Both accumulators are running maxima, so a single test per iteration is + // enough: we only need to tell the two error kinds apart once we know that + // one of them occurred. + const uint32x4_t too_large = vcgtq_u32(currentmax, standardmax); + const uint32x4_t surrogate = vcgtq_u32(currentoffsetmax, standardoffsetmax); + if (simdutf_unlikely(any_lane_set( + vreinterpretq_u16_u32(vorrq_u32(too_large, surrogate))))) { + if (any_lane_set(vreinterpretq_u16_u32(too_large))) { + return result(error_code::TOO_LARGE, input - start); + } return result(error_code::SURROGATE, input - start); } @@ -20200,12 +15697,12 @@ arm_convert_latin1_to_utf16(const char *buf, size_t len, while (end - buf >= 16) { uint8x16_t in8 = vld1q_u8(reinterpret_cast(buf)); uint16x8_t inlow = vmovl_u8(vget_low_u8(in8)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { inlow = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(inlow))); } vst1q_u16(reinterpret_cast(utf16_output), inlow); uint16x8_t inhigh = vmovl_u8(vget_high_u8(in8)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { inhigh = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(inhigh))); } vst1q_u16(reinterpret_cast(utf16_output + 8), inhigh); @@ -20434,7 +15931,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. uint16x4_t composed = convert_utf8_3_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = vreinterpret_u16_u8(vrev16_u8(vreinterpret_u8_u16(composed))); } vst1_u16(reinterpret_cast(utf16_output), composed); @@ -20448,7 +15945,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. uint16x8_t composed = convert_utf8_2_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } @@ -20471,7 +15968,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // Convert to UTF-16 uint16x8_t composed = convert_utf8_1_to_2_byte_to_utf16(in, idx); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } @@ -20518,7 +16015,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 3 byte: aaaabbbb bbcccccc uint16x4_t composed = vsli_n_u16(middlelow, highperm, 12); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = vreinterpret_u16_u8(vrev16_u8(vreinterpret_u8_u16(composed))); } vst1_u16(reinterpret_cast(utf16_output), composed); @@ -20573,7 +16070,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 110111CC CCDDDDDD|110110AA BBBBBBCC uint16x8_t composed = vaddq_u16(blend, magic_with_low_2); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } @@ -20678,7 +16175,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 4 byte: 110110AA BBBBBBCC|110111CC CCDDDDDD uint32x4_t selected = vbslq_u32(is_pair, surrogates, composed); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { selected = vreinterpretq_u32_u8(vrev16q_u8(vreinterpretq_u8_u32(selected))); } @@ -20904,7 +16401,7 @@ arm_convert_utf16_to_latin1(const char16_t *buf, size_t len, const char16_t *end = buf + len; while (end - buf >= 8) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } if (vmaxvq_u16(in) <= 0xff) { @@ -20930,7 +16427,7 @@ arm_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, const char16_t *end = buf + len; while (end - buf >= 8) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } if (vmaxvq_u16(in) <= 0xff) { @@ -20944,8 +16441,7 @@ arm_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } else { // Let us do a scalar fallback. for (int k = 0; k < 8; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if (word <= 0xff) { *latin1_output++ = char(word); } else { @@ -21026,7 +16522,7 @@ arm_convert_utf16_to_utf32(const char16_t *buf, size_t len, while (end - buf >= 8) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } @@ -21034,7 +16530,7 @@ arm_convert_utf16_to_utf32(const char16_t *buf, size_t len, vceqq_u16(vandq_u16(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (vmaxvq_u16(surrogates_bytemask) == 0) { + if (!any_lane_set(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units vst1q_u32(utf32_output, vmovl_u16(vget_low_u16(in))); @@ -21052,16 +16548,14 @@ arm_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -21098,7 +16592,7 @@ arm_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, while ((end - buf) >= 8) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } @@ -21106,7 +16600,7 @@ arm_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, vceqq_u16(vandq_u16(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (vmaxvq_u16(surrogates_bytemask) == 0) { + if (!any_lane_set(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units vst1q_u32(utf32_output, vmovl_u16(vget_low_u16(in))); @@ -21124,16 +16618,14 @@ arm_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -21221,7 +16713,7 @@ arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { // https://github.com/simdutf/simdutf/issues/92 while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } if (vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!! @@ -21229,7 +16721,7 @@ arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { // characters. uint16x8_t nextin = vld1q_u16(reinterpret_cast(buf) + 8); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { nextin = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(nextin))); } if (vmaxvq_u16(nextin) > 0x7F) { @@ -21304,7 +16796,7 @@ arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { vceqq_u16(vandq_u16(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (vmaxvq_u16(surrogates_bytemask) == 0) { + if (!any_lane_set(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint16x8_t dup_even = simdutf_make_uint16x8_t( @@ -21431,8 +16923,7 @@ arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -21445,9 +16936,8 @@ arm_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -21492,7 +16982,7 @@ arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { uint16x8_t in = vld1q_u16(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(in))); } if (vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!! @@ -21500,7 +16990,7 @@ arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, // characters. uint16x8_t nextin = vld1q_u16(reinterpret_cast(buf) + 8); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { nextin = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(nextin))); } if (vmaxvq_u16(nextin) > 0x7F) { @@ -21575,7 +17065,7 @@ arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, vceqq_u16(vandq_u16(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (vmaxvq_u16(surrogates_bytemask) == 0) { + if (!any_lane_set(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint16x8_t dup_even = simdutf_make_uint16x8_t( @@ -21702,8 +17192,7 @@ arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -21716,9 +17205,8 @@ arm_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -21766,7 +17254,7 @@ arm64_utf8_length_from_utf16_bytemask(const char16_t *in, size_t size) { auto base_input = vld2q_u8(reinterpret_cast(in + pos)); // size_t idx = 1; // we use the second lane of the deinterleaved load - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { idx = 0; } size_t idx_lsb = idx ^ 1; @@ -21847,14 +17335,14 @@ arm64_utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { for (; size - pos >= N + 1; pos += N) { auto base_input = vld2q_u8(reinterpret_cast(in + pos)); size_t idx = 1; // we use the second lane of the deinterleaved load - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { idx = 0; } size_t idx_lsb = idx ^ 1; auto is_surrogate = vcleq_u8( vsubq_u8(base_input.val[idx], vdupq_n_u8(0xd8)), vdupq_n_u8(7)); // We count on the fact that most inputs do not have surrogates. - if (vmaxvq_u32(vreinterpretq_u32_u8(is_surrogate)) || + if (any_lane_set(vreinterpretq_u16_u8(is_surrogate)) || scalar::utf16::is_low_surrogate(in[pos + N])) { any_surrogates = true; // there is at least one surrogate in the block @@ -21977,35 +17465,32 @@ arm64_utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { */ inline uint8x16_t insert_line_feed16(uint8x16_t input, size_t K) { static const uint8_t shuffle_masks[16][16] = { - {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; - // Prepare a vector with '\n' (0x0A) - uint8x16_t line_feed_vector = vdupq_n_u8('\n'); + {15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 15, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 15, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 15, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 15, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 15, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 15, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 15, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; + // Add '\n' (0x0A) at the end of input vector + uint8x16_t input_with_line_feed_vector = vsetq_lane_u8('\n', input, 15); // Load the precomputed shuffle mask for K uint8x16_t mask = vld1q_u8(shuffle_masks[K]); - // Create a mask where 0x80 indicates the line feed position - uint8x16_t lf_pos = vceqq_u8(mask, vdupq_n_u8(0x80)); - - uint8x16_t result = vqtbl1q_u8(input, mask); + // Perform the shuffle to reposition the K bytes including '\n' + uint8x16_t result = vqtbl1q_u8(input_with_line_feed_vector, mask); - // Use vbsl to select '\n' where lf_pos is true, else keep input bytes - return vbslq_u8(lf_pos, line_feed_vector, result); + return result; } // offset is the number of characters in the current line. @@ -22069,31 +17554,20 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, } // credit: Wojciech Muła uint8_t *out = (uint8_t *)dst; - constexpr static uint8_t source_table[64] = { - 'A', 'Q', 'g', 'w', 'B', 'R', 'h', 'x', 'C', 'S', 'i', 'y', 'D', - 'T', 'j', 'z', 'E', 'U', 'k', '0', 'F', 'V', 'l', '1', 'G', 'W', - 'm', '2', 'H', 'X', 'n', '3', 'I', 'Y', 'o', '4', 'J', 'Z', 'p', - '5', 'K', 'a', 'q', '6', 'L', 'b', 'r', '7', 'M', 'c', 's', '8', - 'N', 'd', 't', '9', 'O', 'e', 'u', '+', 'P', 'f', 'v', '/', - }; - constexpr static uint8_t source_table_url[64] = { - 'A', 'Q', 'g', 'w', 'B', 'R', 'h', 'x', 'C', 'S', 'i', 'y', 'D', - 'T', 'j', 'z', 'E', 'U', 'k', '0', 'F', 'V', 'l', '1', 'G', 'W', - 'm', '2', 'H', 'X', 'n', '3', 'I', 'Y', 'o', '4', 'J', 'Z', 'p', - '5', 'K', 'a', 'q', '6', 'L', 'b', 'r', '7', 'M', 'c', 's', '8', - 'N', 'd', 't', '9', 'O', 'e', 'u', '-', 'P', 'f', 'v', '_', - }; const uint8x16_t v3f = vdupq_n_u8(0x3f); #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO // When trying to load a uint8_t array, Visual Studio might // error with: error C2664: '__n128x4 neon_ld4m_q8(const char *)': // cannot convert argument 1 from 'const uint8_t [64]' to 'const char * - const uint8x16x4_t table = vld4q_u8( - (reinterpret_cast(options & base64_url) ? source_table_url - : source_table)); + const uint8x16x4_t table = + vld1q_u8_x4((reinterpret_cast(options & base64_url) + ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1)); #else const uint8x16x4_t table = - vld4q_u8((options & base64_url) ? source_table_url : source_table); + vld1q_u8_x4(reinterpret_cast( + (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1)); #endif size_t i = 0; for (; i + 16 * 3 <= srclen; i += 16 * 3) { @@ -22660,7 +18134,9 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; +#if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); +#endif std::memcpy(dst, &triple, 4); dst += 3; @@ -22672,7 +18148,9 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; +#if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); +#endif std::memcpy(dst, &triple, 3); dst += 3; @@ -23091,7 +18569,7 @@ arm_convert_utf32_to_utf16(const char32_t *buf, size_t len, vorrq_u16(vceqq_u16(vandq_u16(utf16_packed, v_f800), v_d800), forbidden_bytemask); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { utf16_packed = vreinterpretq_u16_u8( vrev16q_u8(vreinterpretq_u8_u16(utf16_packed))); } @@ -23114,7 +18592,7 @@ arm_convert_utf32_to_utf16(const char32_t *buf, size_t len, } // check for invalid input - if (vmaxvq_u32(vreinterpretq_u32_u16(forbidden_bytemask)) != 0) { + if (any_lane_set(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf16_output)); } @@ -23144,12 +18622,12 @@ arm_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, const uint16x8_t v_f800 = vmovq_n_u16((uint16_t)0xf800); const uint16x8_t forbidden_bytemask = vceqq_u16(vandq_u16(utf16_packed, v_f800), v_d800); - if (vmaxvq_u16(forbidden_bytemask) != 0) { + if (any_lane_set(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf16_output)); } - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { utf16_packed = vreinterpretq_u16_u8( vrev16q_u8(vreinterpretq_u8_u16(utf16_packed))); } @@ -23173,7 +18651,7 @@ arm_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); @@ -23457,7 +18935,7 @@ arm_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { } // while // check for invalid input - if (vmaxvq_u16(forbidden_bytemask) != 0) { + if (any_lane_set(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf8_output)); } return std::make_pair(buf, reinterpret_cast(utf8_output)); @@ -23548,7 +19026,7 @@ arm_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, const uint16x8_t v_dfff = vmovq_n_u16((uint16_t)0xdfff); const uint16x8_t forbidden_bytemask = vandq_u16( vcleq_u16(utf16_packed, v_dfff), vcgeq_u16(utf16_packed, v_d800)); - if (vmaxvq_u16(forbidden_bytemask) != 0) { + if (any_lane_set(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf8_output)); } @@ -23757,40 +19235,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -24032,10 +19476,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -24156,22 +19600,6 @@ namespace arm64 { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -24198,6 +19626,21 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace } // namespace arm64 @@ -24367,10 +19810,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -24455,10 +19898,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -24616,6 +20059,44 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace arm64 } // namespace simdutf /* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace arm64 { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 // transcoding from UTF-8 to UTF-32 @@ -24779,10 +20260,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -24866,10 +20347,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -24999,7 +20480,7 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t count = 0; for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); @@ -25017,7 +20498,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t *in, // This algorithm could no doubt be improved! for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t ascii_mask = input.lteq(0x7F); @@ -25083,8 +20564,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -25307,10 +20788,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -25396,10 +20877,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -25559,6 +21040,76 @@ simdutf_really_inline size_t convert_valid(const char *in, size_t size, // namespace simdutf /* end file src/generic/utf8_to_latin1/valid_utf8_to_latin1.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 +#if SIMDUTF_FEATURE_BASE64 +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace arm64 { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + // simd16x32::to_bitmask sets two bits per matching 16-bit lane, so the + // vectorized loop counted each unit twice. + count /= 2; + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace arm64 +} // namespace simdutf +/* end file src/generic/base64lengths.h */ +#endif // SIMDUTF_FEATURE_BASE64 // // Implementation-specific overrides @@ -26056,60 +21607,36 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return saved_bytes; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { std::pair ret = - arm_convert_utf16_to_utf8_with_errors(buf, len, - utf8_output); + arm_convert_utf16_to_utf8_with_errors(buf, len, utf8_output); if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } + return full_result(ret.first.error, ret.first.count, + size_t(ret.second - utf8_output)); } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + if (ret.first.count != len) { + full_result sres = + scalar::utf16_to_utf8::convert_with_errors( + buf + ret.first.count, len - ret.first.count, ret.second, 0); + return full_result(sres.error, ret.first.count + sres.input_count, + size_t(ret.second - utf8_output) + sres.output_count); + } + return full_result(error_code::SUCCESS, len, + size_t(ret.second - utf8_output)); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished - std::pair ret = - arm_convert_utf16_to_utf8_with_errors(buf, len, + return convert_utf16_to_utf8_with_details(buf, len, utf8_output); - if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } - } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -26552,6 +22079,26 @@ implementation::utf8_length_from_utf16be_with_replacement( length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -26770,13 +22317,22 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util_find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 } // namespace arm64 } // namespace simdutf /* begin file src/simdutf/arm64/end.h */ -#undef SIMDUTF_SIMD_HAS_BYTEMASK /* end file src/simdutf/arm64/end.h */ /* end file src/arm64/implementation.cpp */ #endif @@ -27230,33 +22786,7 @@ simdutf_warn_unused size_t implementation::latin1_length_from_utf8( #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 simdutf_warn_unused size_t implementation::utf8_length_from_latin1( const char *input, size_t length) const noexcept { - size_t answer = length; - size_t i = 0; - auto pop = [](uint64_t v) { - return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * - UINT64_C(0x0101010101010101) >> - 56); - }; - for (; i + 32 <= length; i += 32) { - uint64_t v; - memcpy(&v, input + i, 8); - answer += pop(v); - memcpy(&v, input + i + 8, sizeof(v)); - answer += pop(v); - memcpy(&v, input + i + 16, sizeof(v)); - answer += pop(v); - memcpy(&v, input + i + 24, sizeof(v)); - answer += pop(v); - } - for (; i + 8 <= length; i += 8) { - uint64_t v; - memcpy(&v, input + i, sizeof(v)); - answer += pop(v); - } - for (; i + 1 <= length; i += 1) { - answer += static_cast(input[i]) >> 7; - } - return answer; + return scalar::latin1_to_utf8::utf8_length_from_latin1(input, length); } #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -27305,6 +22835,20 @@ implementation::utf8_length_from_utf16be_with_replacement( endianness::BIG>(input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return scalar::utf16_to_utf8::convert_with_replacement( + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return scalar::utf16_to_utf8::convert_with_replacement( + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -27373,12 +22917,22 @@ size_t implementation::binary_to_base64_with_lines( const char *implementation::find(const char *start, const char *end, char character) const noexcept { - return std::find(start, end, character); + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { - return std::find(start, end, character); + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } #endif // SIMDUTF_FEATURE_BASE64 @@ -27411,6 +22965,49 @@ SIMDUTF_DISABLE_GCC_WARNING(-Wmaybe-uninitialized) // clang-format on #endif // end of workaround /* end file src/simdutf/icelake/begin.h */ +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + // transcoding from UTF-16 to UTF-8 (self-wrapping generic header; must be + // included at namespace scope zero, unlike icelake's own .inl.cpp files which + // are included inside the simdutf::icelake namespace below) +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace icelake { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace icelake +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 namespace simdutf { namespace icelake { namespace { @@ -27492,7 +23089,7 @@ using namespace simd; \ if (UTF32) { \ if (MASKED) { \ - const __mmask16 valid = uint16_t((1 << valid_count) - 1); \ + const __mmask16 valid = uint16_t((1U << valid_count) - 1); \ _mm512_mask_storeu_epi32((__m512i *)output, valid, out); \ } else { \ _mm512_storeu_si512((__m512i *)output, out); \ @@ -27513,7 +23110,7 @@ using namespace simd; { \ if (UTF32) { \ if (MASKED) { \ - const __mmask16 valid_mask = uint16_t((1 << VALID_COUNT) - 1); \ + const __mmask16 valid_mask = uint16_t((1U << VALID_COUNT) - 1); \ _mm512_mask_storeu_epi32((__m512i *)output, valid_mask, INPUT); \ } else { \ _mm512_storeu_si512((__m512i *)output, INPUT); \ @@ -27640,7 +23237,7 @@ process_block_utf8_to_utf16(const char *&in, char16_t *&out, size_t gap) { 0xdfdfdfdfdfdfdfdf, 0xdfdfdfdfdfdfdfdf, 0xdfdfdfdfdfdfdfdf, 0xdfdfdfdfdfdfdfdf, 0xdfdfdfdfdfdfdfdf); __m512i mask_c2c2c2c2 = _mm512_set1_epi32(0xc2c2c2c2); - __m512i mask_ffffffff = _mm512_set1_epi32(0xffffffff); + __m512i mask_ffffffff = _mm512_set1_epi16(int16_t(-1)); __m512i mask_d7c0d7c0 = _mm512_set1_epi32(0xd7c0d7c0); __m512i mask_dc00dc00 = _mm512_set1_epi32(0xdc00dc00); __m512i byteflip = _mm512_setr_epi64(0x0607040502030001, 0x0e0f0c0d0a0b0809, @@ -28063,7 +23660,7 @@ simdutf_really_inline size_t utf32_to_utf16_masked(const __m512i byteflip, unsigned int count, char16_t *output) { - const __mmask16 valid = uint16_t((1 << count) - 1); + const __mmask16 valid = uint16_t((1U << count) - 1); // 1. check if we have any surrogate pairs const __m512i v_0000_ffff = _mm512_set1_epi32(0x0000ffff); const __mmask16 sp_mask = @@ -28206,7 +23803,9 @@ simdutf_really_inline size_t utf32_to_utf16(const __m512i byteflip, __m512i compressed = _mm512_maskz_compress_epi16(nonzero, t5); _mm512_mask_storeu_epi16( output, - (1 << (count + static_cast(count_ones(sp_mask)))) - 1, + __mmask32((uint64_t(1) << (count + static_cast( + count_ones(sp_mask)))) - + 1), compressed); //_mm512_mask_compressstoreu_epi16(output, nonzero, t5); } @@ -28549,7 +24148,7 @@ valid_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28569,7 +24168,7 @@ valid_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); if (valid_count2 + valid_count3 <= 16) { vec2 = _mm512_mask_expand_epi32( - vec2, __mmask16(((1 << valid_count3) - 1) << valid_count2), vec3); + vec2, __mmask16(((1U << valid_count3) - 1) << valid_count2), vec3); valid_count2 += valid_count3; vec2 = expand_utf8_to_utf32(vec2); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec2, valid_count2, true) @@ -28600,7 +24199,7 @@ valid_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28671,7 +24270,7 @@ validating_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28691,7 +24290,7 @@ validating_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); if (valid_count2 + valid_count3 <= 16) { vec2 = _mm512_mask_expand_epi32( - vec2, __mmask16(((1 << valid_count3) - 1) << valid_count2), vec3); + vec2, __mmask16(((1U << valid_count3) - 1) << valid_count2), vec3); valid_count2 += valid_count3; vec2 = expand_utf8_to_utf32(vec2); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec2, valid_count2, true) @@ -28723,7 +24322,7 @@ validating_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28803,7 +24402,7 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str, __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28823,7 +24422,7 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str, __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); if (valid_count2 + valid_count3 <= 16) { vec2 = _mm512_mask_expand_epi32( - vec2, __mmask16(((1 << valid_count3) - 1) << valid_count2), vec3); + vec2, __mmask16(((1U << valid_count3) - 1) << valid_count2), vec3); valid_count2 += valid_count3; vec2 = expand_utf8_to_utf32(vec2); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec2, valid_count2, true) @@ -28859,7 +24458,7 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str, __m512i vec1 = expand_and_identify(lane1, lane2, valid_count1); if (valid_count0 + valid_count1 <= 16) { vec0 = _mm512_mask_expand_epi32( - vec0, __mmask16(((1 << valid_count1) - 1) << valid_count0), vec1); + vec0, __mmask16(((1U << valid_count1) - 1) << valid_count0), vec1); valid_count0 += valid_count1; vec0 = expand_utf8_to_utf32(vec0); SIMDUTF_ICELAKE_WRITE_UTF16_OR_UTF32(vec0, valid_count0, true) @@ -28908,12 +24507,11 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str, template simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; - }; - __m512i lookback, block, lb_masked, block_masked; __mmask32 lb_is_high, block_is_low, illseq; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); + }; lookback = _mm512_loadu_si512((const __m512i *)(in - 1)); block = _mm512_loadu_si512((const __m512i *)in); @@ -28959,14 +24557,14 @@ simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { * out-of-place operation. */ template -void utf16fix_runt(const char16_t *in, size_t n, char16_t *out) { +void utf16fix_short(const char16_t *in, size_t n, char16_t *out) { const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; - }; __m512i lookback, block, lb_masked, block_masked; __mmask32 lb_is_high, block_is_low, illseq; uint32_t mask = 0xFFFFFFFF >> (32 - n); + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); + }; lookback = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask << 1), (const uint16_t *)(in - 1)); block = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask), (const uint16_t *)in); @@ -29009,7 +24607,7 @@ void utf16fix_avx512(const char16_t *in, size_t n, char16_t *out) { if (n == 0) return; else if (n < 33) { - utf16fix_runt(in, n, out); + utf16fix_short(in, n, out); return; } out[0] = @@ -29247,7 +24845,7 @@ size_t icelake_convert_utf16_to_latin1(const char16_t *buf, size_t len, buf += 32; } if (buf < end) { - uint32_t mask(uint32_t(1 << (end - buf)) - 1); + uint32_t mask((1U << (end - buf)) - 1); __m512i in = _mm512_maskz_loadu_epi16(mask, buf); if (big_endian) { in = _mm512_shuffle_epi8(in, byteflip); @@ -29284,8 +24882,8 @@ icelake_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } if (_mm512_cmpgt_epu16_mask(in, v_0xFF)) { uint16_t word; - while ((word = (big_endian ? scalar::u16_swap_bytes(uint16_t(*buf)) - : uint16_t(*buf))) <= 0xff) { + while ((word = scalar::utf16::swap_if_needed( + uint16_t(*buf))) <= 0xff) { *latin1_output++ = uint8_t(word); buf++; } @@ -29299,7 +24897,7 @@ icelake_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, buf += 32; } if (buf < end) { - uint32_t mask(uint32_t(1 << (end - buf)) - 1); + uint32_t mask((1U << (end - buf)) - 1); __m512i in = _mm512_maskz_loadu_epi16(mask, buf); if (big_endian) { in = _mm512_shuffle_epi8(in, byteflip); @@ -29307,8 +24905,8 @@ icelake_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, if (_mm512_cmpgt_epu16_mask(in, v_0xFF)) { uint16_t word; - while ((word = (big_endian ? scalar::u16_swap_bytes(uint16_t(*buf)) - : uint16_t(*buf))) <= 0xff) { + while ((word = scalar::utf16::swap_if_needed( + uint16_t(*buf))) <= 0xff) { *latin1_output++ = uint8_t(word); buf++; } @@ -29639,7 +25237,7 @@ simdutf_really_inline size_t icelake_utf8_length_from_utf16(const char16_t *in, __m512i input2 = _mm512_loadu_si512(reinterpret_cast(in + pos + N)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input1 = _mm512_shuffle_epi8(input1, byteflip); input2 = _mm512_shuffle_epi8(input2, byteflip); } @@ -29672,7 +25270,7 @@ simdutf_really_inline size_t icelake_utf8_length_from_utf16(const char16_t *in, if (pos + N <= size) { __m512i input = _mm512_loadu_si512(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = _mm512_shuffle_epi8(input, byteflip); } // 0xd800 .. 0xdbff - low surrogate @@ -29699,7 +25297,7 @@ simdutf_really_inline size_t icelake_utf8_length_from_utf16(const char16_t *in, 0xFFFFFFFFULL >> (32 - (size - pos)); // mask for the remaining char16 values __m512i input = _mm512_maskz_loadu_epi16(remaining_mask, in + pos); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = _mm512_shuffle_epi8(input, byteflip); } // 0xd800 .. 0xdbff - low surrogate @@ -29746,9 +25344,9 @@ simdutf_really_inline result icelake_utf8_length_from_utf16_with_replacement( 0x0e0f0c0d0a0b0809, 0x0607040502030001, 0x0e0f0c0d0a0b0809, 0x0607040502030001, 0x0e0f0c0d0a0b0809); - const uint32_t straddle_mask = + constexpr uint32_t straddle_mask = match_system(big_endian) ? 0xfc00fc00 : 0x00fc00fc; - const uint32_t straddle_pair = + constexpr uint32_t straddle_pair = match_system(big_endian) ? 0xdc00d800 : 0x00dc00d8; size_t count = 0; @@ -29760,12 +25358,12 @@ simdutf_really_inline result icelake_utf8_length_from_utf16_with_replacement( for (; pos < (size - 1) / (2 * N) * (2 * N); pos += 2 * N) { __m512i current1 = _mm512_loadu_si512(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { current1 = _mm512_shuffle_epi8(current1, byteflip); } __m512i current2 = _mm512_loadu_si512(reinterpret_cast(in + pos + N)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { current2 = _mm512_shuffle_epi8(current2, byteflip); } @@ -29815,7 +25413,7 @@ simdutf_really_inline result icelake_utf8_length_from_utf16_with_replacement( if (pos + N + 1 <= size) { __m512i input = _mm512_loadu_si512(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = _mm512_shuffle_epi8(input, byteflip); } @@ -29846,10 +25444,10 @@ simdutf_really_inline result icelake_utf8_length_from_utf16_with_replacement( } size_t overshoot = 32 - (size - pos); - __mmask32 remaining_mask = 0xFFFFFFFFULL << overshoot; + __mmask32 remaining_mask(uint32_t(0xFFFFFFFFULL << overshoot)); __m512i input = _mm512_maskz_loadu_epi16(remaining_mask, in + pos - overshoot); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = _mm512_shuffle_epi8(input, byteflip); } @@ -29904,8 +25502,8 @@ convert_utf16_to_utf32(const char16_t *buf, size_t len, 0x0607040502030001, 0x0e0f0c0d0a0b0809, 0x0607040502030001, 0x0e0f0c0d0a0b0809, 0x0607040502030001, 0x0e0f0c0d0a0b0809, 0x0607040502030001, 0x0e0f0c0d0a0b0809); - while (std::distance(buf, end) >= 32) { - // Always safe because buf + 32 <= end so that end - buf >= 32 bytes: + while (end - buf >= 32) { + // Always safe because buf + 32 <= end so that end - buf >= 64 bytes: __m512i in = _mm512_loadu_si512((__m512i *)buf); if (big_endian) { in = _mm512_shuffle_epi8(in, byteflip); @@ -29995,7 +25593,7 @@ convert_utf16_to_utf32(const char16_t *buf, size_t len, // The following could be unsafe in some cases? //_mm512_storeu_epi32((__m512i *) utf32_output, compressed_second); _mm512_mask_storeu_epi32((__m512i *)utf32_output, - __mmask16((1 << howmany2) - 1), + __mmask16((1U << howmany2) - 1), compressed_second); utf32_output += howmany2; // Only process 31 code units, but keep track if the 31st word is a high @@ -30047,7 +25645,7 @@ size_t icelake_convert_utf32_to_latin1(const char32_t *buf, size_t len, buf += 16; } if (buf < end) { - uint16_t mask = uint16_t((1 << (end - buf)) - 1); + uint16_t mask = uint16_t((1U << (end - buf)) - 1); __m512i in = _mm512_maskz_loadu_epi32(mask, buf); if (_mm512_cmpgt_epu32_mask(in, v_0xFF)) { return 0; @@ -30085,7 +25683,7 @@ icelake_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, buf += 16; } if (buf < end) { - uint16_t mask = uint16_t((1 << (end - buf)) - 1); + uint16_t mask = uint16_t((1U << (end - buf)) - 1); __m512i in = _mm512_maskz_loadu_epi32(mask, buf); if (_mm512_cmpgt_epu32_mask(in, v_0xFF)) { while (uint32_t(*buf) <= 0xff) { @@ -30770,7 +26368,7 @@ avx512_convert_utf32_to_utf16(const char32_t *buf, size_t len, size_t remaining_len = size_t(end - buf); if (remaining_len) { - __mmask16 input_mask = __mmask16((1 << remaining_len) - 1); + __mmask16 input_mask = __mmask16((1U << remaining_len) - 1); __m512i in = _mm512_maskz_loadu_epi32(input_mask, buf); const __mmask16 saturation_bitmask = _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000) & @@ -30790,7 +26388,8 @@ avx512_convert_utf32_to_utf16(const char32_t *buf, size_t len, utf16_output += remaining_len; buf += remaining_len; } else { - const __mmask32 output_max_mask = (1 << (remaining_len * 2)) - 1; + const __mmask32 output_max_mask = + __mmask32((uint64_t(1) << (remaining_len * 2)) - 1); const __mmask32 output_mask = (~_pdep_u32(saturation_bitmask, 0xAAAAAAAA)) & output_max_mask; const __mmask16 surrogate_bitmask = @@ -30904,7 +26503,7 @@ avx512_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, code = error_code::SURROGATE; error_idx = surrogate_idx; } - output_mask &= ((1 << (2 * error_idx)) - 1); + output_mask &= __mmask32((uint64_t(1) << (2 * error_idx)) - 1); } __m512i v1, v2, v; in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000); @@ -30940,7 +26539,7 @@ avx512_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, size_t remaining_len = size_t(end - buf); if (remaining_len) { - __mmask16 input_mask = __mmask16((1 << remaining_len) - 1); + __mmask16 input_mask = __mmask16((1U << remaining_len) - 1); __m512i in = _mm512_maskz_loadu_epi32(input_mask, buf); const __mmask16 saturation_bitmask = _mm512_cmpeq_epi32_mask(_mm512_and_si512(in, v_ffff0000), v_00000000) & @@ -30966,7 +26565,8 @@ avx512_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, _mm256_mask_storeu_epi16(utf16_output, input_mask, utf16_packed); utf16_output += remaining_len; } else { - const __mmask32 output_max_mask = (1 << (remaining_len * 2)) - 1; + const __mmask32 output_max_mask = + __mmask32((uint64_t(1) << (remaining_len * 2)) - 1); __mmask32 output_mask = (~_pdep_u32(saturation_bitmask, 0xAAAAAAAA)) & output_max_mask; const __mmask16 surrogate_bitmask = @@ -30986,7 +26586,7 @@ avx512_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, code = error_code::SURROGATE; error_idx = surrogate_idx; } - output_mask &= ((1 << (2 * error_idx)) - 1); + output_mask &= __mmask32((uint64_t(1) << (2 * error_idx)) - 1); } __m512i v1, v2, v; in = _mm512_mask_sub_epi32(in, surrogate_bitmask, in, v_10000); @@ -31062,7 +26662,26 @@ bool validate_utf32(const char32_t *buf, size_t len) { __m512i currentmax = _mm512_setzero_si512(); __m512i currentoffsetmax = _mm512_setzero_si512(); - while (buf < end - 16) { + // Optimized: Process 32 values (2x 512-bit) per iteration for better + // throughput + while (end - buf >= 32) { + __m512i utf32_1 = _mm512_loadu_si512((const __m512i *)buf); + __m512i utf32_2 = _mm512_loadu_si512((const __m512i *)(buf + 16)); + buf += 32; + + // Process both blocks in parallel to maximize instruction-level parallelism + __m512i offsetmax_1 = _mm512_add_epi32(utf32_1, offset); + __m512i offsetmax_2 = _mm512_add_epi32(utf32_2, offset); + + currentoffsetmax = _mm512_max_epu32(offsetmax_1, currentoffsetmax); + currentmax = _mm512_max_epu32(utf32_1, currentmax); + + currentoffsetmax = _mm512_max_epu32(offsetmax_2, currentoffsetmax); + currentmax = _mm512_max_epu32(utf32_2, currentmax); + } + + // Handle remaining 16-31 values + if (end - buf >= 16) { __m512i utf32 = _mm512_loadu_si512((const __m512i *)buf); buf += 16; currentoffsetmax = @@ -31070,11 +26689,14 @@ bool validate_utf32(const char32_t *buf, size_t len) { currentmax = _mm512_max_epu32(utf32, currentmax); } - __m512i utf32 = - _mm512_maskz_loadu_epi32(__mmask16((1 << (end - buf)) - 1), buf); - currentoffsetmax = - _mm512_max_epu32(_mm512_add_epi32(utf32, offset), currentoffsetmax); - currentmax = _mm512_max_epu32(utf32, currentmax); + // Handle remaining 0-15 values with masked load + if (buf < end) { + __m512i utf32 = + _mm512_maskz_loadu_epi32(__mmask16((1U << (end - buf)) - 1), buf); + currentoffsetmax = + _mm512_max_epu32(_mm512_add_epi32(utf32, offset), currentoffsetmax); + currentmax = _mm512_max_epu32(utf32, currentmax); + } const __m512i standardmax = _mm512_set1_epi32((uint32_t)0x10ffff); const __m512i standardoffsetmax = _mm512_set1_epi32((uint32_t)0xfffff7ff); @@ -31228,7 +26850,7 @@ size_t icelake_convert_latin1_to_utf16(const char *latin1_input, size_t len, _mm512_storeu_si512((__m512i *)&utf16_output[i], out); } if (rounded_len != len) { - uint32_t mask = uint32_t(1 << (len - rounded_len)) - 1; + uint32_t mask = (1U << (len - rounded_len)) - 1; __m256i in = _mm256_maskz_loadu_epi8(mask, latin1_input + rounded_len); // Zero extend each set of 8 Latin1 characters to 32 16-bit integers @@ -31264,7 +26886,7 @@ void avx512_convert_latin1_to_utf32(const char *buf, size_t len, utf32_output += 16; } - __mmask16 mask = __mmask16((1 << len) - 1); + __mmask16 mask = __mmask16((1U << len) - 1); __m128i in = _mm_maskz_loadu_epi8(mask, buf); __m512i out = _mm512_cvtepu8_epi32(in); _mm512_mask_storeu_epi32((__m512i *)utf32_output, mask, out); @@ -31306,6 +26928,68 @@ struct block64 { __m512i chunks[1]; }; +static inline size_t write_multi_lf_m256i(__m256i chunk, uint8_t *out, + size_t output_len, size_t line_length, + size_t &offset) { + if (line_length >= 32) { + if (offset + output_len > line_length) { + __m512i expanded = _mm512_mask_expand_epi8( + _mm512_set1_epi8('\n'), ~(1ULL << (line_length - offset)), + _mm512_castsi256_si512(chunk)); + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), + (1ULL << (output_len + 1)) - 1, expanded); + offset = output_len - (line_length - offset); + return output_len + 1; + } else { + __mmask32 write_mask = + output_len == 32 ? 0xffffffff : ((__mmask32)1 << output_len) - 1; + _mm256_mask_storeu_epi8(reinterpret_cast<__m256i *>(out), write_mask, + chunk); + offset += output_len; + return output_len; + } + } else { + // minimum line_length starts from 4 + static const uint64_t masks[28] = { + 0x2700000842108421, 0x2600001041041041, 0x2500000810204081, + 0x2400000101010101, 0x2300000008040201, 0x2300000040100401, + 0x2300000200400801, 0x2200000001001001, 0x2200000004002001, + 0x2200000010004001, 0x2200000040008001, 0x2200000100010001, + 0x2100000000020001, 0x2100000000040001, 0x2100000000080001, + 0x2100000000100001, 0x2100000000200001, 0x2100000000400001, + 0x2100000000800001, 0x2100000001000001, 0x2100000002000001, + 0x2100000004000001, 0x2100000008000001, 0x2100000010000001, + 0x2100000020000001, 0x2100000040000001, 0x2100000080000001, + 0x2100000100000001, + }; + uint64_t mask = masks[line_length - 4]; + uint64_t max_width; + uint64_t num_lf; + if (output_len == 32) { + // use pre-computed width to avoid integer division in main loop + max_width = mask >> 56; + mask = (mask << (line_length - offset)) & ((1ULL << max_width) - 1); + num_lf = _mm_popcnt_u64(mask); + } else { + if (output_len <= line_length - offset) { + num_lf = 0; + } else { + num_lf = 1 + (output_len - (line_length - offset) - 1) / line_length; + } + max_width = num_lf + output_len; + mask = (mask << (line_length - offset)) & ((1ULL << max_width) - 1); + } + __mmask64 write_mask = (1ULL << (num_lf + output_len)) - 1; + __m512i expanded = + _mm512_mask_expand_epi8(_mm512_set1_epi8('\n'), ~((uint64_t)mask), + _mm512_castsi256_si512(chunk)); + _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), write_mask, + expanded); + offset = _lzcnt_u64(mask) - _lzcnt_u64(write_mask); + return num_lf + output_len; + } +} + template size_t encode_base64_impl(char *dst, const char *src, size_t srclen, base64_options options, @@ -31354,21 +27038,10 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, out += 65; offset = 64 - (line_length - offset); } else { // slow path - alignas(64) uint8_t local_buffer[64]; - _mm512_storeu_si512(reinterpret_cast<__m512i *>(local_buffer), - result); - size_t out_pos = 0; - size_t local_offset = offset; - for (size_t j = 0; j < 64;) { - if (local_offset == line_length) { - out[out_pos++] = '\n'; - local_offset = 0; - } - out[out_pos++] = local_buffer[j++]; - local_offset++; - } - offset = local_offset; - out += out_pos; + __m256i lo = _mm512_extracti64x4_epi64(result, 0); + __m256i hi = _mm512_extracti64x4_epi64(result, 1); + out += write_multi_lf_m256i(lo, out, 32, line_length, offset); + out += write_multi_lf_m256i(hi, out, 32, line_length, offset); } } else { _mm512_storeu_si512(reinterpret_cast<__m512i *>(out), result); @@ -31424,20 +27097,16 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, out += output_len + 1; } } else { - alignas(64) uint8_t local_buffer[64]; - _mm512_storeu_si512(reinterpret_cast<__m512i *>(local_buffer), result); - size_t out_pos = 0; - size_t local_offset = offset; - for (size_t j = 0; j < output_len;) { - if (local_offset == line_length) { - out[out_pos++] = '\n'; - local_offset = 0; - } - out[out_pos++] = local_buffer[j++]; - local_offset++; + if (output_len > 32) { + __m256i lo = _mm512_extracti64x4_epi64(result, 0); + __m256i hi = _mm512_extracti64x4_epi64(result, 1); + out += write_multi_lf_m256i(lo, out, 32, line_length, offset); + out += write_multi_lf_m256i(hi, out, output_len - 32, line_length, + offset); + } else { + __m256i lo = _mm512_extracti64x4_epi64(result, 0); + out += write_multi_lf_m256i(lo, out, output_len, line_length, offset); } - offset = local_offset; - out += out_pos; } } else { _mm512_mask_storeu_epi8(reinterpret_cast<__m512i *>(out), output_mask, @@ -31612,7 +27281,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && padding_characters > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -31710,8 +27379,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, // We never should have that the number of base64 characters + the // number of padding characters is more than 4. if (!ignore_garbage && (idx + padding_characters > 4)) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } // The idea here is that in loose mode, // if there is padding at all, it must be used @@ -31721,8 +27390,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, last_chunk_options == last_chunk_handling_options::loose && (idx >= 2) && padding_characters > 0 && ((idx + padding_characters) & 3) != 0) { - return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit), true}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } else // The idea here is that in strict mode, we do not want to accept // incomplete base64 chunks. So if the chunk was otherwise valid, we @@ -31731,9 +27400,12 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, last_chunk_options == last_chunk_handling_options::strict && (idx >= 2) && ((idx + padding_characters) & 3) != 0) { // The partial chunk was at src - idx - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; - return {BASE64_INPUT_REMAINDER, equallocation, size_t(dst - dstinit)}; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); + dst += output_len; + } + return {BASE64_INPUT_REMAINDER, equallocation, size_t(dst - dstinit), + true}; } else // If there is a partial chunk with insufficient padding, with // stop_before_partial, we need to just ignore it. In "only full" mode, @@ -31745,8 +27417,10 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (last_chunk_options == last_chunk_handling_options::only_full_chunks && (idx >= 2 || padding_characters == 0))) { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); + dst += output_len; + } // we need to rewind src to before the partial chunk size_t characters_to_skip = idx; while (characters_to_skip > 0) { @@ -31775,8 +27449,11 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, uint32_t triple = (uint32_t(bufferptr[-2]) << 3 * 6) + (uint32_t(bufferptr[-1]) << 2 * 6); if (triple & 0xffff) { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, + shuffled); + dst += output_len; + } return {BASE64_EXTRA_BITS, size_t(src - srcinit), size_t(dst - dstinit)}; } @@ -31792,8 +27469,11 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (uint32_t(bufferptr[-2]) << 2 * 6) + (uint32_t(bufferptr[-1]) << 1 * 6); if (triple & 0xff) { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, + shuffled); + dst += output_len; + } return {BASE64_EXTRA_BITS, size_t(src - srcinit), size_t(dst - dstinit)}; } @@ -31806,18 +27486,24 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (!is_partial(last_chunk_options) || (is_partial(last_chunk_options) && padding_characters > 0))) { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); + dst += output_len; + } return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), size_t(dst - dstinit)}; } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); + dst += output_len; + } return {INVALID_BASE64_CHARACTER, equallocation, - size_t(dst - dstinit)}; + size_t(dst - dstinit), true}; } else { - _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); - dst += output_len; + if (output_len > 0) { + _mm512_mask_storeu_epi8((__m512i *)dst, output_mask, shuffled); + dst += output_len; + } } } if (!ignore_garbage && !is_partial(last_chunk_options) && @@ -31825,7 +27511,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t output_count = size_t(dst - dstinit); if ((output_count % 3 == 0) || ((output_count % 3) + 1 + padding_characters != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, output_count}; + return {INVALID_BASE64_CHARACTER, equallocation, output_count, true}; } } return {SUCCESS, full_input_length, size_t(dst - dstinit)}; @@ -31834,11 +27520,86 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && padding_characters > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + padding_characters != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; } + +simdutf_warn_unused size_t icelake_binary_length_from_base64(const char *input, + size_t length) { + size_t count = 0; + const char *ptr = input; + const char *end = input + length; + + __m512i spaces = _mm512_set1_epi8(0x20); + while (ptr + 64 <= end) { + __m512i data = _mm512_loadu_si512(reinterpret_cast(ptr)); + uint64_t mask = _mm512_cmpgt_epi8_mask(data, spaces); + count += count_ones(mask); + ptr += 64; + } + + if (ptr < end) { + size_t len = end - ptr; + __mmask64 input_mask = ((__mmask64)1 << len) - 1; + __m512i data = _mm512_maskz_loadu_epi8( + input_mask, reinterpret_cast(ptr)); + uint64_t mask = _mm512_cmpgt_epi8_mask(data, spaces); + count += count_ones(mask); + ptr += len; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t +icelake_binary_length_from_base64(const char16_t *input, size_t length) { + size_t count = 0; + const char16_t *ptr = input; + const char16_t *end = input + length; + + __m512i spaces = _mm512_set1_epi16(0x20); + while (ptr + 32 <= end) { + __m512i data = _mm512_loadu_si512(reinterpret_cast(ptr)); + __mmask32 mask = _mm512_cmpgt_epi16_mask(data, spaces); + count += _mm_popcnt_u32(mask); + ptr += 32; + } + + if (ptr < end) { + size_t len = end - ptr; + __mmask32 input_mask = ((__mmask32)1 << len) - 1; + __m512i data = _mm512_maskz_loadu_epi16( + input_mask, reinterpret_cast(ptr)); + uint32_t mask = _mm512_cmpgt_epi16_mask(data, spaces); + count += _mm_popcnt_u32(mask); + ptr += len; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} /* end file src/icelake/icelake_base64.inl.cpp */ /* begin file src/icelake/icelake_find.inl.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, @@ -31860,6 +27621,8 @@ simdutf_really_inline const char *util_find(const char *start, const char *end, __m512i data = _mm512_maskz_loadu_epi8( load_mask, reinterpret_cast(start)); __mmask64 match_mask = _mm512_cmpeq_epi8_mask(data, char_vec); + match_mask &= load_mask; // When searching for null terminators, this + // prevents false positives if (match_mask != 0) { size_t index = _tzcnt_u64(match_mask); @@ -31949,6 +27712,8 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, __m512i data = _mm512_maskz_loadu_epi16( load_mask, reinterpret_cast(start)); __mmask32 match_mask = _mm512_cmpeq_epi16_mask(data, char_vec); + match_mask &= load_mask; // When searching for null terminators, this + // prevents false positives if (match_mask != 0) { size_t index = _tzcnt_u32(match_mask); @@ -32381,14 +28146,67 @@ implementation::validate_utf16le(const char16_t *buf, size_t len) const noexcept { const char16_t *end = buf + len; + // Optimized: Process 64 code units (2x 512-bit) per iteration + const __m512i surr_base = _mm512_set1_epi16(uint16_t(0xD800)); + const __m512i surr_range = _mm512_set1_epi16(uint16_t(0x0800)); + const __m512i high_range = _mm512_set1_epi16(uint16_t(0x0400)); + + for (; end - buf >= 64;) { + __m512i in_1 = _mm512_loadu_si512((__m512i *)buf); + __m512i in_2 = _mm512_loadu_si512((__m512i *)(buf + 32)); + + __m512i diff_1 = _mm512_sub_epi16(in_1, surr_base); + __m512i diff_2 = _mm512_sub_epi16(in_2, surr_base); + + __mmask32 surrogates_1 = _mm512_cmplt_epu16_mask(diff_1, surr_range); + __mmask32 surrogates_2 = _mm512_cmplt_epu16_mask(diff_2, surr_range); + + if (surrogates_1 | surrogates_2) { + __mmask32 highsurrogates_1 = _mm512_cmplt_epu16_mask(diff_1, high_range); + __mmask32 lowsurrogates_1 = surrogates_1 ^ highsurrogates_1; + + __mmask32 highsurrogates_2 = _mm512_cmplt_epu16_mask(diff_2, high_range); + __mmask32 lowsurrogates_2 = surrogates_2 ^ highsurrogates_2; + + // Validate first block: high must be followed by low + if ((highsurrogates_1 << 1) != lowsurrogates_1) { + return false; + } + + // Check boundary between blocks: if first block ends with high, second + // must start with low + bool ends_with_high_1 = ((highsurrogates_1 & 0x80000000) != 0); + bool starts_with_low_2 = ((lowsurrogates_2 & 0x1) != 0); + if (ends_with_high_1 && !starts_with_low_2) { + return false; + } + + // Validate second block (shift by 1 if first ended with high) + __mmask32 expected_low_2 = ends_with_high_1 + ? (highsurrogates_2 << 1) | 0x1 + : (highsurrogates_2 << 1); + if (expected_low_2 != lowsurrogates_2) { + return false; + } + + bool ends_with_high_2 = ((highsurrogates_2 & 0x80000000) != 0); + if (ends_with_high_2) { + buf += 63; // advance by 63 to start with high surrogate next round + } else { + buf += 64; + } + } else { + buf += 64; + } + } + + // Handle remaining 32-63 code units for (; end - buf >= 32;) { __m512i in = _mm512_loadu_si512((__m512i *)buf); - __m512i diff = _mm512_sub_epi16(in, _mm512_set1_epi16(uint16_t(0xD800))); - __mmask32 surrogates = - _mm512_cmplt_epu16_mask(diff, _mm512_set1_epi16(uint16_t(0x0800))); + __m512i diff = _mm512_sub_epi16(in, surr_base); + __mmask32 surrogates = _mm512_cmplt_epu16_mask(diff, surr_range); if (surrogates) { - __mmask32 highsurrogates = - _mm512_cmplt_epu16_mask(diff, _mm512_set1_epi16(uint16_t(0x0400))); + __mmask32 highsurrogates = _mm512_cmplt_epu16_mask(diff, high_range); __mmask32 lowsurrogates = surrogates ^ highsurrogates; // high must be followed by low if ((highsurrogates << 1) != lowsurrogates) { @@ -33021,32 +28839,32 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return outlen; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { size_t outlen; - size_t inlen = utf16_to_utf8_avx512i( + size_t inlen = utf16_to_utf8_avx512i( buf, len, (unsigned char *)utf8_output, &outlen); if (inlen != len) { - result res = scalar::utf16_to_utf8::convert_with_errors( - buf + inlen, len - inlen, utf8_output + outlen); - res.count += inlen; - return res; + full_result res = + scalar::utf16_to_utf8::convert_with_errors( + buf + inlen, len - inlen, utf8_output + outlen, 0); + return full_result(res.error, inlen + res.input_count, + outlen + res.output_count); } - return {simdutf::SUCCESS, outlen}; + return full_result(error_code::SUCCESS, len, outlen); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - size_t outlen; - size_t inlen = utf16_to_utf8_avx512i( - buf, len, (unsigned char *)utf8_output, &outlen); - if (inlen != len) { - result res = scalar::utf16_to_utf8::convert_with_errors( - buf + inlen, len - inlen, utf8_output + outlen); - res.count += inlen; - return res; - } - return {simdutf::SUCCESS, outlen}; + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -33599,7 +29417,7 @@ simdutf_warn_unused size_t implementation::utf16_length_from_utf8( /* 0110 */ 1, /* 0111 */ 1, - // continutation bytes + // continuation bytes /* 1000 */ 0, /* 1001 */ 0, /* 1010 */ 0, @@ -33672,6 +29490,26 @@ implementation::utf8_length_from_utf16be_with_replacement( input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t implementation::utf8_length_from_utf32( @@ -33863,6 +29701,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util_find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return icelake_binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return icelake_binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 } // namespace icelake @@ -33968,10 +29816,10 @@ simd8 utf16_gather_high_bytes(const simd16 &in0, */ template void utf16fix_block(char16_t *out, const char16_t *in) { - const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); }; + const char16_t replacement = scalar::utf16::replacement(); __m256i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low; __m256i illseq, lb_illseq, block_illseq, lb_illseq_shifted; @@ -34030,11 +29878,10 @@ void utf16fix_block(char16_t *out, const char16_t *in) { template void utf16fix_block_sse(char16_t *out, const char16_t *in) { - const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); }; - + const char16_t replacement = scalar::utf16::replacement(); __m128i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low; __m128i illseq, lb_illseq, block_illseq; @@ -34633,7 +30480,7 @@ avx2_convert_utf16_to_latin1(const char16_t *buf, size_t len, __m256i in1 = _mm256_loadu_si256(reinterpret_cast(buf + 16)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { const __m256i swap = _mm256_setr_epi8( 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); @@ -34668,7 +30515,7 @@ avx2_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, while (end - buf >= 16) { __m256i in = _mm256_loadu_si256(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { const __m256i swap = _mm256_setr_epi8( 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); @@ -34690,8 +30537,7 @@ avx2_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } else { // Fallback to scalar code for handling errors for (int k = 0; k < 16; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if (word <= 0xff) { *latin1_output++ = char(word); } else { @@ -35000,7 +30846,7 @@ avx2_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -35014,7 +30860,7 @@ avx2_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -35279,7 +31125,7 @@ avx2_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -35293,7 +31139,7 @@ avx2_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -35421,7 +31267,7 @@ avx2_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { // No surrogate pair *utf32_output++ = char32_t(word); @@ -35429,7 +31275,7 @@ avx2_convert_utf16_to_utf32(const char16_t *buf, size_t len, // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -35503,7 +31349,7 @@ avx2_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { // No surrogate pair *utf32_output++ = char32_t(word); @@ -35511,7 +31357,7 @@ avx2_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -36519,97 +32365,40 @@ simdutf_really_inline __m256i lookup_pshufb_improved(const __m256i input) { return _mm256_add_epi8(result, input); } -simdutf_really_inline __m256i insert_line_feed32(__m256i input, int K) { +simdutf_really_inline __m128i insert_line_feed16(__m128i input, int K) { + static const uint8_t shuffle_masks[16][16] = { + {15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 15, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 15, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 15, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 15, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 15, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 15, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 15, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; + input = _mm_insert_epi8(input, '\n', 15); + __m128i mask = _mm_loadu_si128((const __m128i *)shuffle_masks[K]); + return _mm_shuffle_epi8(input, mask); +} - static const uint8_t low_table[16][32] = { - {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; - static const uint8_t high_table[16][32] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; - - __m256i line_feed_vector = _mm256_set1_epi8('\n'); +simdutf_really_inline __m256i insert_line_feed32(__m256i input, int K) { + __m128i lo = _mm256_extracti128_si256(input, 0); + __m128i hi = _mm256_extracti128_si256(input, 1); if (K >= 16) { - __m256i mask = _mm256_loadu_si256((const __m256i *)high_table[K - 16]); - __m256i lf_pos = - _mm256_cmpeq_epi8(mask, _mm256_set1_epi8(static_cast(0x80))); - __m256i shuffled = _mm256_shuffle_epi8(input, mask); - __m256i result = _mm256_blendv_epi8(shuffled, line_feed_vector, lf_pos); - return result; + hi = insert_line_feed16(hi, K - 16); + } else { + hi = _mm_alignr_epi8(hi, lo, 15); + lo = insert_line_feed16(lo, K); } - // Shift input right by 1 byte - __m256i shift = _mm256_alignr_epi8( - input, _mm256_permute2x128_si256(input, input, 0x21), 15); - - input = _mm256_blend_epi32(input, shift, 0xF0); - - __m256i mask = _mm256_loadu_si256((const __m256i *)low_table[K]); - - __m256i lf_pos = - _mm256_cmpeq_epi8(mask, _mm256_set1_epi8(static_cast(0x80))); - __m256i shuffled = _mm256_shuffle_epi8(input, mask); - - __m256i result = _mm256_blendv_epi8(shuffled, line_feed_vector, lf_pos); + __m256i result = _mm256_castsi128_si256(lo); + result = _mm256_inserti128_si256(result, hi, 1); return result; } @@ -37232,6 +33021,73 @@ class block64 { return 63; } }; + +simdutf_warn_unused size_t avx2_binary_length_from_base64(const char *input, + size_t length) { + size_t count = 0; + const char *ptr = input; + const char *end = input + length; + + __m256i spaces = _mm256_set1_epi8(0x20); + while (ptr + 32 <= end) { + __m256i data = _mm256_loadu_si256(reinterpret_cast(ptr)); + __m256i gt_space = _mm256_cmpgt_epi8(data, spaces); + uint32_t mask = static_cast(_mm256_movemask_epi8(gt_space)); + count += count_ones(mask); + ptr += 32; + } + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t avx2_binary_length_from_base64(const char16_t *input, + size_t length) { + size_t count = 0; + const char16_t *ptr = input; + const char16_t *end = input + length; + + __m256i spaces = _mm256_set1_epi16(0x20); + while (ptr + 16 <= end) { + __m256i data = _mm256_loadu_si256(reinterpret_cast(ptr)); + __m256i gt_space = _mm256_cmpgt_epi16(data, spaces); + uint32_t mask = static_cast(_mm256_movemask_epi8(gt_space)); + count += count_ones(mask); + ptr += 16; + } + count /= 2; + + while (ptr < end) { + count += (*ptr > 0x20) ? 1 : 0; + ptr++; + } + + size_t padding = 0; + size_t pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} /* end file src/haswell/avx2_base64.cpp */ #endif // SIMDUTF_FEATURE_BASE64 @@ -37272,40 +33128,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -37547,10 +33369,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -37671,22 +33493,6 @@ namespace haswell { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -37713,6 +33519,21 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace } // namespace haswell @@ -37959,10 +33780,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -38047,10 +33868,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -38187,6 +34008,45 @@ simdutf_really_inline size_t utf16_length_from_utf8_bytemask(const char *in, } // namespace haswell } // namespace simdutf /* end file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ + // transcoding from UTF-16 to UTF-8 +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace haswell { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace haswell +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -38395,10 +34255,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -38482,10 +34342,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -38717,8 +34577,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -38807,7 +34667,7 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t count = 0; for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); @@ -38825,7 +34685,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t *in, // This algorithm could no doubt be improved! for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t ascii_mask = input.lteq(0x7F); @@ -38899,7 +34759,7 @@ simdutf_really_inline size_t utf8_length_from_utf16_bytemask(const char16_t *in, for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -38988,7 +34848,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { for (; pos < (size - 1) / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -39009,7 +34869,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { any_surrogates = true; auto input_next = vector_u16::load(reinterpret_cast(in + pos + 1)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_next = input_next.swap_bytes(); } @@ -39210,7 +35070,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, for (; pos < size / 32 * 32; pos += 32) { simd16x32 input_vec( reinterpret_cast(input + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_vec.swap_bytes(); } uint64_t matches = input_vec.lteq(uint16_t(0x7f)); @@ -39223,7 +35083,8 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, // Scalar tail while (pos < size) { - char16_t v = big_endian ? scalar::u16_swap_bytes(input[pos]) : input[pos]; + + char16_t v = scalar::utf16::swap_if_needed(input[pos]); if (v > 0x7F) { return result(error_code::TOO_LARGE, pos); } @@ -39388,10 +35249,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -39477,10 +35338,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -39669,7 +35530,7 @@ simdutf_really_inline bool validate(const char32_t *input, size_t size) { while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -39712,7 +35573,7 @@ simdutf_really_inline result validate_with_errors(const char32_t *input, while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -39803,7 +35664,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -39970,7 +35831,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -39996,8 +35858,8 @@ simdutf_really_inline const char *find(const char *start, const char *end, uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0) { size_t adjustment = 64 - misalignment; - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -40008,7 +35870,7 @@ simdutf_really_inline const char *find(const char *start, const char *end, } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 64; start += 64) { + for (; size_t(end - start) >= 64; start += 64) { simd8x64 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint8_t(character)); if (matches != 0) { @@ -40017,7 +35879,13 @@ simdutf_really_inline const char *find(const char *start, const char *end, return start + index; } } - return std::find(start, end, character); + // Handle remaining bytes with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } simdutf_really_inline const char16_t * @@ -40029,8 +35897,8 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0 && misalignment % 2 == 0) { size_t adjustment = (64 - misalignment) / sizeof(char16_t); - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -40041,7 +35909,7 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 32; start += 32) { + for (; size_t(end - start) >= 32; start += 32) { simd16x32 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint16_t(character)); if (matches != 0) { @@ -40050,7 +35918,13 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { return start + index; } } - return std::find(start, end, character); + // Handle remaining elements with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } } // namespace util @@ -40630,60 +36504,37 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return saved_bytes; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { std::pair ret = - haswell::avx2_convert_utf16_to_utf8_with_errors( - buf, len, utf8_output); + haswell::avx2_convert_utf16_to_utf8_with_errors(buf, len, + utf8_output); if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } + return full_result(ret.first.error, ret.first.count, + size_t(ret.second - utf8_output)); } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + if (ret.first.count != len) { + full_result sres = + scalar::utf16_to_utf8::convert_with_errors( + buf + ret.first.count, len - ret.first.count, ret.second, 0); + return full_result(sres.error, ret.first.count + sres.input_count, + size_t(ret.second - utf8_output) + sres.output_count); + } + return full_result(error_code::SUCCESS, len, + size_t(ret.second - utf8_output)); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished - std::pair ret = - haswell::avx2_convert_utf16_to_utf8_with_errors( - buf, len, utf8_output); - if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } - } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -41083,6 +36934,26 @@ implementation::utf8_length_from_utf16be_with_replacement( input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -41330,6 +37201,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util::find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return avx2_binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return avx2_binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 } // namespace haswell @@ -42407,7 +38288,7 @@ enum class ErrorReporting { none, // no error checking is done, we assume valid inputs }; -#if SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/ppc64/ppc64_validate_utf16.cpp */ template simd8 utf16_gather_high_bytes(const simd16 in0, @@ -42429,10 +38310,10 @@ simd8 utf16_gather_high_bytes(const simd16 in0, } } /* end file src/ppc64/ppc64_validate_utf16.cpp */ -#endif // SIMDUTF_FEATURE_UTF16 +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_LATIN1 && SIMDUTF_FEATURE_UTF8 -/* begin file src/ppc64/ppc64_convert_latin1_to_utf8.cpp */ +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 +/* begin file src/ppc64/ppc64_write_to_utf8.cpp */ /* * reads a vector of uint16 values * bits after 11th are ignored @@ -42505,7 +38386,11 @@ inline void write_v_u16_11bits_to_utf8(const vector_u16 v_u16, write_v_u16_11bits_to_utf8(v_u16, utf8_output, as_vector_u8(one_byte_bytemask), one_byte_bitmask); } +/* end file src/ppc64/ppc64_write_to_utf8.cpp */ +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_UTF32 +#if SIMDUTF_FEATURE_LATIN1 && SIMDUTF_FEATURE_UTF8 +/* begin file src/ppc64/ppc64_convert_latin1_to_utf8.cpp */ std::pair ppc64_convert_latin1_to_utf8(const char *latin_input, const size_t latin_input_length, @@ -42738,7 +38623,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, const auto hi = in16.shr<2>(); auto composed = select(uint16_t(0x1f00 >> 2), hi, lo); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = composed.swap_bytes(); } @@ -42767,7 +38652,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, const auto composed = b2; auto packed = vector_u32::pack(composed, composed); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { packed = packed.swap_bytes(); } @@ -42801,7 +38686,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, auto composed = b0 | b1.shr<2>(); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = composed.swap_bytes(); } @@ -42824,7 +38709,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, auto packed = vector_u32::pack(composed, composed); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { packed = packed.swap_bytes(); } @@ -42897,12 +38782,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, utf16_output += 2; } else { const auto chr = uint16_t(basic_buffer[i]); - if (match_system(big_endian)) { - utf16_output[0] = chr; - } else { - utf16_output[0] = scalar::u16_swap_bytes(chr); - } - + utf16_output[0] = scalar::utf16::swap_if_needed(chr); utf16_output++; } } @@ -43090,7 +38970,7 @@ utf16_to_latin1_t ppc64_convert_utf16_to_latin1(const char16_t *buf, size_t len, // Move low bytes of UTF-16 chars to lower half of `in` // and upper bytes to upper half of `in`. - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { const auto perm = vector_u8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); in = perm.lookup_16(in); @@ -43144,7 +39024,7 @@ utf16_to_latin1_t ppc64_convert_utf16_to_latin1(const char16_t *buf, size_t len, /* end file src/ppc64/ppc64_convert_utf16_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 +#if (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32) && SIMDUTF_FEATURE_UTF8 /* begin file src/ppc64/ppc64_convert_utf16_to_utf8.cpp */ /* The vectorized algorithm works on single SSE register i.e., it @@ -43408,9 +39288,7 @@ utf16_to_utf8_t ppc64_convert_utf16_to_utf8(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = not match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k]) - : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = uint8_t(word); } else if ((word & 0xF800) == 0) { @@ -43423,9 +39301,8 @@ utf16_to_utf8_t ppc64_convert_utf16_to_utf8(const char16_t *buf, size_t len, } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = not match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -43446,7 +39323,8 @@ utf16_to_utf8_t ppc64_convert_utf16_to_utf8(const char16_t *buf, size_t len, return utf16_to_utf8_t{error_code::SUCCESS, buf, utf8_output}; } /* end file src/ppc64/ppc64_convert_utf16_to_utf8.cpp */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 +#endif // (SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32) && + // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 /* begin file src/ppc64/ppc64_convert_utf16_to_utf32.cpp */ @@ -43513,17 +39391,14 @@ utf16_to_utf32_t ppc64_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - const uint16_t word = not match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k]) - : buf[k]; + const uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = not match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -43692,9 +39567,8 @@ utf32_to_utf16_t ppc64_convert_utf32_to_utf16(const char32_t *buf, size_t len, return utf32_to_utf16_t{error_code::SURROGATE, buf + k, utf16_output}; } - *utf16_output++ = not match_system(big_endian) - ? scalar::u16_swap_bytes(uint16_t(word)) - : uint16_t(word); + *utf16_output++ = + scalar::utf16::swap_if_needed(uint16_t(word)); } else { // will generate a surrogate pair if (word > 0x10FFFF) { @@ -43704,10 +39578,10 @@ utf32_to_utf16_t ppc64_convert_utf32_to_utf16(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (not match_system(big_endian)) { - high_surrogate = scalar::u16_swap_bytes(high_surrogate); - low_surrogate = scalar::u16_swap_bytes(low_surrogate); - } + high_surrogate = + scalar::utf16::swap_if_needed(high_surrogate); + low_surrogate = + scalar::utf16::swap_if_needed(low_surrogate); *utf16_output++ = char16_t(high_surrogate); *utf16_output++ = char16_t(low_surrogate); } @@ -43728,7 +39602,7 @@ utf32_to_utf16_t ppc64_convert_utf32_to_utf16(const char32_t *buf, size_t len, /* end file src/ppc64/ppc64_convert_utf32_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_UTF16 -#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_UTF32 +#if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_UTF8 /* begin file src/ppc64/ppc64_convert_utf32_to_utf8.cpp */ struct utf32_to_utf8_t { error_code err; @@ -43897,7 +39771,7 @@ utf32_to_utf8_t ppc64_convert_utf32_to_utf8(const char32_t *buf, size_t len, }; } /* end file src/ppc64/ppc64_convert_utf32_to_utf8.cpp */ -#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 /* begin file src/ppc64/ppc64_utf8_length_from_latin1.cpp */ @@ -43971,9 +39845,9 @@ std::pair ppc64_utf8_length_from_latin1(const char *input, * https://www.codeproject.com/Articles/276993/Base-Encoding-on-a-GPU. (2013). * * AMD XOP specific: http://0x80.pl/notesen/2016-01-12-sse-base64-encoding.html - * Altivec has capabilites of AMD XOP (or vice versa): shuffle using 2 vectors + * Altivec has capabilities of AMD XOP (or vice versa): shuffle using 2 vectors * and variable shifts, thus this implementation shares some code solution - * (modulo intrisic function names). + * (modulo intrinsic function names). */ constexpr bool with_base64_std = false; @@ -44430,7 +40304,8 @@ class block64 { } // namespace ppc64 } // namespace simdutf -#if SIMDUTF_FEATURE_UTF8 +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_ASCII || \ + SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/buf_block_reader.h */ namespace simdutf { namespace ppc64 { @@ -44464,40 +40339,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -44542,6 +40383,9 @@ simdutf_really_inline void buf_block_reader::advance() { } // namespace ppc64 } // namespace simdutf /* end file src/generic/buf_block_reader.h */ +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_ASCII || + // SIMDUTF_FEATURE_DETECT_ENCODING +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ namespace simdutf { namespace ppc64 { @@ -44738,10 +40582,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -44853,7 +40697,7 @@ result generic_validate_utf8_with_errors(const char *input, size_t length) { } // namespace ppc64 } // namespace simdutf /* end file src/generic/utf8_validation/utf8_validator.h */ -#endif // SIMDUTF_FEATURE_UTF8 +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 /* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ @@ -45017,10 +40861,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -45105,10 +40949,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -45266,6 +41110,44 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace ppc64 } // namespace simdutf /* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace ppc64 { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace ppc64 +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -45429,10 +41311,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -45516,10 +41398,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -45635,7 +41517,7 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, /* end file src/generic/utf8_to_utf32/valid_utf8_to_utf32.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -#if SIMDUTF_FEATURE_UTF8 +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf8.h */ namespace simdutf { namespace ppc64 { @@ -45656,8 +41538,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -45730,9 +41612,9 @@ simdutf_really_inline size_t utf16_length_from_utf8(const char *in, } // namespace ppc64 } // namespace simdutf /* end file src/generic/utf8.h */ -#endif // SIMDUTF_FEATURE_UTF8 +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf16.h */ namespace simdutf { namespace ppc64 { @@ -45746,7 +41628,7 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t count = 0; for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); @@ -45764,7 +41646,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t *in, // This algorithm could no doubt be improved! for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t ascii_mask = input.lteq(0x7F); @@ -45946,7 +41828,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, for (; pos < size / 32 * 32; pos += 32) { simd16x32 input_vec( reinterpret_cast(input + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_vec.swap_bytes(); } uint64_t matches = input_vec.lteq(uint16_t(0x7f)); @@ -45959,7 +41841,8 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, // Scalar tail while (pos < size) { - char16_t v = big_endian ? scalar::u16_swap_bytes(input[pos]) : input[pos]; + + char16_t v = scalar::utf16::swap_if_needed(input[pos]); if (v > 0x7F) { return result(error_code::TOO_LARGE, pos); } @@ -45973,9 +41856,9 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, } // namespace ppc64 } // namespace simdutf /* end file src/generic/validate_utf16.h */ -#endif // SIMDUTF_FEATURE_UTF16 +#endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_UTF32 +#if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf32.h */ #include @@ -46141,7 +42024,7 @@ simdutf_really_inline bool validate(const char32_t *input, size_t size) { while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -46184,7 +42067,7 @@ simdutf_really_inline result validate_with_errors(const char32_t *input, while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -46215,7 +42098,7 @@ simdutf_really_inline result validate_with_errors(const char32_t *input, } // namespace ppc64 } // namespace simdutf /* end file src/generic/validate_utf32.h */ -#endif // SIMDUTF_FEATURE_UTF32 +#endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_ASCII /* begin file src/generic/ascii_validation.h */ @@ -46224,22 +42107,6 @@ namespace ppc64 { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -46266,6 +42133,21 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace } // namespace ppc64 @@ -46421,10 +42303,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -46510,10 +42392,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -46732,7 +42614,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -46899,7 +42781,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -46925,8 +42808,8 @@ simdutf_really_inline const char *find(const char *start, const char *end, uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0) { size_t adjustment = 64 - misalignment; - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -46937,7 +42820,7 @@ simdutf_really_inline const char *find(const char *start, const char *end, } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 64; start += 64) { + for (; size_t(end - start) >= 64; start += 64) { simd8x64 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint8_t(character)); if (matches != 0) { @@ -46946,7 +42829,13 @@ simdutf_really_inline const char *find(const char *start, const char *end, return start + index; } } - return std::find(start, end, character); + // Handle remaining bytes with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } simdutf_really_inline const char16_t * @@ -46958,8 +42847,8 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0 && misalignment % 2 == 0) { size_t adjustment = (64 - misalignment) / sizeof(char16_t); - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -46970,7 +42859,7 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 32; start += 32) { + for (; size_t(end - start) >= 32; start += 32) { simd16x32 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint16_t(character)); if (matches != 0) { @@ -46979,7 +42868,13 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { return start + index; } } - return std::find(start, end, character); + // Handle remaining elements with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } } // namespace util @@ -47381,17 +43276,21 @@ simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf32( simdutf_warn_unused size_t implementation::convert_utf16le_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { - return convert_impl(ppc64_convert_utf16_to_latin1, - scalar::utf16_to_latin1::convert, buf, - len, latin1_output); + return convert_impl( + ppc64_convert_utf16_to_latin1, + scalar::utf16_to_latin1::convert, + buf, len, latin1_output); } simdutf_warn_unused size_t implementation::convert_utf16be_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { - return convert_impl(ppc64_convert_utf16_to_latin1, - scalar::utf16_to_latin1::convert, buf, - len, latin1_output); + return convert_impl( + ppc64_convert_utf16_to_latin1, + scalar::utf16_to_latin1::convert, + buf, len, latin1_output); } simdutf_warn_unused result @@ -47400,8 +43299,9 @@ implementation::convert_utf16le_to_latin1_with_errors( return convert_with_errors_impl( ppc64_convert_utf16_to_latin1, - scalar::utf16_to_latin1::convert_with_errors, buf, - len, latin1_output); + scalar::utf16_to_latin1::convert_with_errors, + buf, len, latin1_output); } simdutf_warn_unused result @@ -47410,8 +43310,9 @@ implementation::convert_utf16be_to_latin1_with_errors( return convert_with_errors_impl( ppc64_convert_utf16_to_latin1, - scalar::utf16_to_latin1::convert_with_errors, buf, len, - latin1_output); + scalar::utf16_to_latin1::convert_with_errors, + buf, len, latin1_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16be_to_latin1( @@ -47432,34 +43333,47 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { return convert_impl(ppc64_convert_utf16_to_utf8, - scalar::utf16_to_utf8::convert, buf, - len, utf8_output); + scalar::utf16_to_utf8::convert, + buf, len, utf8_output); } simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - return convert_impl(ppc64_convert_utf16_to_utf8, - scalar::utf16_to_utf8::convert, buf, len, - utf8_output); + return convert_impl( + ppc64_convert_utf16_to_utf8, + scalar::utf16_to_utf8::convert, + buf, len, utf8_output); +} + +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { + const auto vr = + ppc64_convert_utf16_to_utf8(buf, len, utf8_output); + const size_t consumed = size_t(vr.input - buf); + const size_t written = size_t(vr.output - utf8_output); + if (vr.err != error_code::SUCCESS) { + return full_result(vr.err, consumed, written); + } + full_result sr = + scalar::utf16_to_utf8::convert_with_errors( + vr.input, len - consumed, vr.output, 0); + return full_result(sr.error, consumed + sr.input_count, + written + sr.output_count); } simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - - return convert_with_errors_impl( - ppc64_convert_utf16_to_utf8, - scalar::utf16_to_utf8::simple_convert_with_errors, - buf, len, utf8_output); + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - - return convert_with_errors_impl( - ppc64_convert_utf16_to_utf8, - scalar::utf16_to_utf8::simple_convert_with_errors, buf, - len, utf8_output); + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -47500,20 +43414,23 @@ simdutf_warn_unused size_t implementation::convert_valid_utf32_to_latin1( simdutf_warn_unused size_t implementation::convert_utf32_to_utf8( const char32_t *buf, size_t len, char *utf8_output) const noexcept { return convert_impl(ppc64_convert_utf32_to_utf8, - scalar::utf32_to_utf8::convert, buf, len, utf8_output); + scalar::utf32_to_utf8::convert, + buf, len, utf8_output); } simdutf_warn_unused result implementation::convert_utf32_to_utf8_with_errors( const char32_t *buf, size_t len, char *utf8_output) const noexcept { return convert_with_errors_impl( ppc64_convert_utf32_to_utf8, - scalar::utf32_to_utf8::convert_with_errors, buf, len, utf8_output); + scalar::utf32_to_utf8::convert_with_errors, buf, + len, utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf32_to_utf8( const char32_t *buf, size_t len, char *utf8_output) const noexcept { return convert_impl(ppc64_convert_utf32_to_utf8, - scalar::utf32_to_utf8::convert, buf, len, utf8_output); + scalar::utf32_to_utf8::convert, + buf, len, utf8_output); } #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -47702,6 +43619,26 @@ implementation::utf8_length_from_utf16be_with_replacement( endianness::BIG>(input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -47927,6 +43864,49 @@ namespace { } // namespace rvv } // namespace simdutf +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + // transcoding from UTF-16 to UTF-8 (self-wrapping generic header, must be + // included at namespace scope zero) +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace rvv { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace rvv +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + // // Implementation-specific overrides // @@ -48548,8 +44528,8 @@ simdutf_warn_unused size_t implementation::convert_valid_utf16be_to_latin1( #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 template -simdutf_really_inline static result -rvv_utf16_to_utf8_with_errors(const char16_t *src, size_t len, char *dst) { +simdutf_really_inline static full_result +rvv_utf16_to_utf8_with_details(const char16_t *src, size_t len, char *dst) { size_t n = len; const char16_t *srcBeg = src; const char *dstBeg = dst; @@ -48665,13 +44645,16 @@ rvv_utf16_to_utf8_with_errors(const char16_t *src, size_t len, char *dst) { } else { // must be a surrogate pair if (n <= 1) - return result(error_code::SURROGATE, src - srcBeg); + return full_result(error_code::SURROGATE, size_t(src - srcBeg), + size_t(dst - dstBeg)); uint16_t diff = word - 0xD800; if (diff > 0x3FF) - return result(error_code::SURROGATE, src - srcBeg); + return full_result(error_code::SURROGATE, size_t(src - srcBeg), + size_t(dst - dstBeg)); uint16_t diff2 = simdutf_byteflip(src[1]) - 0xDC00; if (diff2 > 0x3FF) - return result(error_code::SURROGATE, src - srcBeg); + return full_result(error_code::SURROGATE, size_t(src - srcBeg), + size_t(dst - dstBeg)); uint32_t value = ((diff + 0x40) << 10) + diff2; @@ -48687,7 +44670,8 @@ rvv_utf16_to_utf8_with_errors(const char16_t *src, size_t len, char *dst) { } } - return result(error_code::SUCCESS, dst - dstBeg); + return full_result(error_code::SUCCESS, size_t(src - srcBeg), + size_t(dst - dstBeg)); } simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( @@ -48704,15 +44688,16 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( const char16_t *src, size_t len, char *dst) const noexcept { - return rvv_utf16_to_utf8_with_errors(src, len, dst); + return rvv_utf16_to_utf8_with_details(src, len, dst); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *src, size_t len, char *dst) const noexcept { if (supports_zvbb()) - return rvv_utf16_to_utf8_with_errors(src, len, dst); + return rvv_utf16_to_utf8_with_details(src, len, + dst); else - return rvv_utf16_to_utf8_with_errors(src, len, dst); + return rvv_utf16_to_utf8_with_details(src, len, dst); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -49625,6 +45610,122 @@ simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf32( /* end file src/rvv/rvv_utf8_to.inl.cpp */ #if SIMDUTF_FEATURE_BASE64 +/* begin file src/rvv/rvv_base64.cpp */ +template +size_t encode_base64_rvv(char *dst, const char *src, size_t srclen, + base64_options options, + size_t line_length = simdutf::default_line_length) { + size_t offset = 0; + if constexpr (insert_line_feeds) { + if (line_length < 4) { + line_length = 4; + } + } + + static constexpr uint8_t table_standard[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', + }; + static constexpr uint8_t table_url[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', + }; + + const uint8_t *table = (options & base64_url) ? table_url : table_standard; + uint8_t *out = (uint8_t *)dst; + + size_t triplets = srclen / 3; + size_t i = 0; + + while (triplets > 0) { + size_t max_vl = triplets; + if constexpr (insert_line_feeds) { + size_t max_per_line = line_length / 4; + if (max_per_line < max_vl) + max_vl = max_per_line; + } + size_t vl = __riscv_vsetvl_e8m2(max_vl); + if (vl == 0) + break; + + // 3-way deinterleaved load: a[k]=src[3k], b[k]=src[3k+1], c[k]=src[3k+2] + vuint8m2_t a = __riscv_vlse8_v_u8m2((const uint8_t *)src + i, 3, vl); + vuint8m2_t b = __riscv_vlse8_v_u8m2((const uint8_t *)src + i + 1, 3, vl); + vuint8m2_t c = __riscv_vlse8_v_u8m2((const uint8_t *)src + i + 2, 3, vl); + + // Extract 6-bit indices from each triplet + vuint8m2_t idx0 = __riscv_vsrl_vx_u8m2(a, 2, vl); + vuint8m2_t idx1 = __riscv_vand_vx_u8m2( + __riscv_vor_vv_u8m2(__riscv_vsll_vx_u8m2(a, 4, vl), + __riscv_vsrl_vx_u8m2(b, 4, vl), vl), + 0x3F, vl); + vuint8m2_t idx2 = __riscv_vand_vx_u8m2( + __riscv_vor_vv_u8m2(__riscv_vsll_vx_u8m2(b, 2, vl), + __riscv_vsrl_vx_u8m2(c, 6, vl), vl), + 0x3F, vl); + vuint8m2_t idx3 = __riscv_vand_vx_u8m2(c, 0x3F, vl); + + // Table lookup: map 6-bit indices to base64 characters + vuint8m2_t out0 = __riscv_vluxei8_v_u8m2(table, idx0, vl); + vuint8m2_t out1 = __riscv_vluxei8_v_u8m2(table, idx1, vl); + vuint8m2_t out2 = __riscv_vluxei8_v_u8m2(table, idx2, vl); + vuint8m2_t out3 = __riscv_vluxei8_v_u8m2(table, idx3, vl); + + size_t chunk = vl * 4; + + if constexpr (insert_line_feeds) { + if (offset >= line_length) { + *out++ = '\n'; + offset = 0; + } + + // 4-way interleaved store + __riscv_vsse8_v_u8m2(out, 4, out0, vl); + __riscv_vsse8_v_u8m2(out + 1, 4, out1, vl); + __riscv_vsse8_v_u8m2(out + 2, 4, out2, vl); + __riscv_vsse8_v_u8m2(out + 3, 4, out3, vl); + + if (offset + chunk <= line_length) { + out += chunk; + offset += chunk; + } else { + size_t before = line_length - offset; + size_t after = chunk - before; + std::memmove(out + before + 1, out + before, after); + out[before] = '\n'; + offset = after; + out += chunk + 1; + } + } else { + // 4-way interleaved store + __riscv_vsse8_v_u8m2(out, 4, out0, vl); + __riscv_vsse8_v_u8m2(out + 1, 4, out1, vl); + __riscv_vsse8_v_u8m2(out + 2, 4, out2, vl); + __riscv_vsse8_v_u8m2(out + 3, 4, out3, vl); + out += chunk; + } + + triplets -= vl; + i += vl * 3; + } + + out += scalar::base64::tail_encode_base64_impl( + (char *)out, src + i, srclen - i, options, line_length, offset); + + return (size_t)((char *)out - dst); +} + +size_t encode_base64(char *dst, const char *src, size_t srclen, + base64_options options) { + return encode_base64_rvv(dst, src, srclen, options); +} +/* end file src/rvv/rvv_base64.cpp */ /* begin file src/rvv/rvv_find.cpp */ const char *implementation::find(const char *start, const char *end, char character) const noexcept { @@ -49662,20 +45763,16 @@ template simdutf_really_inline void utf16fix_block_rvv(char16_t *out, const char16_t *in, size_t vl) { const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; - }; - vuint16m8_t block = __riscv_vle16_v_u16m8((const uint16_t *)in, vl); vuint16m8_t lookback = __riscv_vslide1up_vx_u16m8(block, in[-1], vl); - vuint16m8_t lb_masked = - __riscv_vand_vx_u16m8(lookback, swap_if_needed(0xfc00U), vl); - vuint16m8_t block_masked = - __riscv_vand_vx_u16m8(block, swap_if_needed(0xfc00U), vl); - vbool2_t lb_is_high = - __riscv_vmseq_vx_u16m8_b2(lb_masked, swap_if_needed(0xd800U), vl); - vbool2_t block_is_low = - __riscv_vmseq_vx_u16m8_b2(block_masked, swap_if_needed(0xdc00U), vl); + vuint16m8_t lb_masked = __riscv_vand_vx_u16m8( + lookback, scalar::utf16::swap_if_needed(0xfc00U), vl); + vuint16m8_t block_masked = __riscv_vand_vx_u16m8( + block, scalar::utf16::swap_if_needed(0xfc00U), vl); + vbool2_t lb_is_high = __riscv_vmseq_vx_u16m8_b2( + lb_masked, scalar::utf16::swap_if_needed(0xd800U), vl); + vbool2_t block_is_low = __riscv_vmseq_vx_u16m8_b2( + block_masked, scalar::utf16::swap_if_needed(0xdc00U), vl); vbool2_t illseq = __riscv_vmxor_mm_b2(lb_is_high, block_is_low, vl); if (__riscv_vfirst_m_b2(illseq, vl) >= 0) { @@ -49695,10 +45792,10 @@ simdutf_really_inline void utf16fix_block_rvv(char16_t *out, const char16_t *in, lb_illseq_right_shifted = __riscv_vmandn_mm_b2( __riscv_vmseq_vx_u16m8_b2( __riscv_vslide1down_vx_u16m8(lb_masked, 0, vl), - swap_if_needed(0xd800U), vl), + scalar::utf16::swap_if_needed(0xd800U), vl), __riscv_vmseq_vx_u16m8_b2( __riscv_vslide1down_vx_u16m8(block_masked, 0, vl), - swap_if_needed(0xdc00U), vl), + scalar::utf16::swap_if_needed(0xdc00U), vl), vl); } @@ -49832,14 +45929,13 @@ simdutf_warn_unused full_result implementation::base64_to_binary_details( size_t implementation::binary_to_base64(const char *input, size_t length, char *output, base64_options options) const noexcept { - return scalar::base64::tail_encode_base64(output, input, length, options); + return encode_base64(output, input, length, options); } size_t implementation::binary_to_base64_with_lines( const char *input, size_t length, char *output, size_t line_length, base64_options options) const noexcept { - return scalar::base64::tail_encode_base64_impl(output, input, length, - options, line_length); + return encode_base64_rvv(output, input, length, options, line_length); } #endif // SIMDUTF_FEATURE_BASE64 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -49857,6 +45953,30 @@ implementation::utf8_length_from_utf16be_with_replacement( endianness::BIG>(input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return rvv_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [this](const char16_t *b, size_t l, char *o) { + return supports_zvbb() + ? rvv_utf16_to_utf8_with_details( + b, l, o) + : rvv_utf16_to_utf8_with_details(b, l, + o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 } // namespace rvv @@ -50006,10 +46126,10 @@ inline void write_v_u16_11bits_to_utf8(const __m128i v_u16, char *&utf8_output, template simdutf_really_inline void utf16fix_block_sse(char16_t *out, const char16_t *in) { - const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); }; + const char16_t replacement = scalar::utf16::replacement(); __m128i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low; __m128i illseq, lb_illseq, block_illseq; @@ -50661,7 +46781,7 @@ sse_convert_utf16_to_latin1(const char16_t *buf, size_t len, // Load 8 UTF-16 characters into 128-bit SSE register __m128i in = _mm_loadu_si128(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { const __m128i swap = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); in = _mm_shuffle_epi8(in, swap); @@ -50692,7 +46812,7 @@ sse_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, while (end - buf >= 8) { __m128i in = _mm_loadu_si128(reinterpret_cast(buf)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { const __m128i swap = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); in = _mm_shuffle_epi8(in, swap); @@ -50708,8 +46828,7 @@ sse_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } else { // Fallback to scalar code for handling errors for (int k = 0; k < 8; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if (word <= 0xff) { *latin1_output++ = char(word); } else { @@ -50968,7 +47087,7 @@ sse_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -50982,7 +47101,7 @@ sse_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -51197,7 +47316,7 @@ sse_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -51211,7 +47330,7 @@ sse_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -51339,14 +47458,14 @@ sse_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -51419,14 +47538,14 @@ sse_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = big_endian ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); uint16_t next_word = - big_endian ? scalar::u16_swap_bytes(buf[k + 1]) : buf[k + 1]; + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -51670,8 +47789,7 @@ sse_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output) { // t0 = [000a|aaaa|bbbb|bb00] const __m128i t0 = _mm_slli_epi16(in_16, 2); // shift packed vector by two // t1 = [000a|aaaa|0000|0000] - const __m128i t1 = - _mm_and_si128(t0, v_1f00); // potentital first utf8 byte + const __m128i t1 = _mm_and_si128(t0, v_1f00); // potential first utf8 byte // t2 = [0000|0000|00bb|bbbb] const __m128i t2 = _mm_and_si128(in_16, v_003f); // potential second utf8 byte @@ -52406,34 +48524,29 @@ template __m128i lookup_pshufb_improved(const __m128i input) { inline __m128i insert_line_feed16(__m128i input, size_t K) { static const uint8_t shuffle_masks[16][16] = { - {0x80, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 0x80, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 0x80, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 0x80, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 0x80, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 0x80, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 0x80, 6, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 0x80, 7, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 0x80, 8, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 0x80, 9, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0x80, 10, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0x80, 11, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0x80, 12, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0x80, 13, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x80, 14}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0x80}}; - // Prepare a vector with '\n' (0x0A) - __m128i line_feed_vector = _mm_set1_epi8('\n'); + {15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 15, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 15, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 15, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 15, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 15, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 15, 7, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 15, 8, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 15, 9, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 10, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 11, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15, 12, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 13, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 14}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}; + input = _mm_insert_epi8(input, '\n', 15); // Load the precomputed shuffle mask for K (index K-1) __m128i mask = _mm_loadu_si128((__m128i *)shuffle_masks[K]); - __m128i lf_pos = _mm_cmpeq_epi8(mask, _mm_set1_epi8(static_cast(0x80))); // Perform the shuffle to reposition the K bytes - __m128i shuffled = _mm_shuffle_epi8(input, mask); - - // Blend with line_feed_vector to insert '\n' at the appropriate positions - __m128i result = _mm_blendv_epi8(shuffled, line_feed_vector, lf_pos); + __m128i result = _mm_shuffle_epi8(input, mask); return result; } @@ -53056,40 +49169,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -53331,10 +49410,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -53454,22 +49533,6 @@ namespace westmere { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -53496,6 +49559,21 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace } // namespace westmere @@ -53742,10 +49820,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -53830,10 +49908,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -53970,6 +50048,45 @@ simdutf_really_inline size_t utf16_length_from_utf8_bytemask(const char *in, } // namespace westmere } // namespace simdutf /* end file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ + // transcoding from UTF-16 to UTF-8 +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace westmere { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -54177,10 +50294,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -54264,10 +50381,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -54498,8 +50615,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -54587,7 +50704,7 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t count = 0; for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); @@ -54605,7 +50722,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t *in, // This algorithm could no doubt be improved! for (; pos < size / 32 * 32; pos += 32) { simd16x32 input(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input.swap_bytes(); } uint64_t ascii_mask = input.lteq(0x7F); @@ -54679,7 +50796,7 @@ simdutf_really_inline size_t utf8_length_from_utf16_bytemask(const char16_t *in, for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -54768,7 +50885,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { for (; pos < (size - 1) / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -54789,7 +50906,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { any_surrogates = true; auto input_next = vector_u16::load(reinterpret_cast(in + pos + 1)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_next = input_next.swap_bytes(); } @@ -54990,7 +51107,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, for (; pos < size / 32 * 32; pos += 32) { simd16x32 input_vec( reinterpret_cast(input + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_vec.swap_bytes(); } uint64_t matches = input_vec.lteq(uint16_t(0x7f)); @@ -55003,7 +51120,8 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, // Scalar tail while (pos < size) { - char16_t v = big_endian ? scalar::u16_swap_bytes(input[pos]) : input[pos]; + + char16_t v = scalar::utf16::swap_if_needed(input[pos]); if (v > 0x7F) { return result(error_code::TOO_LARGE, pos); } @@ -55166,10 +51284,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -55255,10 +51373,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -55447,7 +51565,7 @@ simdutf_really_inline bool validate(const char32_t *input, size_t size) { while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -55490,7 +51608,7 @@ simdutf_really_inline result validate_with_errors(const char32_t *input, while (input + N < end) { auto in = vector_u32(input); - if (!match_system(endianness::BIG)) { + if constexpr (!match_system(endianness::BIG)) { in.swap_bytes(); } @@ -55581,7 +51699,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, size_t full_input_length = ri.full_input_length; if (srclen == 0) { if (!ignore_garbage && equalsigns > 0) { - return {INVALID_BASE64_CHARACTER, equallocation, 0}; + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; } return {SUCCESS, full_input_length, 0}; } @@ -55748,7 +51866,8 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, if (!ignore_garbage && equalsigns > 0) { if ((size_t(dst - dstinit) % 3 == 0) || ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { - return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit)}; + return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), + true}; } } return {SUCCESS, srclen, size_t(dst - dstinit)}; @@ -55774,8 +51893,8 @@ simdutf_really_inline const char *find(const char *start, const char *end, uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0) { size_t adjustment = 64 - misalignment; - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -55786,7 +51905,7 @@ simdutf_really_inline const char *find(const char *start, const char *end, } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 64; start += 64) { + for (; size_t(end - start) >= 64; start += 64) { simd8x64 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint8_t(character)); if (matches != 0) { @@ -55795,7 +51914,13 @@ simdutf_really_inline const char *find(const char *start, const char *end, return start + index; } } - return std::find(start, end, character); + // Handle remaining bytes with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } simdutf_really_inline const char16_t * @@ -55807,8 +51932,8 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { uintptr_t misalignment = reinterpret_cast(start) % 64; if (misalignment != 0 && misalignment % 2 == 0) { size_t adjustment = (64 - misalignment) / sizeof(char16_t); - if (size_t(std::distance(start, end)) < adjustment) { - adjustment = std::distance(start, end); + if (size_t(end - start) < adjustment) { + adjustment = end - start; } for (size_t i = 0; i < adjustment; i++) { if (start[i] == character) { @@ -55819,7 +51944,7 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { } // Main loop for 64-byte aligned data - for (; std::distance(start, end) >= 32; start += 32) { + for (; size_t(end - start) >= 32; start += 32) { simd16x32 input(reinterpret_cast(start)); uint64_t matches = input.eq(uint16_t(character)); if (matches != 0) { @@ -55828,7 +51953,13 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { return start + index; } } - return std::find(start, end, character); + // Handle remaining elements with scalar loop + for (; start < end; ++start) { + if (*start == character) { + return start; + } + } + return end; } } // namespace util @@ -55836,6 +51967,74 @@ find(const char16_t *start, const char16_t *end, char16_t character) noexcept { } // namespace westmere } // namespace simdutf /* end file src/generic/find.h */ +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace westmere { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + // simd16x32::to_bitmask sets two bits per matching 16-bit lane, so the + // vectorized loop counted each unit twice. + count /= 2; + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace westmere +} // namespace simdutf +/* end file src/generic/base64lengths.h */ #endif // SIMDUTF_FEATURE_BASE64 // @@ -56439,60 +52638,37 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return saved_bytes; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { std::pair ret = - westmere::sse_convert_utf16_to_utf8_with_errors( - buf, len, utf8_output); + westmere::sse_convert_utf16_to_utf8_with_errors(buf, len, + utf8_output); if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } + return full_result(ret.first.error, ret.first.count, + size_t(ret.second - utf8_output)); } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + if (ret.first.count != len) { + full_result sres = + scalar::utf16_to_utf8::convert_with_errors( + buf + ret.first.count, len - ret.first.count, ret.second, 0); + return full_result(sres.error, ret.first.count + sres.input_count, + size_t(ret.second - utf8_output) + sres.output_count); + } + return full_result(error_code::SUCCESS, len, + size_t(ret.second - utf8_output)); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished - std::pair ret = - westmere::sse_convert_utf16_to_utf8_with_errors( - buf, len, utf8_output); - if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } - } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -56953,6 +53129,26 @@ implementation::utf8_length_from_utf16be_with_replacement( input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -57143,6 +53339,16 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util::find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 } // namespace westmere @@ -57159,25 +53365,31 @@ SIMDUTF_UNTARGET_REGION /* end file src/simdutf/westmere/end.h */ /* end file src/westmere/implementation.cpp */ #endif -#if SIMDUTF_IMPLEMENTATION_LSX -/* begin file src/lsx/implementation.cpp */ -/* begin file src/simdutf/lsx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lsx" -// #define SIMDUTF_IMPLEMENTATION lsx +#if SIMDUTF_IMPLEMENTATION_LASX +/* begin file src/lasx/implementation.cpp */ +/* begin file src/simdutf/lasx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lasx" +// #define SIMDUTF_IMPLEMENTATION lasx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lsx/begin.h */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_TARGET_LASX +#endif +/* end file src/simdutf/lasx/begin.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { -#ifndef SIMDUTF_LSX_H - #error "lsx.h must be included" +#ifndef SIMDUTF_LASX_H + #error "lasx.h must be included" #endif using namespace simd; #if SIMDUTF_FEATURE_UTF8 // convert vmskltz/vmskgez/vmsknz to // simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes index -const uint8_t lsx_1_2_utf8_bytes_mask[] = { +const uint8_t lasx_1_2_utf8_bytes_mask[] = { 0, 1, 4, 5, 16, 17, 20, 21, 64, 65, 68, 69, 80, 81, 84, 85, 2, 3, 6, 7, 18, 19, 22, 23, 66, 67, 70, 71, 82, 83, 86, 87, 8, 9, 12, 13, 24, 25, 28, 29, 72, 73, 76, 77, 88, @@ -57202,6 +53414,9 @@ const uint8_t lsx_1_2_utf8_bytes_mask[] = { simdutf_really_inline __m128i lsx_swap_bytes(__m128i vec) { return __lsx_vshuf4i_b(vec, 0b10110001); } +simdutf_really_inline __m256i lasx_swap_bytes(__m256i vec) { + return __lasx_xvshuf4i_b(vec, 0b10110001); +} #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_ASCII || SIMDUTF_FEATURE_DETECT_ENCODING || \ @@ -57272,7 +53487,7 @@ convert_utf8_1_to_2_byte_to_utf16(__m128i in, size_t shufutf8_idx) { __m128i ascii = __lsx_vand_v(perm, __lsx_vrepli_h(0x7f)); // 6 or 7 bits // 1 byte: 00000000 00000000 // 2 byte: 00000aaa aa000000 - const __m128i v1f00 = lsx_splat_u16(0x1f00); + __m128i v1f00 = lsx_splat_u16(0x1f00); __m128i composed = __lsx_vsrli_h(__lsx_vand_v(perm, v1f00), 2); // 5 bits // Combine with a shift right accumulate // 1 byte: 00000000 0bbbbbbb @@ -57284,7 +53499,7 @@ convert_utf8_1_to_2_byte_to_utf16(__m128i in, size_t shufutf8_idx) { // SIMDUTF_FEATURE_UTF32) #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/lsx/lsx_validate_utf16.cpp */ +/* begin file src/lasx/lasx_validate_utf16.cpp */ template simd8 utf16_gather_high_bytes(const simd16 in0, const simd16 in1) { @@ -57298,101 +53513,117 @@ simd8 utf16_gather_high_bytes(const simd16 in0, return simd16::pack_shifted_right<8>(in0, in1); } } -/* end file src/lsx/lsx_validate_utf16.cpp */ +/* end file src/lasx/lasx_validate_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/lsx/lsx_validate_utf32le.cpp */ -const char32_t *lsx_validate_utf32le(const char32_t *input, size_t size) { +/* begin file src/lasx/lasx_validate_utf32le.cpp */ +const char32_t *lasx_validate_utf32le(const char32_t *input, size_t size) { const char32_t *end = input + size; - __m128i offset = lsx_splat_u32(0xffff2000); - __m128i standardoffsetmax = lsx_splat_u32(0xfffff7ff); - __m128i standardmax = lsx_splat_u32(0x10ffff); - __m128i currentmax = lsx_splat_u32(0); - __m128i currentoffsetmax = lsx_splat_u32(0); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)input & 0x1F) && input < end) { + uint32_t word = *input++; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return nullptr; + } + } - while (input + 4 < end) { - __m128i in = __lsx_vld(reinterpret_cast(input), 0); - currentmax = __lsx_vmax_wu(in, currentmax); + __m256i offset = lasx_splat_u32(0xffff2000); + __m256i standardoffsetmax = lasx_splat_u32(0xfffff7ff); + __m256i standardmax = lasx_splat_u32(0x10ffff); + __m256i currentmax = __lasx_xvldi(0x0); + __m256i currentoffsetmax = __lasx_xvldi(0x0); + + while (input + 8 < end) { + __m256i in = __lasx_xvld(reinterpret_cast(input), 0); + currentmax = __lasx_xvmax_wu(in, currentmax); // 0xD8__ + 0x2000 = 0xF8__ => 0xF8__ > 0xF7FF currentoffsetmax = - __lsx_vmax_wu(__lsx_vadd_w(in, offset), currentoffsetmax); - - input += 4; + __lasx_xvmax_wu(__lasx_xvadd_w(in, offset), currentoffsetmax); + input += 8; } - - __m128i is_zero = - __lsx_vxor_v(__lsx_vmax_wu(currentmax, standardmax), standardmax); - if (__lsx_bnz_v(is_zero)) { + __m256i is_zero = + __lasx_xvxor_v(__lasx_xvmax_wu(currentmax, standardmax), standardmax); + if (__lasx_xbnz_v(is_zero)) { return nullptr; } - is_zero = __lsx_vxor_v(__lsx_vmax_wu(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (__lsx_bnz_v(is_zero)) { + is_zero = __lasx_xvxor_v(__lasx_xvmax_wu(currentoffsetmax, standardoffsetmax), + standardoffsetmax); + if (__lasx_xbnz_v(is_zero)) { return nullptr; } - return input; } -const result lsx_validate_utf32le_with_errors(const char32_t *input, - size_t size) { +const result lasx_validate_utf32le_with_errors(const char32_t *input, + size_t size) { const char32_t *start = input; const char32_t *end = input + size; - __m128i offset = lsx_splat_u32(0xffff2000); - __m128i standardoffsetmax = lsx_splat_u32(0xfffff7ff); - __m128i standardmax = lsx_splat_u32(0x10ffff); - __m128i currentmax = lsx_splat_u32(0); - __m128i currentoffsetmax = lsx_splat_u32(0); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)input & 0x1F) && input < end) { + uint32_t word = *input; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, input - start); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, input - start); + } + input++; + } - while (input + 4 < end) { - __m128i in = __lsx_vld(reinterpret_cast(input), 0); - currentmax = __lsx_vmax_wu(in, currentmax); + __m256i offset = lasx_splat_u32(0xffff2000); + __m256i standardoffsetmax = lasx_splat_u32(0xfffff7ff); + __m256i standardmax = lasx_splat_u32(0x10ffff); + __m256i currentmax = __lasx_xvldi(0x0); + __m256i currentoffsetmax = __lasx_xvldi(0x0); + + while (input + 8 < end) { + __m256i in = __lasx_xvld(reinterpret_cast(input), 0); + currentmax = __lasx_xvmax_wu(in, currentmax); currentoffsetmax = - __lsx_vmax_wu(__lsx_vadd_w(in, offset), currentoffsetmax); + __lasx_xvmax_wu(__lasx_xvadd_w(in, offset), currentoffsetmax); - __m128i is_zero = - __lsx_vxor_v(__lsx_vmax_wu(currentmax, standardmax), standardmax); - if (__lsx_bnz_v(is_zero)) { + __m256i is_zero = + __lasx_xvxor_v(__lasx_xvmax_wu(currentmax, standardmax), standardmax); + if (__lasx_xbnz_v(is_zero)) { return result(error_code::TOO_LARGE, input - start); } - - is_zero = __lsx_vxor_v(__lsx_vmax_wu(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (__lsx_bnz_v(is_zero)) { + is_zero = + __lasx_xvxor_v(__lasx_xvmax_wu(currentoffsetmax, standardoffsetmax), + standardoffsetmax); + if (__lasx_xbnz_v(is_zero)) { return result(error_code::SURROGATE, input - start); } - - input += 4; + input += 8; } return result(error_code::SUCCESS, input - start); } -/* end file src/lsx/lsx_validate_utf32le.cpp */ +/* end file src/lasx/lasx_validate_utf32le.cpp */ #endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_latin1_to_utf8.cpp */ +/* begin file src/lasx/lasx_convert_latin1_to_utf8.cpp */ /* Returns a pair: the first unprocessed byte from buf and utf8_output A scalar routing should carry on the conversion of the tail. */ std::pair -lsx_convert_latin1_to_utf8(const char *latin1_input, size_t len, - char *utf8_out) { +lasx_convert_latin1_to_utf8(const char *latin1_input, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); + const size_t safety_margin = 12; const char *end = latin1_input + len; - __m128i zero = __lsx_vldi(0); // We always write 16 bytes, of which more than the first 8 bytes // are valid. A safety margin of 8 is more than sufficient. - while (end - latin1_input >= 16) { + while (end - latin1_input >= std::ptrdiff_t(16 + safety_margin)) { __m128i in8 = __lsx_vld(reinterpret_cast(latin1_input), 0); - uint32_t ascii = __lsx_vpickve2gr_hu(__lsx_vmskgez_b(in8), 0); - if (ascii == 0xffff) { // ASCII fast path!!!! + uint32_t ascii_mask = __lsx_vpickve2gr_wu(__lsx_vmskgez_b(in8), 0); + if (ascii_mask == 0xFFFF) { __lsx_vst(in8, utf8_output, 0); utf8_output += 16; latin1_input += 16; @@ -57400,48 +53631,76 @@ lsx_convert_latin1_to_utf8(const char *latin1_input, size_t len, } // We just fallback on UTF-16 code. This could be optimized/simplified // further. - __m128i in16 = __lsx_vilvl_b(zero, in8); + __m256i in16 = __lasx_vext2xv_hu_bu(____m256i(in8)); // 1. prepare 2-byte values - // input 8-bit word : [aabb|bbbb] x 8 - // expected output : [1100|00aa|10bb|bbbb] x 8 + // input 8-bit word : [aabb|bbbb] x 16 + // expected output : [1100|00aa|10bb|bbbb] x 16 // t0 = [0000|00aa|bbbb|bb00] - __m128i t0 = __lsx_vslli_h(in16, 2); + __m256i t0 = __lasx_xvslli_h(in16, 2); // t1 = [0000|00aa|0000|0000] - __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x300)); + __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x300)); // t3 = [0000|00aa|00bb|bbbb] - __m128i t2 = __lsx_vbitsel_v(t1, in16, __lsx_vrepli_h(0x3f)); + __m256i t2 = __lasx_xvbitsel_v(t1, in16, __lasx_xvrepli_h(0x3f)); // t4 = [1100|00aa|10bb|bbbb] - __m128i t3 = __lsx_vor_v(t2, __lsx_vreplgr2vr_h(uint16_t(0xc080))); + __m256i t3 = __lasx_xvor_v(t2, __lasx_xvreplgr2vr_h(uint16_t(0xc080))); // merge ASCII and 2-byte codewords - __m128i one_byte_bytemask = __lsx_vsle_hu(in16, __lsx_vrepli_h(0x7F)); - __m128i utf8_unpacked = __lsx_vbitsel_v(t3, in16, one_byte_bytemask); + __m256i one_byte_bytemask = __lasx_xvsle_hu(in16, __lasx_xvrepli_h(0x7F)); + __m256i utf8_unpacked = __lasx_xvbitsel_v(t3, in16, one_byte_bytemask); - const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lsx_1_2_utf8_bytes_mask[(ascii & 0xff)]][0]; - __m128i shuffle = __lsx_vld(row + 1, 0); - __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); + const uint8_t *row0 = + &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[(ascii_mask & 0xFF)]][0]; + __m128i shuffle0 = __lsx_vld(row0 + 1, 0); + __m128i utf8_unpacked_lo = lasx_extracti128_lo(utf8_unpacked); + __m128i utf8_packed0 = + __lsx_vshuf_b(utf8_unpacked_lo, utf8_unpacked_lo, shuffle0); + __lsx_vst(utf8_packed0, utf8_output, 0); + utf8_output += row0[0]; - // store bytes - __lsx_vst(utf8_packed, utf8_output, 0); - // adjust pointers - latin1_input += 8; - utf8_output += row[0]; + const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[(ascii_mask >> 8)]][0]; + __m128i shuffle1 = __lsx_vld(row1 + 1, 0); + __m128i utf8_unpacked_hi = lasx_extracti128_hi(utf8_unpacked); + __m128i utf8_packed1 = + __lsx_vshuf_b(utf8_unpacked_hi, utf8_unpacked_hi, shuffle1); + __lsx_vst(utf8_packed1, utf8_output, 0); + utf8_output += row1[0]; + latin1_input += 16; } // while return std::make_pair(latin1_input, reinterpret_cast(utf8_output)); } -/* end file src/lsx/lsx_convert_latin1_to_utf8.cpp */ +/* end file src/lasx/lasx_convert_latin1_to_utf8.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_latin1_to_utf16.cpp */ +/* begin file src/lasx/lasx_convert_latin1_to_utf16.cpp */ std::pair -lsx_convert_latin1_to_utf16le(const char *buf, size_t len, - char16_t *utf16_output) { +lasx_convert_latin1_to_utf16le(const char *buf, size_t len, + char16_t *utf16_output) { const char *end = buf + len; - __m128i zero = __lsx_vldi(0); - while (end - buf >= 16) { + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)utf16_output & 0x1F) && buf < end) { + *utf16_output++ = uint8_t(*buf) & 0xFF; + buf++; + } + + while (end - buf >= 32) { + __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); + + __m256i inlow = __lasx_vext2xv_hu_bu(in8); + __m256i in8_high = __lasx_xvpermi_q(in8, in8, 0b00000001); + __m256i inhigh = __lasx_vext2xv_hu_bu(in8_high); + __lasx_xvst(inlow, reinterpret_cast(utf16_output), 0); + __lasx_xvst(inhigh, reinterpret_cast(utf16_output), 32); + + utf16_output += 32; + buf += 32; + } + + if (end - buf >= 16) { + __m128i zero = __lsx_vldi(0); __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); __m128i inlow = __lsx_vilvl_b(zero, in8); @@ -57452,20 +53711,38 @@ lsx_convert_latin1_to_utf16le(const char *buf, size_t len, utf16_output += 16; buf += 16; } - return std::make_pair(buf, utf16_output); } std::pair -lsx_convert_latin1_to_utf16be(const char *buf, size_t len, - char16_t *utf16_output) { +lasx_convert_latin1_to_utf16be(const char *buf, size_t len, + char16_t *utf16_output) { const char *end = buf + len; - __m128i zero = __lsx_vldi(0); - while (end - buf >= 16) { + + while (((uint64_t)utf16_output & 0x1F) && buf < end) { + *utf16_output++ = char16_t((uint16_t(*buf++) << 8)); + } + + __m256i zero = __lasx_xvldi(0); + while (end - buf >= 32) { + __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); + + __m256i in8_shuf = __lasx_xvpermi_d(in8, 0b11011000); + + __m256i inlow = __lasx_xvilvl_b(in8_shuf, zero); + __m256i inhigh = __lasx_xvilvh_b(in8_shuf, zero); + __lasx_xvst(inlow, reinterpret_cast(utf16_output), 0); + __lasx_xvst(inhigh, reinterpret_cast(utf16_output), 32); + utf16_output += 32; + buf += 32; + } + + if (end - buf >= 16) { + __m128i zero_128 = __lsx_vldi(0); __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i inlow = __lsx_vilvl_b(in8, zero); - __m128i inhigh = __lsx_vilvh_b(in8, zero); + __m128i inlow = __lsx_vilvl_b(in8, zero_128); + __m128i inhigh = __lsx_vilvh_b(in8, zero_128); __lsx_vst(inlow, reinterpret_cast(utf16_output), 0); __lsx_vst(inhigh, reinterpret_cast(utf16_output), 16); utf16_output += 16; @@ -57474,16 +53751,44 @@ lsx_convert_latin1_to_utf16be(const char *buf, size_t len, return std::make_pair(buf, utf16_output); } -/* end file src/lsx/lsx_convert_latin1_to_utf16.cpp */ +/* end file src/lasx/lasx_convert_latin1_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_latin1_to_utf32.cpp */ +/* begin file src/lasx/lasx_convert_latin1_to_utf32.cpp */ std::pair -lsx_convert_latin1_to_utf32(const char *buf, size_t len, - char32_t *utf32_output) { +lasx_convert_latin1_to_utf32(const char *buf, size_t len, + char32_t *utf32_output) { const char *end = buf + len; - while (end - buf >= 16) { + // LASX requires 32-byte alignment, otherwise performance will be degraded + while (((uint64_t)utf32_output & 0x1F) && buf < end) { + *utf32_output++ = ((uint32_t)*buf) & 0xFF; + buf++; + } + + while (end - buf >= 32) { + __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); + + __m256i in32_0 = __lasx_vext2xv_wu_bu(in8); + __lasx_xvst(in32_0, reinterpret_cast(utf32_output), 0); + + __m256i in8_1 = __lasx_xvpermi_d(in8, 0b00000001); + __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); + __lasx_xvst(in32_1, reinterpret_cast(utf32_output), 32); + + __m256i in8_2 = __lasx_xvpermi_d(in8, 0b00000010); + __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); + __lasx_xvst(in32_2, reinterpret_cast(utf32_output), 64); + + __m256i in8_3 = __lasx_xvpermi_d(in8, 0b00000011); + __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); + __lasx_xvst(in32_3, reinterpret_cast(utf32_output), 96); + + utf32_output += 32; + buf += 32; + } + + if (end - buf >= 16) { __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); __m128i zero = __lsx_vldi(0); @@ -57495,9 +53800,9 @@ lsx_convert_latin1_to_utf32(const char *buf, size_t len, __m128i in32_3 = __lsx_vilvh_h(zero, in16high); __lsx_vst(in32_0, reinterpret_cast(utf32_output), 0); - __lsx_vst(in32_1, reinterpret_cast(utf32_output + 4), 0); - __lsx_vst(in32_2, reinterpret_cast(utf32_output + 8), 0); - __lsx_vst(in32_3, reinterpret_cast(utf32_output + 12), 0); + __lsx_vst(in32_1, reinterpret_cast(utf32_output), 16); + __lsx_vst(in32_2, reinterpret_cast(utf32_output), 32); + __lsx_vst(in32_3, reinterpret_cast(utf32_output), 48); utf32_output += 16; buf += 16; @@ -57505,11 +53810,11 @@ lsx_convert_latin1_to_utf32(const char *buf, size_t len, return std::make_pair(buf, utf32_output); } -/* end file src/lsx/lsx_convert_latin1_to_utf32.cpp */ +/* end file src/lasx/lasx_convert_latin1_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -/* begin file src/lsx/lsx_convert_utf8_to_utf16.cpp */ +/* begin file src/lasx/lasx_convert_utf8_to_utf16.cpp */ // Convert up to 16 bytes from utf8 to utf16 using a mask indicating the // end of the code points. Only the least significant 12 bits of the mask // are accessed. @@ -57534,15 +53839,22 @@ size_t convert_masked_utf8_to_utf16(const char *input, // We first try a few fast paths. // The obvious first test is ASCII, which actually consumes the full 16. if ((utf8_end_of_code_point_mask & 0xFFFF) == 0xFFFF) { - // We process in chunks of 16 bytes - // The routine in simd.h is reused. - simd8 temp{in}; - temp.store_ascii_as_utf16(utf16_output); + __m128i zero = __lsx_vldi(0); + if constexpr (match_system(big_endian)) { + __lsx_vst(__lsx_vilvl_b(zero, in), + reinterpret_cast(utf16_output), 0); + __lsx_vst(__lsx_vilvh_b(zero, in), + reinterpret_cast(utf16_output), 16); + } else { + __lsx_vst(__lsx_vilvl_b(in, zero), + reinterpret_cast(utf16_output), 0); + __lsx_vst(__lsx_vilvh_b(in, zero), + reinterpret_cast(utf16_output), 16); + } utf16_output += 16; // We wrote 16 16-bit characters. return 16; // We consumed 16 bytes. } - uint64_t buffer[2]; // 3 byte sequences are the next most common, as seen in CJK, which has long // sequences of these. if (input_utf8_end_of_code_point_mask == 0x924) { @@ -57550,7 +53862,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. __m128i composed = convert_utf8_3_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } @@ -57565,13 +53877,13 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. __m128i composed = convert_utf8_2_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } __lsx_vst(composed, reinterpret_cast(utf16_output), 0); - utf16_output += 6; // We wrote 6 16-bit characters. - return 12; // We consumed 12 bytes. + utf16_output += 8; // We wrote 6 16-bit characters. + return 16; // We consumed 12 bytes. } /// We do not have a fast path available, or the fast path is unimportant, so @@ -57587,7 +53899,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // Convert to UTF-16 __m128i composed = convert_utf8_1_to_2_byte_to_utf16(in, idx); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } // Store @@ -57633,7 +53945,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // aaaabbbb bbcccccc composed = __lsx_vbitsel_v(highperm, composed, v0fff); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } @@ -57643,13 +53955,6 @@ size_t convert_masked_utf8_to_utf16(const char *input, } else if (idx < 209) { // THREE (3) input code-code units if (input_utf8_end_of_code_point_mask == 0x888) { - // We want to take 3 4-byte UTF-8 code units and turn them into 3 4-byte - // UTF-16 pairs. Generating surrogate pairs is a little tricky though, but - // it is easier when we can assume they are all pairs. This version does - // not use the LUT, but 4 byte sequences are less common and the overhead - // of the extra memory access is less important than the early branch - // overhead in shorter sequences. - __m128i expected_mask = (__m128i)v16u8{0xf8, 0xc0, 0xc0, 0xc0, 0xf8, 0xc0, 0xc0, 0xc0, 0xf8, 0xc0, 0xc0, 0xc0, 0x0, 0x0, 0x0, 0x0}; @@ -57659,6 +53964,13 @@ size_t convert_masked_utf8_to_utf16(const char *input, __m128i check = __lsx_vseq_b(__lsx_vand_v(in, expected_mask), expected); if (__lsx_bz_b(check)) return 12; + // We want to take 3 4-byte UTF-8 code units and turn them into 3 4-byte + // UTF-16 pairs. Generating surrogate pairs is a little tricky though, but + // it is easier when we can assume they are all pairs. This version does + // not use the LUT, but 4 byte sequences are less common and the overhead + // of the extra memory access is less important than the early branch + // overhead in shorter sequences. + // Swap byte pairs // 10dddddd 10cccccc|10bbbbbb 11110aaa // 10cccccc 10dddddd|11110aaa 10bbbbbb @@ -57676,7 +53988,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, __m128i magic = __lsx_vreplgr2vr_w(uint32_t(0xDC00E7C0)); // Generate unadjusted trail surrogate minus lowest 2 bits // xxxxxxxx xxxxxxxx|11110aaa bbbbbb00 - __m128i trail = __lsx_vbitsel_v(shift, swap, lsx_splat_u32(0x0000ff00)); + __m128i trail = __lsx_vbitsel_v(shift, swap, lsx_splat_u32(0x0000FF00)); // Insert low 2 bits of trail surrogate to magic number for later // 11011100 00000000 11100111 110000cc __m128i magic_with_low_2 = __lsx_vor_v(__lsx_vsrli_w(shift, 30), magic); @@ -57696,12 +54008,10 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 110111CC CCDDDDDD|110110AA BBBBBBCC __m128i composed = __lsx_vadd_h(blend, magic_with_low_2); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } - // __lsx_vst(composed, reinterpret_cast(utf16_output), 0); - __lsx_vst(composed, reinterpret_cast(buffer), 0); - std::memcpy(utf16_output, buffer, 12); + __lsx_vst(composed, reinterpret_cast(utf16_output), 0); utf16_output += 6; // We 3 32-bit surrogate pairs. return 12; // We consumed 12 bytes. } @@ -57772,7 +54082,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 4 byte: 110110AA BBBBBBCC|110111CC CCDDDDDD __m128i selected = __lsx_vbitsel_v(composed, surrogates, is_pair); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { selected = lsx_swap_bytes(selected); } // Attempting to shuffle and store would be complex, just scalarize. @@ -57801,10 +54111,10 @@ size_t convert_masked_utf8_to_utf16(const char *input, return 12; } } -/* end file src/lsx/lsx_convert_utf8_to_utf16.cpp */ +/* end file src/lasx/lasx_convert_utf8_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lsx/lsx_convert_utf8_to_utf32.cpp */ +/* begin file src/lasx/lasx_convert_utf8_to_utf32.cpp */ // Convert up to 12 bytes from utf8 to utf32 using a mask indicating the // end of the code points. Only the least significant 12 bits of the mask // are accessed. @@ -57831,8 +54141,19 @@ size_t convert_masked_utf8_to_utf32(const char *input, // We process in chunks of 16 bytes. // use fast implementation in src/simdutf/arm64/simd.h // Ideally the compiler can keep the tables in registers. - simd8 temp{in}; - temp.store_ascii_as_utf32_tbl(utf32_out); + __m128i zero = __lsx_vldi(0); + __m128i in16low = __lsx_vilvl_b(zero, in); + __m128i in16high = __lsx_vilvh_b(zero, in); + __m128i in32_0 = __lsx_vilvl_h(zero, in16low); + __m128i in32_1 = __lsx_vilvh_h(zero, in16low); + __m128i in32_2 = __lsx_vilvl_h(zero, in16high); + __m128i in32_3 = __lsx_vilvh_h(zero, in16high); + + __lsx_vst(in32_0, reinterpret_cast(utf32_output), 0); + __lsx_vst(in32_1, reinterpret_cast(utf32_output), 16); + __lsx_vst(in32_2, reinterpret_cast(utf32_output), 32); + __lsx_vst(in32_3, reinterpret_cast(utf32_output), 48); + utf32_output += 16; // We wrote 16 32-bit characters. return 16; // We consumed 16 bytes. } @@ -57861,7 +54182,7 @@ size_t convert_masked_utf8_to_utf32(const char *input, utf32_output += 6; return 12; // We consumed 12 bytes. } - /// Either no fast path or an unimportant fast path. + // Either no fast path or an unimportant fast path. const uint8_t idx = simdutf::tables::utf8_to_utf16::utf8bigindex [input_utf8_end_of_code_point_mask][0]; @@ -57984,10 +54305,10 @@ size_t convert_masked_utf8_to_utf32(const char *input, return 12; } } -/* end file src/lsx/lsx_convert_utf8_to_utf32.cpp */ +/* end file src/lasx/lasx_convert_utf8_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_utf8_to_latin1.cpp */ +/* begin file src/lasx/lasx_convert_utf8_to_latin1.cpp */ size_t convert_masked_utf8_to_latin1(const char *input, uint64_t utf8_end_of_code_point_mask, char *&latin1_output) { @@ -58056,27 +54377,24 @@ size_t convert_masked_utf8_to_latin1(const char *input, // writing 8 bytes even though we only care about the first 6 bytes. __m128i latin1_packed = __lsx_vpickev_b(__lsx_vldi(0), composed); - uint64_t buffer[2]; - // __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); - __lsx_vst(latin1_packed, reinterpret_cast(buffer), 0); - std::memcpy(latin1_output, buffer, 6); + __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); latin1_output += 6; // We wrote 6 bytes. return consumed; } -/* end file src/lsx/lsx_convert_utf8_to_latin1.cpp */ +/* end file src/lasx/lasx_convert_utf8_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_utf16_to_latin1.cpp */ +/* begin file src/lasx/lasx_convert_utf16_to_latin1.cpp */ template std::pair -lsx_convert_utf16_to_latin1(const char16_t *buf, size_t len, - char *latin1_output) { +lasx_convert_utf16_to_latin1(const char16_t *buf, size_t len, + char *latin1_output) { const char16_t *end = buf + len; while (end - buf >= 16) { __m128i in = __lsx_vld(reinterpret_cast(buf), 0); __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = lsx_swap_bytes(in); in1 = lsx_swap_bytes(in1); } @@ -58097,14 +54415,14 @@ lsx_convert_utf16_to_latin1(const char16_t *buf, size_t len, template std::pair -lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, - char *latin1_output) { +lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, + char *latin1_output) { const char16_t *start = buf; const char16_t *end = buf + len; while (end - buf >= 16) { __m128i in = __lsx_vld(reinterpret_cast(buf), 0); __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = lsx_swap_bytes(in); in1 = lsx_swap_bytes(in1); } @@ -58119,8 +54437,7 @@ lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } else { // Let us do a scalar fallback. for (int k = 0; k < 16; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if (word <= 0xff) { *latin1_output++ = char(word); } else { @@ -58133,12 +54450,12 @@ lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), latin1_output); } -/* end file src/lsx/lsx_convert_utf16_to_latin1.cpp */ +/* end file src/lasx/lasx_convert_utf16_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 -/* begin file src/lsx/lsx_convert_utf16_to_utf8.cpp */ +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/* begin file src/lasx/lasx_convert_utf16_to_utf8.cpp */ /* - The vectorized algorithm works on single SSE register i.e., it + The vectorized algorithm works on single LASX register i.e., it loads eight 16-bit code units. We consider three cases: @@ -58156,7 +54473,7 @@ lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, char) or 2) two UTF8 bytes. For this case we do only some shuffle to obtain these 2-byte - codes and finally compress the whole SSE register with a single + codes and finally compress the whole LASX register with a single shuffle. We need 256-entry lookup table to get a compression pattern @@ -58174,7 +54491,7 @@ lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, the three-UTF8-bytes case. Finally these two registers are interleaved forming eight-element - array of 32-bit values. The array spans two SSE registers. + array of 32-bit values. The array spans two LASX registers. The bytes from the registers are compressed using two shuffles. We need 256-entry lookup table to get a compression pattern @@ -58189,9 +54506,10 @@ lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, Returns a pair: the first unprocessed byte from buf and utf8_output A scalar routing should carry on the conversion of the tail. */ + template std::pair -lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { +lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char16_t *end = buf + len; @@ -58199,82 +54517,77 @@ lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 - __m128i v_07ff = __lsx_vreplgr2vr_h(uint16_t(0x7ff)); + __m256i v_07ff = __lasx_xvreplgr2vr_h(uint16_t(0x7ff)); + __m256i zero = __lasx_xvldi(0); + __m128i zero_128 = __lsx_vldi(0); while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lsx_swap_bytes(in); + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lasx_swap_bytes(in); } - if (__lsx_bz_v( - __lsx_vslt_hu(__lsx_vrepli_h(0x7F), in))) { // ASCII fast path!!!! - // It is common enough that we have sequences of 16 consecutive ASCII - // characters. - __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { - nextin = lsx_swap_bytes(nextin); - } - if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), nextin))) { - // 1. pack the bytes - // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(nextin, in); - // 2. store (16 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); - // 3. adjust pointers - buf += 16; - utf8_output += 16; - continue; // we are done for this round! - } else { - // 1. pack the bytes - // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(in, in); - // 2. store (8 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); - // 3. adjust pointers - buf += 8; - utf8_output += 8; - in = nextin; - } + if (__lasx_xbnz_h(__lasx_xvslt_hu( + in, __lasx_xvrepli_h(0x7F)))) { // ASCII fast path!!!! + // 1. pack the bytes + __m256i utf8_packed = + __lasx_xvpermi_d(__lasx_xvpickev_b(in, in), 0b00001000); + // 2. store (16 bytes) + __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! } - __m128i zero = __lsx_vldi(0); - if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, in))) { + if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, in))) { // 1. prepare 2-byte values - // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 - // expected output : [110a|aaaa|10bb|bbbb] x 8 + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 16 + // expected output : [110a|aaaa|10bb|bbbb] x 16 // t0 = [000a|aaaa|bbbb|bb00] - __m128i t0 = __lsx_vslli_h(in, 2); + __m256i t0 = __lasx_xvslli_h(in, 2); // t1 = [000a|aaaa|0000|0000] - __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); + __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - __m128i t2 = __lsx_vand_v(in, __lsx_vrepli_h(0x3f)); + __m256i t2 = __lasx_xvand_v(in, __lasx_xvrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - __m128i t3 = __lsx_vor_v(t1, t2); + __m256i t3 = __lasx_xvor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - __m128i v_c080 = __lsx_vreplgr2vr_h(uint16_t(0xc080)); - __m128i t4 = __lsx_vor_v(t3, v_c080); + __m256i v_c080 = __lasx_xvreplgr2vr_h(uint16_t(0xc080)); + __m256i t4 = __lasx_xvor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m128i one_byte_bytemask = - __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F /*0x007F*/)); - __m128i utf8_unpacked = __lsx_vbitsel_v(t4, in, one_byte_bytemask); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F /*0x007F*/)); + __m256i utf8_unpacked = __lasx_xvbitsel_v(t4, in, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - uint32_t m2 = __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); + __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); + uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); // 4. pack the bytes - const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lsx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle = __lsx_vld(row, 1); - __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); + const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m1]][0]; + __m128i shuffle1 = __lsx_vld(row1, 1); + __m128i utf8_packed1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); + + const uint8_t *row2 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_packed2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); // 5. store bytes - __lsx_vst(utf8_packed, utf8_output, 0); - // 6. adjust pointers - buf += 8; - utf8_output += row[0]; + __lsx_vst(utf8_packed1, utf8_output, 0); + utf8_output += row1[0]; + + __lsx_vst(utf8_packed2, utf8_output, 0); + utf8_output += row2[0]; + + buf += 16; continue; } - __m128i surrogates_bytemask = __lsx_vseq_h( - __lsx_vand_v(in, lsx_splat_u16(0xf800)), lsx_splat_u16(0xd800)); + __m256i surrogates_bytemask = __lasx_xvseq_h( + __lasx_xvand_v(in, lasx_splat_u16(0xf800)), lasx_splat_u16(0xd800)); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lsx_bz_v(surrogates_bytemask)) { + if (__lasx_xbz_v(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes /* In this branch we handle three cases: 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - @@ -58304,74 +54617,94 @@ lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m128i t0 = __lsx_vpickev_b(in, in); - t0 = __lsx_vilvl_b(t0, t0); + __m256i t0 = __lasx_xvpickev_b(in, in); + t0 = __lasx_xvilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|00cc|cccc] - __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); - __m128i t1 = __lsx_vand_v(t0, v_3f7f); + __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); + __m256i t1 = __lasx_xvand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); + __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m128i s0 = __lsx_vsrli_h(in, 12); + __m256i s0 = __lasx_xvsrli_h(in, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m128i s1 = __lsx_vslli_h(in, 2); + __m256i s1 = __lasx_xvslli_h(in, 2); // s1: [aabb|bbbb|cccc|cc00] => [00bb|bbbb|0000|0000] - s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3f00)); + s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); // [00bb|bbbb|0000|aaaa] - __m128i s2 = __lsx_vor_v(s0, s1); + __m256i s2 = __lasx_xvor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); - __m128i s3 = __lsx_vor_v(s2, v_c0e0); - __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(in, v_07ff); - __m128i m0 = - __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); - __m128i s4 = __lsx_vxor_v(s3, m0); + __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); + __m256i s3 = __lasx_xvor_v(s2, v_c0e0); + __m256i one_or_two_bytes_bytemask = __lasx_xvsle_hu(in, v_07ff); + __m256i m0 = + __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); + __m256i s4 = __lasx_xvxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m128i out0 = __lsx_vilvl_h(s4, t2); - __m128i out1 = __lsx_vilvh_h(s4, t2); + __m256i out0 = __lasx_xvilvl_h(s4, t2); + __m256i out1 = __lasx_xvilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m128i one_byte_bytemask = __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F)); - - __m128i one_or_two_bytes_bytemask_low = - __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); - __m128i one_or_two_bytes_bytemask_high = - __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); + __m256i one_byte_bytemask = __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F)); + __m256i one_byte_bytemask_low = + __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_high = + __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_low = - __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_high = - __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_or_two_bytes_bytemask_low = + __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_high = + __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); - const uint32_t mask0 = __lsx_vpickve2gr_bu( - __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_low, - one_byte_bytemask_low)), - 0); - const uint32_t mask1 = __lsx_vpickve2gr_bu( - __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_high, - one_byte_bytemask_high)), - 0); + __m256i mask0 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_low, one_byte_bytemask_low)); + __m256i mask1 = __lasx_xvmskltz_h(__lasx_xvor_v( + one_or_two_bytes_bytemask_high, one_byte_bytemask_high)); + uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); + __m128i utf8_0 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; + mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); - - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - buf += 8; + mask = __lasx_xvpickve2gr_wu(mask0, 4); + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); + __lsx_vst(utf8_2, utf8_output, 0); + utf8_output += row2[0]; + + mask = __lasx_xvpickve2gr_wu(mask1, 4); + const uint8_t *row3 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle3 = __lsx_vld(row3, 1); + __m128i utf8_3 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); + __lsx_vst(utf8_3, utf8_output, 0); + utf8_output += row3[0]; + + buf += 16; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -58383,8 +54716,7 @@ lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -58397,9 +54729,8 @@ lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -58428,8 +54759,8 @@ lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { */ template std::pair -lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, - char *utf8_out) { +lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char16_t *start = buf; const char16_t *end = buf + len; @@ -58437,82 +54768,78 @@ lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 + + __m256i v_07ff = __lasx_xvreplgr2vr_h(uint16_t(0x7ff)); + __m256i zero = __lasx_xvldi(0); + __m128i zero_128 = __lsx_vldi(0); while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lsx_swap_bytes(in); + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lasx_swap_bytes(in); } - if (__lsx_bz_v( - __lsx_vslt_hu(__lsx_vrepli_h(0x7F), in))) { // ASCII fast path!!!! - // It is common enough that we have sequences of 16 consecutive ASCII - // characters. - __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { - nextin = lsx_swap_bytes(nextin); - } - if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), nextin))) { - // 1. pack the bytes - // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(nextin, in); - // 2. store (16 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); - // 3. adjust pointers - buf += 16; - utf8_output += 16; - continue; // we are done for this round! - } else { - // 1. pack the bytes - // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(in, in); - // 2. store (8 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); - // 3. adjust pointers - buf += 8; - utf8_output += 8; - in = nextin; - } + if (__lasx_xbnz_h(__lasx_xvslt_hu( + in, __lasx_xvrepli_h(0x7F)))) { // ASCII fast path!!!! + // 1. pack the bytes + __m256i utf8_packed = + __lasx_xvpermi_d(__lasx_xvpickev_b(in, in), 0b00001000); + // 2. store (16 bytes) + __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! } - __m128i v_07ff = __lsx_vreplgr2vr_h(uint16_t(0x7ff)); - __m128i zero = __lsx_vldi(0); - if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, in))) { + if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, in))) { // 1. prepare 2-byte values - // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 - // expected output : [110a|aaaa|10bb|bbbb] x 8 + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 16 + // expected output : [110a|aaaa|10bb|bbbb] x 16 // t0 = [000a|aaaa|bbbb|bb00] - __m128i t0 = __lsx_vslli_h(in, 2); + __m256i t0 = __lasx_xvslli_h(in, 2); // t1 = [000a|aaaa|0000|0000] - __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); + __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - __m128i t2 = __lsx_vand_v(in, __lsx_vrepli_h(0x3f)); + __m256i t2 = __lasx_xvand_v(in, __lasx_xvrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - __m128i t3 = __lsx_vor_v(t1, t2); + __m256i t3 = __lasx_xvor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - __m128i v_c080 = __lsx_vreplgr2vr_h(uint16_t(0xc080)); - __m128i t4 = __lsx_vor_v(t3, v_c080); + __m256i v_c080 = __lasx_xvreplgr2vr_h(uint16_t(0xc080)); + __m256i t4 = __lasx_xvor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m128i one_byte_bytemask = - __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F /*0x007F*/)); - __m128i utf8_unpacked = __lsx_vbitsel_v(t4, in, one_byte_bytemask); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F /*0x007F*/)); + __m256i utf8_unpacked = __lasx_xvbitsel_v(t4, in, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - uint32_t m2 = __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); + __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); + uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); // 4. pack the bytes - const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lsx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle = __lsx_vld(row, 1); - __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); + const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m1]][0]; + __m128i shuffle1 = __lsx_vld(row1, 1); + __m128i utf8_packed1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); + + const uint8_t *row2 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_packed2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); // 5. store bytes - __lsx_vst(utf8_packed, utf8_output, 0); - // 6. adjust pointers - buf += 8; - utf8_output += row[0]; + __lsx_vst(utf8_packed1, utf8_output, 0); + utf8_output += row1[0]; + + __lsx_vst(utf8_packed2, utf8_output, 0); + utf8_output += row2[0]; + + buf += 16; continue; } - __m128i surrogates_bytemask = __lsx_vseq_h( - __lsx_vand_v(in, lsx_splat_u16(0xf800)), lsx_splat_u16(0xd800)); + __m256i surrogates_bytemask = __lasx_xvseq_h( + __lasx_xvand_v(in, lasx_splat_u16(0xf800)), lasx_splat_u16(0xd800)); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lsx_bz_v(surrogates_bytemask)) { + if (__lasx_xbz_v(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes /* In this branch we handle three cases: 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - @@ -58542,74 +54869,94 @@ lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m128i t0 = __lsx_vpickev_b(in, in); - t0 = __lsx_vilvl_b(t0, t0); + __m256i t0 = __lasx_xvpickev_b(in, in); + t0 = __lasx_xvilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|00cc|cccc] - __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); - __m128i t1 = __lsx_vand_v(t0, v_3f7f); + __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); + __m256i t1 = __lasx_xvand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); + __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m128i s0 = __lsx_vsrli_h(in, 12); + __m256i s0 = __lasx_xvsrli_h(in, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m128i s1 = __lsx_vslli_h(in, 2); + __m256i s1 = __lasx_xvslli_h(in, 2); // s1: [aabb|bbbb|cccc|cc00] => [00bb|bbbb|0000|0000] - s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3f00)); + s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); // [00bb|bbbb|0000|aaaa] - __m128i s2 = __lsx_vor_v(s0, s1); + __m256i s2 = __lasx_xvor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); - __m128i s3 = __lsx_vor_v(s2, v_c0e0); - __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(in, v_07ff); - __m128i m0 = - __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); - __m128i s4 = __lsx_vxor_v(s3, m0); + __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); + __m256i s3 = __lasx_xvor_v(s2, v_c0e0); + __m256i one_or_two_bytes_bytemask = __lasx_xvsle_hu(in, v_07ff); + __m256i m0 = + __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); + __m256i s4 = __lasx_xvxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m128i out0 = __lsx_vilvl_h(s4, t2); - __m128i out1 = __lsx_vilvh_h(s4, t2); + __m256i out0 = __lasx_xvilvl_h(s4, t2); + __m256i out1 = __lasx_xvilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m128i one_byte_bytemask = __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F)); - - __m128i one_or_two_bytes_bytemask_low = - __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); - __m128i one_or_two_bytes_bytemask_high = - __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); + __m256i one_byte_bytemask = __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F)); + __m256i one_byte_bytemask_low = + __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_high = + __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_low = - __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_high = - __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_or_two_bytes_bytemask_low = + __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_high = + __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); - const uint32_t mask0 = __lsx_vpickve2gr_bu( - __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_low, - one_byte_bytemask_low)), - 0); - const uint32_t mask1 = __lsx_vpickve2gr_bu( - __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_high, - one_byte_bytemask_high)), - 0); + __m256i mask0 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_low, one_byte_bytemask_low)); + __m256i mask1 = __lasx_xvmskltz_h(__lasx_xvor_v( + one_or_two_bytes_bytemask_high, one_byte_bytemask_high)); + uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); + __m128i utf8_0 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; + mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); - - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - buf += 8; + mask = __lasx_xvpickve2gr_wu(mask0, 4); + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); + __lsx_vst(utf8_2, utf8_output, 0); + utf8_output += row2[0]; + + mask = __lasx_xvpickve2gr_wu(mask1, 4); + const uint8_t *row3 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle3 = __lsx_vld(row3, 1); + __m128i utf8_3 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); + __lsx_vst(utf8_3, utf8_output, 0); + utf8_output += row3[0]; + + buf += 16; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -58621,8 +54968,7 @@ lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -58635,9 +54981,8 @@ lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -58659,38 +55004,63 @@ lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf8_output)); } -/* end file src/lsx/lsx_convert_utf16_to_utf8.cpp */ -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 +/* end file src/lasx/lasx_convert_utf16_to_utf8.cpp */ +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lsx/lsx_convert_utf16_to_utf32.cpp */ +/* begin file src/lasx/lasx_convert_utf16_to_utf32.cpp */ template std::pair -lsx_convert_utf16_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_out) { +lasx_convert_utf16_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_out) { uint32_t *utf32_output = reinterpret_cast(utf32_out); const char16_t *end = buf + len; - __m128i zero = __lsx_vldi(0); - __m128i v_f800 = lsx_splat_u16(0xf800); - __m128i v_d800 = lsx_splat_u16(0xd800); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)utf32_output & 0x1f) && buf < end) { + uint16_t word = scalar::utf16::swap_if_needed(buf[0]); + if ((word & 0xF800) != 0xD800) { + *utf32_output++ = char32_t(word); + buf++; + } else { + if (buf + 1 >= end) { + return std::make_pair(nullptr, + reinterpret_cast(utf32_output)); + } + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = scalar::utf16::swap_if_needed(buf[1]); + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if ((diff | diff2) > 0x3FF) { + return std::make_pair(nullptr, + reinterpret_cast(utf32_output)); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + buf += 2; + } + } - while (end - buf >= 8) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lsx_swap_bytes(in); + __m256i v_f800 = lasx_splat_u16(0xf800); + __m256i v_d800 = lasx_splat_u16(0xd800); + + while (end - buf >= 16) { + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lasx_swap_bytes(in); } - __m128i surrogates_bytemask = - __lsx_vseq_h(__lsx_vand_v(in, v_f800), v_d800); + __m256i surrogates_bytemask = + __lasx_xvseq_h(__lasx_xvand_v(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lsx_bz_v(surrogates_bytemask)) { + if (__lasx_xbz_v(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units - __lsx_vst(__lsx_vilvl_h(zero, in), utf32_output, 0); - __lsx_vst(__lsx_vilvh_h(zero, in), utf32_output, 16); - utf32_output += 8; - buf += 8; + __m256i in_hi = __lasx_xvpermi_q(in, in, 0b00000001); + __lasx_xvst(__lasx_vext2xv_wu_hu(in), utf32_output, 0); + __lasx_xvst(__lasx_vext2xv_wu_hu(in_hi), utf32_output, 32); + utf32_output += 16; + buf += 16; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -58702,16 +55072,14 @@ lsx_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -58737,31 +55105,56 @@ lsx_convert_utf16_to_utf32(const char16_t *buf, size_t len, */ template std::pair -lsx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, - char32_t *utf32_out) { +lasx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, + char32_t *utf32_out) { uint32_t *utf32_output = reinterpret_cast(utf32_out); const char16_t *start = buf; const char16_t *end = buf + len; - __m128i zero = __lsx_vldi(0); - __m128i v_f800 = lsx_splat_u16(0xf800); - __m128i v_d800 = lsx_splat_u16(0xd800); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)utf32_output & 0x1f) && buf < end) { + uint16_t word = scalar::utf16::swap_if_needed(buf[0]); + if ((word & 0xF800) != 0xD800) { + *utf32_output++ = char32_t(word); + buf++; + } else if (buf + 1 < end) { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + uint16_t next_word = scalar::utf16::swap_if_needed(buf[1]); + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if ((diff | diff2) > 0x3FF) { + return std::make_pair(result(error_code::SURROGATE, buf - start), + reinterpret_cast(utf32_output)); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + buf += 2; + } else { + return std::make_pair(result(error_code::SURROGATE, buf - start), + reinterpret_cast(utf32_output)); + } + } - while (end - buf >= 8) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lsx_swap_bytes(in); + __m256i v_f800 = lasx_splat_u16(0xf800); + __m256i v_d800 = lasx_splat_u16(0xd800); + while (end - buf >= 16) { + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lasx_swap_bytes(in); } - __m128i surrogates_bytemask = - __lsx_vseq_h(__lsx_vand_v(in, v_f800), v_d800); - if (__lsx_bz_v(surrogates_bytemask)) { + __m256i surrogates_bytemask = + __lasx_xvseq_h(__lasx_xvand_v(in, v_f800), v_d800); + // It might seem like checking for surrogates_bitmask == 0xc000 could help. + // However, it is likely an uncommon occurrence. + if (__lasx_xbz_v(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units - __lsx_vst(__lsx_vilvl_h(zero, in), utf32_output, 0); - __lsx_vst(__lsx_vilvh_h(zero, in), utf32_output, 16); - utf32_output += 8; - buf += 8; + __m256i in_hi = __lasx_xvpermi_q(in, in, 0b00000001); + __lasx_xvst(__lasx_vext2xv_wu_hu(in), utf32_output, 0); + __lasx_xvst(__lasx_vext2xv_wu_hu(in_hi), utf32_output, 32); + utf32_output += 16; + buf += 16; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -58773,16 +55166,14 @@ lsx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -58800,31 +55191,35 @@ lsx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf32_output)); } -/* end file src/lsx/lsx_convert_utf16_to_utf32.cpp */ +/* end file src/lasx/lasx_convert_utf16_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lsx/lsx_convert_utf32_to_latin1.cpp */ +/* begin file src/lasx/lasx_convert_utf32_to_latin1.cpp */ std::pair -lsx_convert_utf32_to_latin1(const char32_t *buf, size_t len, - char *latin1_output) { +lasx_convert_utf32_to_latin1(const char32_t *buf, size_t len, + char *latin1_output) { const char32_t *end = buf + len; - const v16u8 shuf_mask = {0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}; - __m128i v_ff = __lsx_vrepli_w(0xFF); + const __m256i shuf_mask = ____m256i( + (__m128i)v16u8{0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}); + __m256i v_ff = __lasx_xvrepli_w(0xFF); while (end - buf >= 16) { - __m128i in1 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i in2 = __lsx_vld(reinterpret_cast(buf), 16); + __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i in2 = __lasx_xvld(reinterpret_cast(buf), 32); - __m128i in12 = __lsx_vor_v(in1, in2); - if (__lsx_bz_v(__lsx_vslt_wu(v_ff, in12))) { + __m256i in12 = __lasx_xvor_v(in1, in2); + if (__lasx_xbz_v(__lasx_xvslt_wu(v_ff, in12))) { // 1. pack the bytes - __m128i latin1_packed = __lsx_vshuf_b(in2, in1, (__m128i)shuf_mask); + __m256i latin1_packed_tmp = __lasx_xvshuf_b(in2, in1, shuf_mask); + latin1_packed_tmp = __lasx_xvpermi_d(latin1_packed_tmp, 0b00001000); + __m128i latin1_packed = lasx_extracti128_lo(latin1_packed_tmp); + latin1_packed = __lsx_vpermi_w(latin1_packed, latin1_packed, 0b11011000); // 2. store (8 bytes) __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); // 3. adjust pointers - buf += 8; - latin1_output += 8; + buf += 16; + latin1_output += 16; } else { return std::make_pair(nullptr, reinterpret_cast(latin1_output)); } @@ -58833,31 +55228,34 @@ lsx_convert_utf32_to_latin1(const char32_t *buf, size_t len, } std::pair -lsx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, - char *latin1_output) { +lasx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, + char *latin1_output) { const char32_t *start = buf; const char32_t *end = buf + len; - const v16u8 shuf_mask = {0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}; - __m128i v_ff = __lsx_vrepli_w(0xFF); + const __m256i shuf_mask = ____m256i( + (__m128i)v16u8{0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}); + __m256i v_ff = __lasx_xvrepli_w(0xFF); while (end - buf >= 16) { - __m128i in1 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i in2 = __lsx_vld(reinterpret_cast(buf), 16); - - __m128i in12 = __lsx_vor_v(in1, in2); + __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i in2 = __lasx_xvld(reinterpret_cast(buf), 32); - if (__lsx_bz_v(__lsx_vslt_wu(v_ff, in12))) { + __m256i in12 = __lasx_xvor_v(in1, in2); + if (__lasx_xbz_v(__lasx_xvslt_wu(v_ff, in12))) { // 1. pack the bytes - __m128i latin1_packed = __lsx_vshuf_b(in2, in1, (__m128i)shuf_mask); + __m256i latin1_packed_tmp = __lasx_xvshuf_b(in2, in1, shuf_mask); + latin1_packed_tmp = __lasx_xvpermi_d(latin1_packed_tmp, 0b00001000); + __m128i latin1_packed = lasx_extracti128_lo(latin1_packed_tmp); + latin1_packed = __lsx_vpermi_w(latin1_packed, latin1_packed, 0b11011000); // 2. store (8 bytes) __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); // 3. adjust pointers - buf += 8; - latin1_output += 8; + buf += 16; + latin1_output += 16; } else { // Let us do a scalar fallback. - for (int k = 0; k < 8; k++) { + for (int k = 0; k < 16; k++) { uint32_t word = buf[k]; if (word <= 0xff) { *latin1_output++ = char(word); @@ -58871,188 +55269,253 @@ lsx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), latin1_output); } -/* end file src/lsx/lsx_convert_utf32_to_latin1.cpp */ +/* end file src/lasx/lasx_convert_utf32_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lsx/lsx_convert_utf32_to_utf8.cpp */ +/* begin file src/lasx/lasx_convert_utf32_to_utf8.cpp */ std::pair -lsx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { +lasx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char32_t *end = buf + len; - __m128i v_c080 = lsx_splat_u16(0xc080); - __m128i v_07ff = lsx_splat_u16(0x07ff); - __m128i v_dfff = lsx_splat_u16(0xdfff); - __m128i v_d800 = lsx_splat_u16(0xd800); - __m128i forbidden_bytemask = __lsx_vldi(0x0); + // load addr align 32 + while (((uint64_t)buf & 0x1F) && buf < end) { + uint32_t word = *buf; + if ((word & 0xFFFFFF80) == 0) { + *utf8_output++ = char(word); + } else if ((word & 0xFFFFF800) == 0) { + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return std::make_pair(nullptr, reinterpret_cast(utf8_output)); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else { + if (word > 0x10FFFF) { + return std::make_pair(nullptr, reinterpret_cast(utf8_output)); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } + buf++; + } + + __m256i v_c080 = lasx_splat_u16(0xc080); + __m256i v_07ff = lasx_splat_u16(0x07ff); + __m256i v_dfff = lasx_splat_u16(0xdfff); + __m256i v_d800 = lasx_splat_u16(0xd800); + __m256i zero = __lasx_xvldi(0); + __m128i zero_128 = __lsx_vldi(0); + __m256i forbidden_bytemask = __lasx_xvldi(0x0); const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 while (end - buf > std::ptrdiff_t(16 + safety_margin)) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i nextin = __lasx_xvld(reinterpret_cast(buf), 32); // Check if no bits set above 16th - if (__lsx_bz_v(__lsx_vpickod_h(in, nextin))) { + if (__lasx_xbz_v(__lasx_xvpickod_h(in, nextin))) { // Pack UTF-32 to UTF-16 safely (without surrogate pairs) - // Apply UTF-16 => UTF-8 routine (lsx_convert_utf16_to_utf8.cpp) - __m128i utf16_packed = __lsx_vpickev_h(nextin, in); + // Apply UTF-16 => UTF-8 routine (lasx_convert_utf16_to_utf8.cpp) + __m256i utf16_packed = + __lasx_xvpermi_d(__lasx_xvpickev_h(nextin, in), 0b11011000); - if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), - utf16_packed))) { // ASCII fast path!!!! + if (__lasx_xbz_v(__lasx_xvslt_hu(__lasx_xvrepli_h(0x7F), + utf16_packed))) { // ASCII fast path!!!! // 1. pack the bytes // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(utf16_packed, utf16_packed); + __m256i utf8_packed = __lasx_xvpermi_d( + __lasx_xvpickev_b(utf16_packed, utf16_packed), 0b00001000); // 2. store (8 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); + __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); // 3. adjust pointers - buf += 8; - utf8_output += 8; + buf += 16; + utf8_output += 16; continue; // we are done for this round! } - __m128i zero = __lsx_vldi(0); - if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, utf16_packed))) { + + if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, utf16_packed))) { // 1. prepare 2-byte values // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - const __m128i t0 = __lsx_vslli_h(utf16_packed, 2); + const __m256i t0 = __lasx_xvslli_h(utf16_packed, 2); // t1 = [000a|aaaa|0000|0000] - const __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); + const __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - const __m128i t2 = __lsx_vand_v(utf16_packed, __lsx_vrepli_h(0x3f)); + const __m256i t2 = __lasx_xvand_v(utf16_packed, __lasx_xvrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - const __m128i t3 = __lsx_vor_v(t1, t2); + const __m256i t3 = __lasx_xvor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - const __m128i t4 = __lsx_vor_v(t3, v_c080); + const __m256i t4 = __lasx_xvor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m128i one_byte_bytemask = - __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F /*0x007F*/)); - __m128i utf8_unpacked = - __lsx_vbitsel_v(t4, utf16_packed, one_byte_bytemask); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F /*0x007F*/)); + __m256i utf8_unpacked = + __lasx_xvbitsel_v(t4, utf16_packed, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - uint32_t m2 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); + __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); + uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); // 4. pack the bytes - const uint8_t *row = + const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lsx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle = __lsx_vld(row, 1); - __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); + [lasx_1_2_utf8_bytes_mask[m1]][0]; + __m128i shuffle1 = __lsx_vld(row1, 1); + __m128i utf8_packed1 = __lsx_vshuf_b( + zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); + + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_packed2 = __lsx_vshuf_b( + zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); // 5. store bytes - __lsx_vst(utf8_packed, utf8_output, 0); + __lsx_vst(utf8_packed1, utf8_output, 0); + utf8_output += row1[0]; - // 6. adjust pointers - buf += 8; - utf8_output += row[0]; + __lsx_vst(utf8_packed2, utf8_output, 0); + utf8_output += row2[0]; + + buf += 16; continue; } else { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes - forbidden_bytemask = __lsx_vor_v( - __lsx_vand_v( - __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + forbidden_bytemask = __lasx_xvor_v( + __lasx_xvand_v( + __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); /* In this branch we handle three cases: - 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single - UFT-8 byte - 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two - UTF-8 bytes - 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three - UTF-8 bytes + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - + single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - + two UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - + three UTF-8 bytes - We expand the input word (16-bit) into two code units (32-bit), thus - we have room for four bytes. However, we need five distinct bit - layouts. Note that the last byte in cases #2 and #3 is the same. + We expand the input word (16-bit) into two code units (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. - We precompute byte 1 for case #1 and the common byte for cases #2 & #3 - in register t2. + We precompute byte 1 for case #1 and the common byte for cases #2 & + #3 in register t2. - We precompute byte 1 for case #3 and -- **conditionally** -- precompute - either byte 1 for case #2 or byte 2 for case #3. Note that they - differ by exactly one bit. + We precompute byte 1 for case #3 and -- **conditionally** -- + precompute either byte 1 for case #2 or byte 2 for case #3. Note that + they differ by exactly one bit. - Finally from these two code units we build proper UTF-8 sequence, taking - into account the case (i.e, the number of bytes to write). - */ + Finally from these two code units we build proper UTF-8 sequence, + taking into account the case (i.e, the number of bytes to write). + */ /** * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: * t2 => [0ccc|cccc] [10cc|cccc] * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m128i t0 = __lsx_vpickev_b(utf16_packed, utf16_packed); - t0 = __lsx_vilvl_b(t0, t0); + __m256i t0 = __lasx_xvpickev_b(utf16_packed, utf16_packed); + t0 = __lasx_xvilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] - __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); - __m128i t1 = __lsx_vand_v(t0, v_3f7f); + __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); + __m256i t1 = __lasx_xvand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); + __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m128i s0 = __lsx_vsrli_h(utf16_packed, 12); + __m256i s0 = __lasx_xvsrli_h(utf16_packed, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m128i s1 = __lsx_vslli_h(utf16_packed, 2); + __m256i s1 = __lasx_xvslli_h(utf16_packed, 2); // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000] - s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3F00)); + s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); // [00bb|bbbb|0000|aaaa] - __m128i s2 = __lsx_vor_v(s0, s1); + __m256i s2 = __lasx_xvor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); - __m128i s3 = __lsx_vor_v(s2, v_c0e0); - __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(utf16_packed, v_07ff); - __m128i m0 = - __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); - __m128i s4 = __lsx_vxor_v(s3, m0); + __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); + __m256i s3 = __lasx_xvor_v(s2, v_c0e0); + // __m256i v_07ff = vmovq_n_u16((uint16_t)0x07FF); + __m256i one_or_two_bytes_bytemask = + __lasx_xvsle_hu(utf16_packed, v_07ff); + __m256i m0 = + __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); + __m256i s4 = __lasx_xvxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m128i out0 = __lsx_vilvl_h(s4, t2); - __m128i out1 = __lsx_vilvh_h(s4, t2); + __m256i out0 = __lasx_xvilvl_h(s4, t2); + __m256i out1 = __lasx_xvilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m128i one_byte_bytemask = - __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F)); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F)); - __m128i one_or_two_bytes_bytemask_u16_to_u32_low = - __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); - __m128i one_or_two_bytes_bytemask_u16_to_u32_high = - __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_u16_to_u32_low = + __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_u16_to_u32_high = + __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); - __m128i one_byte_bytemask_u16_to_u32_low = - __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_u16_to_u32_high = - __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_u16_to_u32_low = + __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_u16_to_u32_high = + __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); - const uint32_t mask0 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( - one_or_two_bytes_bytemask_u16_to_u32_low, - one_byte_bytemask_u16_to_u32_low)), - 0); - const uint32_t mask1 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( - one_or_two_bytes_bytemask_u16_to_u32_high, - one_byte_bytemask_u16_to_u32_high)), - 0); + __m256i mask0 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_low, + one_byte_bytemask_u16_to_u32_low)); + __m256i mask1 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_high, + one_byte_bytemask_u16_to_u32_high)); + uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); + __m128i utf8_0 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; + mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); - - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - buf += 8; + mask = __lasx_xvpickve2gr_wu(mask0, 4); + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); + __lsx_vst(utf8_2, utf8_output, 0); + utf8_output += row2[0]; + + mask = __lasx_xvpickve2gr_wu(mask1, 4); + const uint8_t *row3 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle3 = __lsx_vld(row3, 1); + __m128i utf8_3 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); + __lsx_vst(utf8_3, utf8_output, 0); + utf8_output += row3[0]; + + buf += 16; } // At least one 32-bit word will produce a surrogate pair in UTF-16 <=> // will produce four UTF-8 bytes. @@ -59096,197 +55559,262 @@ lsx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { } // while // check for invalid input - if (__lsx_bnz_v(forbidden_bytemask)) { + if (__lasx_xbnz_v(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf8_output)); } - return std::make_pair(buf, reinterpret_cast(utf8_output)); } std::pair -lsx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, - char *utf8_out) { +lasx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char32_t *start = buf; const char32_t *end = buf + len; - __m128i v_c080 = lsx_splat_u16(0xc080); - __m128i v_07ff = lsx_splat_u16(0x07ff); - __m128i v_dfff = lsx_splat_u16(0xdfff); - __m128i v_d800 = lsx_splat_u16(0xd800); - __m128i forbidden_bytemask = __lsx_vldi(0x0); + // load addr align 32 + while (((uint64_t)buf & 0x1F) && buf < end) { + uint32_t word = *buf; + if ((word & 0xFFFFFF80) == 0) { + *utf8_output++ = char(word); + } else if ((word & 0xFFFFF800) == 0) { + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return std::make_pair(result(error_code::SURROGATE, buf - start), + reinterpret_cast(utf8_output)); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } else { + if (word > 0x10FFFF) { + return std::make_pair(result(error_code::TOO_LARGE, buf - start), + reinterpret_cast(utf8_output)); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + } + buf++; + } + + __m256i v_c080 = lasx_splat_u16(0xc080); + __m256i v_07ff = lasx_splat_u16(0x07ff); + __m256i v_dfff = lasx_splat_u16(0xdfff); + __m256i v_d800 = lasx_splat_u16(0xd800); + __m256i zero = __lasx_xvldi(0); + __m128i zero_128 = __lsx_vldi(0); + __m256i forbidden_bytemask = __lasx_xvldi(0x0); const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 while (end - buf > std::ptrdiff_t(16 + safety_margin)) { - __m128i in = __lsx_vld(reinterpret_cast(buf), 0); - __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); + __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i nextin = __lasx_xvld(reinterpret_cast(buf), 32); // Check if no bits set above 16th - if (__lsx_bz_v(__lsx_vpickod_h(in, nextin))) { + if (__lasx_xbz_v(__lasx_xvpickod_h(in, nextin))) { // Pack UTF-32 to UTF-16 safely (without surrogate pairs) - // Apply UTF-16 => UTF-8 routine (lsx_convert_utf16_to_utf8.cpp) - __m128i utf16_packed = __lsx_vpickev_h(nextin, in); + // Apply UTF-16 => UTF-8 routine (lasx_convert_utf16_to_utf8.cpp) + __m256i utf16_packed = + __lasx_xvpermi_d(__lasx_xvpickev_h(nextin, in), 0b11011000); - if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), - utf16_packed))) { // ASCII fast path!!!! + if (__lasx_xbz_v(__lasx_xvslt_hu(__lasx_xvrepli_h(0x7F), + utf16_packed))) { // ASCII fast path!!!! // 1. pack the bytes // obviously suboptimal. - __m128i utf8_packed = __lsx_vpickev_b(utf16_packed, utf16_packed); + __m256i utf8_packed = __lasx_xvpermi_d( + __lasx_xvpickev_b(utf16_packed, utf16_packed), 0b00001000); // 2. store (8 bytes) - __lsx_vst(utf8_packed, utf8_output, 0); + __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); // 3. adjust pointers - buf += 8; - utf8_output += 8; + buf += 16; + utf8_output += 16; continue; // we are done for this round! } - __m128i zero = __lsx_vldi(0); - if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, utf16_packed))) { + + if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, utf16_packed))) { // 1. prepare 2-byte values // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - const __m128i t0 = __lsx_vslli_h(utf16_packed, 2); + const __m256i t0 = __lasx_xvslli_h(utf16_packed, 2); // t1 = [000a|aaaa|0000|0000] - const __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); + const __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - const __m128i t2 = __lsx_vand_v(utf16_packed, __lsx_vrepli_h(0x3f)); + const __m256i t2 = __lasx_xvand_v(utf16_packed, __lasx_xvrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - const __m128i t3 = __lsx_vor_v(t1, t2); + const __m256i t3 = __lasx_xvor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - const __m128i t4 = __lsx_vor_v(t3, v_c080); + const __m256i t4 = __lasx_xvor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m128i one_byte_bytemask = - __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F /*0x007F*/)); - __m128i utf8_unpacked = - __lsx_vbitsel_v(t4, utf16_packed, one_byte_bytemask); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F /*0x007F*/)); + __m256i utf8_unpacked = + __lasx_xvbitsel_v(t4, utf16_packed, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - uint32_t m2 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); + __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); + uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); + uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); // 4. pack the bytes - const uint8_t *row = + const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lsx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle = __lsx_vld(row, 1); - __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); + [lasx_1_2_utf8_bytes_mask[m1]][0]; + __m128i shuffle1 = __lsx_vld(row1, 1); + __m128i utf8_packed1 = __lsx_vshuf_b( + zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); + + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lasx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_packed2 = __lsx_vshuf_b( + zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); // 5. store bytes - __lsx_vst(utf8_packed, utf8_output, 0); + __lsx_vst(utf8_packed1, utf8_output, 0); + utf8_output += row1[0]; - // 6. adjust pointers - buf += 8; - utf8_output += row[0]; + __lsx_vst(utf8_packed2, utf8_output, 0); + utf8_output += row2[0]; + + buf += 16; continue; } else { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes - forbidden_bytemask = __lsx_vor_v( - __lsx_vand_v( - __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + forbidden_bytemask = __lasx_xvor_v( + __lasx_xvand_v( + __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (__lsx_bnz_v(forbidden_bytemask)) { + if (__lasx_xbnz_v(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf8_output)); } /* In this branch we handle three cases: - 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single - UFT-8 byte - 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two - UTF-8 bytes - 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three - UTF-8 bytes + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - + single UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - + two UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - + three UTF-8 bytes - We expand the input word (16-bit) into two code units (32-bit), thus - we have room for four bytes. However, we need five distinct bit - layouts. Note that the last byte in cases #2 and #3 is the same. + We expand the input word (16-bit) into two code units (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. - We precompute byte 1 for case #1 and the common byte for cases #2 & #3 - in register t2. + We precompute byte 1 for case #1 and the common byte for cases #2 & + #3 in register t2. - We precompute byte 1 for case #3 and -- **conditionally** -- precompute - either byte 1 for case #2 or byte 2 for case #3. Note that they - differ by exactly one bit. + We precompute byte 1 for case #3 and -- **conditionally** -- + precompute either byte 1 for case #2 or byte 2 for case #3. Note that + they differ by exactly one bit. - Finally from these two code units we build proper UTF-8 sequence, taking - into account the case (i.e, the number of bytes to write). - */ + Finally from these two code units we build proper UTF-8 sequence, + taking into account the case (i.e, the number of bytes to write). + */ /** * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: * t2 => [0ccc|cccc] [10cc|cccc] * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m128i t0 = __lsx_vpickev_b(utf16_packed, utf16_packed); - t0 = __lsx_vilvl_b(t0, t0); + __m256i t0 = __lasx_xvpickev_b(utf16_packed, utf16_packed); + t0 = __lasx_xvilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] - __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); - __m128i t1 = __lsx_vand_v(t0, v_3f7f); + __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); + __m256i t1 = __lasx_xvand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); + __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m128i s0 = __lsx_vsrli_h(utf16_packed, 12); + __m256i s0 = __lasx_xvsrli_h(utf16_packed, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m128i s1 = __lsx_vslli_h(utf16_packed, 2); + __m256i s1 = __lasx_xvslli_h(utf16_packed, 2); // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000] - s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3F00)); + s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3F00)); // [00bb|bbbb|0000|aaaa] - __m128i s2 = __lsx_vor_v(s0, s1); + __m256i s2 = __lasx_xvor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); - __m128i s3 = __lsx_vor_v(s2, v_c0e0); - // __m128i v_07ff = vmovq_n_u16((uint16_t)0x07FF); - __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(utf16_packed, v_07ff); - __m128i m0 = - __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); - __m128i s4 = __lsx_vxor_v(s3, m0); + __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); + __m256i s3 = __lasx_xvor_v(s2, v_c0e0); + // __m256i v_07ff = vmovq_n_u16((uint16_t)0x07FF); + __m256i one_or_two_bytes_bytemask = + __lasx_xvsle_hu(utf16_packed, v_07ff); + __m256i m0 = + __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); + __m256i s4 = __lasx_xvxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m128i out0 = __lsx_vilvl_h(s4, t2); - __m128i out1 = __lsx_vilvh_h(s4, t2); + __m256i out0 = __lasx_xvilvl_h(s4, t2); + __m256i out1 = __lasx_xvilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m128i one_byte_bytemask = - __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F)); + __m256i one_byte_bytemask = + __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F)); - __m128i one_or_two_bytes_bytemask_u16_to_u32_low = - __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); - __m128i one_or_two_bytes_bytemask_u16_to_u32_high = - __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_u16_to_u32_low = + __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); + __m256i one_or_two_bytes_bytemask_u16_to_u32_high = + __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); - __m128i one_byte_bytemask_u16_to_u32_low = - __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); - __m128i one_byte_bytemask_u16_to_u32_high = - __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_u16_to_u32_low = + __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); + __m256i one_byte_bytemask_u16_to_u32_high = + __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); - const uint32_t mask0 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( - one_or_two_bytes_bytemask_u16_to_u32_low, - one_byte_bytemask_u16_to_u32_low)), - 0); - const uint32_t mask1 = - __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( - one_or_two_bytes_bytemask_u16_to_u32_high, - one_byte_bytemask_u16_to_u32_high)), - 0); + __m256i mask0 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_low, + one_byte_bytemask_u16_to_u32_low)); + __m256i mask1 = __lasx_xvmskltz_h( + __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_high, + one_byte_bytemask_u16_to_u32_high)); + uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); + __m128i utf8_0 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; + mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); - - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_1 = + __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - buf += 8; + mask = __lasx_xvpickve2gr_wu(mask0, 4); + const uint8_t *row2 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle2 = __lsx_vld(row2, 1); + __m128i utf8_2 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); + __lsx_vst(utf8_2, utf8_output, 0); + utf8_output += row2[0]; + + mask = __lasx_xvpickve2gr_wu(mask1, 4); + const uint8_t *row3 = + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] + [0]; + __m128i shuffle3 = __lsx_vld(row3, 1); + __m128i utf8_3 = + __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); + __lsx_vst(utf8_3, utf8_output, 0); + utf8_output += row3[0]; + + buf += 16; } // At least one 32-bit word will produce a surrogate pair in UTF-16 <=> // will produce four UTF-8 bytes. @@ -59334,41 +55862,74 @@ lsx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf8_output)); } -/* end file src/lsx/lsx_convert_utf32_to_utf8.cpp */ +/* end file src/lasx/lasx_convert_utf32_to_utf8.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lsx/lsx_convert_utf32_to_utf16.cpp */ +/* begin file src/lasx/lasx_convert_utf32_to_utf16.cpp */ template std::pair -lsx_convert_utf32_to_utf16(const char32_t *buf, size_t len, - char16_t *utf16_out) { +lasx_convert_utf32_to_utf16(const char32_t *buf, size_t len, + char16_t *utf16_out) { uint16_t *utf16_output = reinterpret_cast(utf16_out); const char32_t *end = buf + len; - __m128i forbidden_bytemask = __lsx_vrepli_h(0); - __m128i v_d800 = lsx_splat_u16(0xd800); - __m128i v_dfff = lsx_splat_u16(0xdfff); - while (end - buf >= 8) { - __m128i in0 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)utf16_output & 0x1F) && buf < end) { + uint32_t word = *buf++; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + if (word >= 0xD800 && word <= 0xDFFF) { + return std::make_pair(nullptr, + reinterpret_cast(utf16_output)); + } + *utf16_output++ = !match_system(big_endian) + ? char16_t(word >> 8 | word << 8) + : char16_t(word); + // buf++; + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return std::make_pair(nullptr, + reinterpret_cast(utf16_output)); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); + low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + // buf++; + } + } + + __m256i forbidden_bytemask = __lasx_xvrepli_h(0); + __m256i v_d800 = lasx_splat_u16(0xd800); + __m256i v_dfff = lasx_splat_u16(0xdfff); + while (end - buf >= 16) { + __m256i in0 = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 32); // Check if no bits set above 16th - if (__lsx_bz_v(__lsx_vpickod_h(in1, in0))) { - __m128i utf16_packed = __lsx_vpickev_h(in1, in0); - forbidden_bytemask = __lsx_vor_v( - __lsx_vand_v( - __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + if (__lasx_xbz_v(__lasx_xvpickod_h(in1, in0))) { + __m256i utf16_packed = + __lasx_xvpermi_d(__lasx_xvpickev_h(in1, in0), 0b11011000); + forbidden_bytemask = __lasx_xvor_v( + __lasx_xvand_v( + __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (!match_system(big_endian)) { - utf16_packed = lsx_swap_bytes(utf16_packed); + if constexpr (!match_system(big_endian)) { + utf16_packed = lasx_swap_bytes(utf16_packed); } - __lsx_vst(utf16_packed, utf16_output, 0); - utf16_output += 8; - buf += 8; + __lasx_xvst(utf16_packed, utf16_output, 0); + utf16_output += 16; + buf += 16; } else { - size_t forward = 3; + size_t forward = 15; size_t k = 0; if (size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1); @@ -59393,7 +55954,7 @@ lsx_convert_utf32_to_utf16(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); @@ -59407,7 +55968,7 @@ lsx_convert_utf32_to_utf16(const char32_t *buf, size_t len, } // check for invalid input - if (__lsx_bnz_v(forbidden_bytemask)) { + if (__lasx_xbnz_v(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf16_output)); } return std::make_pair(buf, reinterpret_cast(utf16_output)); @@ -59415,42 +55976,72 @@ lsx_convert_utf32_to_utf16(const char32_t *buf, size_t len, template std::pair -lsx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, - char16_t *utf16_out) { +lasx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, + char16_t *utf16_out) { uint16_t *utf16_output = reinterpret_cast(utf16_out); const char32_t *start = buf; const char32_t *end = buf + len; - __m128i forbidden_bytemask = __lsx_vrepli_h(0); - __m128i v_d800 = lsx_splat_u16(0xd800); - __m128i v_dfff = lsx_splat_u16(0xdfff); + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)utf16_output & 0x1F) && buf < end) { + uint32_t word = *buf++; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + if (word >= 0xD800 && word <= 0xDFFF) { + return std::make_pair(result(error_code::SURROGATE, buf - start - 1), + reinterpret_cast(utf16_output)); + } + *utf16_output++ = !match_system(big_endian) + ? char16_t(word >> 8 | word << 8) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return std::make_pair(result(error_code::TOO_LARGE, buf - start - 1), + reinterpret_cast(utf16_output)); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); + low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + } - while (end - buf >= 8) { - __m128i in0 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); - // Check if no bits set above 16th - if (__lsx_bz_v(__lsx_vpickod_h(in1, in0))) { - __m128i utf16_packed = __lsx_vpickev_h(in1, in0); + __m256i forbidden_bytemask = __lasx_xvrepli_h(0); + __m256i v_d800 = lasx_splat_u16(0xd800); + __m256i v_dfff = lasx_splat_u16(0xdfff); + while (end - buf >= 16) { + __m256i in0 = __lasx_xvld(reinterpret_cast(buf), 0); + __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 32); - forbidden_bytemask = __lsx_vor_v( - __lsx_vand_v( - __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + // Check if no bits set above 16th + if (__lasx_xbz_v(__lasx_xvpickod_h(in1, in0))) { + __m256i utf16_packed = + __lasx_xvpermi_d(__lasx_xvpickev_h(in1, in0), 0b11011000); + forbidden_bytemask = __lasx_xvor_v( + __lasx_xvand_v( + __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (__lsx_bnz_v(forbidden_bytemask)) { + if (__lasx_xbnz_v(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf16_output)); } - if (!match_system(big_endian)) { - utf16_packed = lsx_swap_bytes(utf16_packed); + if constexpr (!match_system(big_endian)) { + utf16_packed = lasx_swap_bytes(utf16_packed); } - __lsx_vst(utf16_packed, utf16_output, 0); - utf16_output += 8; - buf += 8; + __lasx_xvst(utf16_packed, utf16_output, 0); + utf16_output += 16; + buf += 16; } else { - size_t forward = 3; + size_t forward = 15; size_t k = 0; if (size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1); @@ -59477,7 +56068,7 @@ lsx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); @@ -59493,10 +56084,10 @@ lsx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf16_output)); } -/* end file src/lsx/lsx_convert_utf32_to_utf16.cpp */ +/* end file src/lasx/lasx_convert_utf32_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_BASE64 -/* begin file src/lsx/lsx_base64.cpp */ +/* begin file src/lasx/lasx_base64.cpp */ /** * References and further reading: * @@ -59537,100 +56128,118 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; uint8_t *out = (uint8_t *)dst; - v16u8 shuf; - __m128i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, + v32u8 shuf; + __m256i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, base64_tbl2, base64_tbl3; - if (srclen >= 16) { - shuf = v16u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; - v_fc0fc00 = __lsx_vreplgr2vr_w(uint32_t(0x0fc0fc00)); - v_3f03f0 = __lsx_vreplgr2vr_w(uint32_t(0x003f03f0)); - shift_r = __lsx_vreplgr2vr_w(uint32_t(0x0006000a)); - shift_l = __lsx_vreplgr2vr_w(uint32_t(0x00080004)); - base64_tbl0 = __lsx_vld(lookup_tbl, 0); - base64_tbl1 = __lsx_vld(lookup_tbl, 16); - base64_tbl2 = __lsx_vld(lookup_tbl, 32); - base64_tbl3 = __lsx_vld(lookup_tbl, 48); - } + if (srclen >= 28) { + shuf = v32u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, + 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; + v_fc0fc00 = __lasx_xvreplgr2vr_w(uint32_t(0x0fc0fc00)); + v_3f03f0 = __lasx_xvreplgr2vr_w(uint32_t(0x003f03f0)); + shift_r = __lasx_xvreplgr2vr_w(uint32_t(0x0006000a)); + shift_l = __lasx_xvreplgr2vr_w(uint32_t(0x00080004)); + base64_tbl0 = ____m256i(__lsx_vld(lookup_tbl, 0)); + base64_tbl1 = ____m256i(__lsx_vld(lookup_tbl, 16)); + base64_tbl2 = ____m256i(__lsx_vld(lookup_tbl, 32)); + base64_tbl3 = ____m256i(__lsx_vld(lookup_tbl, 48)); + } size_t i = 0; - for (; i + 52 <= srclen; i += 48) { - __m128i in0 = + for (; i + 100 <= srclen; i += 96) { + __m128i in0_lo = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 0); - __m128i in1 = + __m128i in0_hi = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); - __m128i in2 = + __m128i in1_lo = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 2); - __m128i in3 = + __m128i in1_hi = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 3); + __m128i in2_lo = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 4); + __m128i in2_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 5); + __m128i in3_lo = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 6); + __m128i in3_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 7); - in0 = __lsx_vshuf_b(in0, in0, (__m128i)shuf); - in1 = __lsx_vshuf_b(in1, in1, (__m128i)shuf); - in2 = __lsx_vshuf_b(in2, in2, (__m128i)shuf); - in3 = __lsx_vshuf_b(in3, in3, (__m128i)shuf); + __m256i in0 = lasx_set_q(in0_hi, in0_lo); + __m256i in1 = lasx_set_q(in1_hi, in1_lo); + __m256i in2 = lasx_set_q(in2_hi, in2_lo); + __m256i in3 = lasx_set_q(in3_hi, in3_lo); - __m128i t0_0 = __lsx_vand_v(in0, v_fc0fc00); - __m128i t0_1 = __lsx_vand_v(in1, v_fc0fc00); - __m128i t0_2 = __lsx_vand_v(in2, v_fc0fc00); - __m128i t0_3 = __lsx_vand_v(in3, v_fc0fc00); + in0 = __lasx_xvshuf_b(in0, in0, (__m256i)shuf); + in1 = __lasx_xvshuf_b(in1, in1, (__m256i)shuf); + in2 = __lasx_xvshuf_b(in2, in2, (__m256i)shuf); + in3 = __lasx_xvshuf_b(in3, in3, (__m256i)shuf); - __m128i t1_0 = __lsx_vsrl_h(t0_0, shift_r); - __m128i t1_1 = __lsx_vsrl_h(t0_1, shift_r); - __m128i t1_2 = __lsx_vsrl_h(t0_2, shift_r); - __m128i t1_3 = __lsx_vsrl_h(t0_3, shift_r); + __m256i t0_0 = __lasx_xvand_v(in0, v_fc0fc00); + __m256i t0_1 = __lasx_xvand_v(in1, v_fc0fc00); + __m256i t0_2 = __lasx_xvand_v(in2, v_fc0fc00); + __m256i t0_3 = __lasx_xvand_v(in3, v_fc0fc00); - __m128i t2_0 = __lsx_vand_v(in0, v_3f03f0); - __m128i t2_1 = __lsx_vand_v(in1, v_3f03f0); - __m128i t2_2 = __lsx_vand_v(in2, v_3f03f0); - __m128i t2_3 = __lsx_vand_v(in3, v_3f03f0); + __m256i t1_0 = __lasx_xvsrl_h(t0_0, shift_r); + __m256i t1_1 = __lasx_xvsrl_h(t0_1, shift_r); + __m256i t1_2 = __lasx_xvsrl_h(t0_2, shift_r); + __m256i t1_3 = __lasx_xvsrl_h(t0_3, shift_r); - __m128i t3_0 = __lsx_vsll_h(t2_0, shift_l); - __m128i t3_1 = __lsx_vsll_h(t2_1, shift_l); - __m128i t3_2 = __lsx_vsll_h(t2_2, shift_l); - __m128i t3_3 = __lsx_vsll_h(t2_3, shift_l); + __m256i t2_0 = __lasx_xvand_v(in0, v_3f03f0); + __m256i t2_1 = __lasx_xvand_v(in1, v_3f03f0); + __m256i t2_2 = __lasx_xvand_v(in2, v_3f03f0); + __m256i t2_3 = __lasx_xvand_v(in3, v_3f03f0); - __m128i input0 = __lsx_vor_v(t1_0, t3_0); - __m128i input0_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input0); - __m128i input0_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input0, __lsx_vldi(32))); - __m128i input0_mask = __lsx_vslei_bu(input0, 31); - __m128i input0_result = - __lsx_vbitsel_v(input0_shuf1, input0_shuf0, input0_mask); - __lsx_vst(input0_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i t3_0 = __lasx_xvsll_h(t2_0, shift_l); + __m256i t3_1 = __lasx_xvsll_h(t2_1, shift_l); + __m256i t3_2 = __lasx_xvsll_h(t2_2, shift_l); + __m256i t3_3 = __lasx_xvsll_h(t2_3, shift_l); - __m128i input1 = __lsx_vor_v(t1_1, t3_1); - __m128i input1_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input1); - __m128i input1_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input1, __lsx_vldi(32))); - __m128i input1_mask = __lsx_vslei_bu(input1, 31); - __m128i input1_result = - __lsx_vbitsel_v(input1_shuf1, input1_shuf0, input1_mask); - __lsx_vst(input1_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input0 = __lasx_xvor_v(t1_0, t3_0); + __m256i input0_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input0); + __m256i input0_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input0, __lasx_xvldi(32))); + __m256i input0_mask = __lasx_xvslei_bu(input0, 31); + __m256i input0_result = + __lasx_xvbitsel_v(input0_shuf1, input0_shuf0, input0_mask); + __lasx_xvst(input0_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; - __m128i input2 = __lsx_vor_v(t1_2, t3_2); - __m128i input2_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input2); - __m128i input2_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input2, __lsx_vldi(32))); - __m128i input2_mask = __lsx_vslei_bu(input2, 31); - __m128i input2_result = - __lsx_vbitsel_v(input2_shuf1, input2_shuf0, input2_mask); - __lsx_vst(input2_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input1 = __lasx_xvor_v(t1_1, t3_1); + __m256i input1_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input1); + __m256i input1_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input1, __lasx_xvldi(32))); + __m256i input1_mask = __lasx_xvslei_bu(input1, 31); + __m256i input1_result = + __lasx_xvbitsel_v(input1_shuf1, input1_shuf0, input1_mask); + __lasx_xvst(input1_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; - __m128i input3 = __lsx_vor_v(t1_3, t3_3); - __m128i input3_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input3); - __m128i input3_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, - __lsx_vsub_b(input3, __lsx_vldi(32))); - __m128i input3_mask = __lsx_vslei_bu(input3, 31); - __m128i input3_result = - __lsx_vbitsel_v(input3_shuf1, input3_shuf0, input3_mask); - __lsx_vst(input3_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i input2 = __lasx_xvor_v(t1_2, t3_2); + __m256i input2_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input2); + __m256i input2_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input2, __lasx_xvldi(32))); + __m256i input2_mask = __lasx_xvslei_bu(input2, 31); + __m256i input2_result = + __lasx_xvbitsel_v(input2_shuf1, input2_shuf0, input2_mask); + __lasx_xvst(input2_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; + + __m256i input3 = __lasx_xvor_v(t1_3, t3_3); + __m256i input3_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input3); + __m256i input3_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(input3, __lasx_xvldi(32))); + __m256i input3_mask = __lasx_xvslei_bu(input3, 31); + __m256i input3_result = + __lasx_xvbitsel_v(input3_shuf1, input3_shuf0, input3_mask); + __lasx_xvst(input3_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; } - for (; i + 16 <= srclen; i += 12) { + for (; i + 28 <= srclen; i += 24) { - __m128i in = __lsx_vld(reinterpret_cast(input + i), 0); + __m128i in_lo = __lsx_vld(reinterpret_cast(input + i), 0); + __m128i in_hi = + __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); + + __m256i in = lasx_set_q(in_hi, in_lo); // bytes from groups A, B and C are needed in separate 32-bit lanes // in = [DDDD|CCCC|BBBB|AAAA] @@ -59644,33 +56253,32 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, // [bbbbcccc|ccdddddd|aaaaaabb|bbbbcccc] // ^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ // processed bits - in = __lsx_vshuf_b(in, in, (__m128i)shuf); + in = __lasx_xvshuf_b(in, in, (__m256i)shuf); // unpacking // t0 = [0000cccc|cc000000|aaaaaa00|00000000] - __m128i t0 = __lsx_vand_v(in, v_fc0fc00); + __m256i t0 = __lasx_xvand_v(in, v_fc0fc00); // t1 = [00000000|00cccccc|00000000|00aaaaaa] // ((c >> 6), (a >> 10)) - __m128i t1 = __lsx_vsrl_h(t0, shift_r); + __m256i t1 = __lasx_xvsrl_h(t0, shift_r); // t2 = [00000000|00dddddd|000000bb|bbbb0000] - __m128i t2 = __lsx_vand_v(in, v_3f03f0); + __m256i t2 = __lasx_xvand_v(in, v_3f03f0); // t3 = [00dddddd|00000000|00bbbbbb|00000000] // ((d << 8), (b << 4)) - __m128i t3 = __lsx_vsll_h(t2, shift_l); + __m256i t3 = __lasx_xvsll_h(t2, shift_l); // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 - __m128i indices = __lsx_vor_v(t1, t3); - - __m128i indices_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, indices); - __m128i indices_shuf1 = __lsx_vshuf_b( - base64_tbl3, base64_tbl2, __lsx_vsub_b(indices, __lsx_vldi(32))); - __m128i indices_mask = __lsx_vslei_bu(indices, 31); - __m128i indices_result = - __lsx_vbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + __m256i indices = __lasx_xvor_v(t1, t3); - __lsx_vst(indices_result, reinterpret_cast<__m128i *>(out), 0); - out += 16; + __m256i indices_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, indices); + __m256i indices_shuf1 = __lasx_xvshuf_b( + base64_tbl3, base64_tbl2, __lasx_xvsub_b(indices, __lasx_xvldi(32))); + __m256i indices_mask = __lasx_xvslei_bu(indices, 31); + __m256i indices_result = + __lasx_xvbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + __lasx_xvst(indices_result, reinterpret_cast<__m256i *>(out), 0); + out += 32; } return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, @@ -59694,7 +56302,7 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { tables::base64::thintable_epi8[mask2]}; // we increment by 0x08 the second half of the mask - v4u32 hi = {0, 0, 0x08080808, 0x08080808}; + const v4u32 hi = {0, 0, 0x08080808, 0x08080808}; __m128i shufmask1 = __lsx_vadd_b((__m128i)shufmask, (__m128i)hi); // this is the version "nearly pruned" @@ -59716,115 +56324,111 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { } struct block64 { - __m128i chunks[4]; + __m256i chunks[2]; }; template -static inline uint16_t to_base64_mask(__m128i *src, bool *error) { - const v16u8 ascii_space_tbl = {0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}; +static inline uint32_t to_base64_mask(__m256i *src, bool *error) { + __m256i ascii_space_tbl = + ____m256i((__m128i)v16u8{0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}); // credit: aqrit - /* - '0'(0x30)-'9'(0x39) => delta_values_index = 4 - 'A'(0x41)-'Z'(0x5a) => delta_values_index = 4/5/12(4+8) - 'a'(0x61)-'z'(0x7a) => delta_values_index = 6/7/14(6+8) - '+'(0x2b) => delta_values_index = 3 - '/'(0x2f) => delta_values_index = 2+8 = 10 - '-'(0x2d) => delta_values_index = 2+8 = 10 - '_'(0x5f) => delta_values_index = 5+8 = 13 - */ - v16u8 delta_asso; + __m256i delta_asso; if (default_or_url) { - delta_asso = v16u8{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16}; + delta_asso = + ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x11, 0x0, 0x16}); } else { - delta_asso = v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, - 0x0, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}; + delta_asso = + ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, + 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}); } - v16i8 delta_values; + __m256i delta_values; if (default_or_url) { - delta_values = - v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), - int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), + int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}); } else if (base64_url) { - delta_values = - v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), - int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), + int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}); } else { - delta_values = - v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), - int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}; + delta_values = ____m256i( + (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), + int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}); } - v16u8 check_asso; + __m256i check_asso; if (default_or_url) { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x0E, 0x0B, 0x06}); + } else if (base64_url) { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x06, 0x0B, 0x12}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x06, 0x0B, 0x12}); } else { - check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F}; + check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, + 0x0B, 0x0B, 0x0B, 0x0F}); } - v16i8 check_values; + __m256i check_values; if (default_or_url) { - check_values = - v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), - int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}; + + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), + int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}); } else if (base64_url) { - check_values = v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), - int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}; + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), + int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}); } else { - check_values = - v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), - int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}; + check_values = ____m256i( + (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), + int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}); } - const __m128i shifted = __lsx_vsrli_b(*src, 3); - __m128i asso_index = __lsx_vand_v(*src, __lsx_vldi(0xF)); - const __m128i delta_hash = - __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)delta_asso, (__m128i)delta_asso, - (__m128i)asso_index), - shifted); - const __m128i check_hash = - __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)check_asso, (__m128i)check_asso, - (__m128i)asso_index), - shifted); + __m256i shifted = __lasx_xvsrli_b(*src, 3); + __m256i asso_index = __lasx_xvand_v(*src, __lasx_xvldi(0xF)); + __m256i delta_hash = __lasx_xvavgr_bu( + __lasx_xvshuf_b(delta_asso, delta_asso, asso_index), shifted); + __m256i check_hash = __lasx_xvavgr_bu( + __lasx_xvshuf_b(check_asso, check_asso, asso_index), shifted); - const __m128i out = - __lsx_vsadd_b(__lsx_vshuf_b((__m128i)delta_values, (__m128i)delta_values, - (__m128i)delta_hash), - *src); - const __m128i chk = - __lsx_vsadd_b(__lsx_vshuf_b((__m128i)check_values, (__m128i)check_values, - (__m128i)check_hash), - *src); - unsigned int mask = __lsx_vpickve2gr_hu(__lsx_vmskltz_b(chk), 0); + __m256i out = __lasx_xvsadd_b( + __lasx_xvshuf_b(delta_values, delta_values, delta_hash), *src); + __m256i chk = __lasx_xvsadd_b( + __lasx_xvshuf_b(check_values, check_values, check_hash), *src); + __m256i chk_ltz = __lasx_xvmskltz_b(chk); + unsigned int mask = __lasx_xvpickve2gr_wu(chk_ltz, 0); + mask = mask | (__lsx_vpickve2gr_hu(lasx_extracti128_hi(chk_ltz), 0) << 16); if (mask) { - __m128i ascii_space = __lsx_vseq_b(__lsx_vshuf_b((__m128i)ascii_space_tbl, - (__m128i)ascii_space_tbl, - (__m128i)asso_index), - *src); - *error |= - (mask != __lsx_vpickve2gr_hu(__lsx_vmskltz_b((__m128i)ascii_space), 0)); + __m256i ascii_space = __lasx_xvseq_b( + __lasx_xvshuf_b(ascii_space_tbl, ascii_space_tbl, asso_index), *src); + __m256i ascii_space_ltz = __lasx_xvmskltz_b(ascii_space); + unsigned int ascii_space_mask = __lasx_xvpickve2gr_wu(ascii_space_ltz, 0); + ascii_space_mask = + ascii_space_mask | + (__lsx_vpickve2gr_hu(lasx_extracti128_hi(ascii_space_ltz), 0) << 16); + *error |= (mask != ascii_space_mask); } *src = out; - return (uint16_t)mask; + return (uint32_t)mask; } template @@ -59834,18 +56438,12 @@ static inline uint64_t to_base64_mask(block64 *b, bool *error) { to_base64_mask(&b->chunks[0], error); uint64_t m1 = to_base64_mask(&b->chunks[1], error); - uint64_t m2 = - to_base64_mask(&b->chunks[2], error); - uint64_t m3 = - to_base64_mask(&b->chunks[3], error); - return m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); + return m0 | (m1 << 32); } static inline void copy_block(block64 *b, char *output) { - __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output), 0); - __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output), 16); - __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output), 32); - __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output), 48); + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output), 32); } static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { @@ -59853,84 +56451,150 @@ static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { uint64_t count = __lsx_vpickve2gr_d(__lsx_vpcnt_h(__lsx_vreplgr2vr_d(nmask)), 0); uint16_t *count_ptr = (uint16_t *)&count; - compress(b->chunks[0], uint16_t(mask), output); - compress(b->chunks[1], uint16_t(mask >> 16), output + count_ptr[0]); - compress(b->chunks[2], uint16_t(mask >> 32), + compress(lasx_extracti128_lo(b->chunks[0]), uint16_t(mask), output); + compress(lasx_extracti128_hi(b->chunks[0]), uint16_t(mask >> 16), + output + count_ptr[0]); + compress(lasx_extracti128_lo(b->chunks[1]), uint16_t(mask >> 32), output + count_ptr[0] + count_ptr[1]); - compress(b->chunks[3], uint16_t(mask >> 48), + compress(lasx_extracti128_hi(b->chunks[1]), uint16_t(mask >> 48), output + count_ptr[0] + count_ptr[1] + count_ptr[2]); return count_ones(nmask); } +template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } + +inline size_t compress_block_single(block64 *b, uint64_t mask, char *output) { + const size_t pos64 = trailing_zeroes(mask); + const int8_t pos = pos64 & 0xf; + + // Predefine the index vector + const v16u8 v1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + switch (pos64 >> 4) { + case 0b00: { + const __m128i lane0 = lasx_extracti128_lo(b->chunks[0]); + const __m128i lane1 = lasx_extracti128_hi(b->chunks[0]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); // v1 > v0 + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane0, lane0, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(lane1, reinterpret_cast<__m128i *>(output + 1 * 16 - 1), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output + 2 * 16 - 1), + 0); + } break; + case 0b01: { + const __m128i lane0 = lasx_extracti128_lo(b->chunks[0]); + const __m128i lane1 = lasx_extracti128_hi(b->chunks[0]); + __lsx_vst(lane0, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane1, lane1, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output + 2 * 16 - 1), + 0); + } break; + case 0b10: { + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output + 0 * 16), 0); + + const __m128i lane2 = lasx_extracti128_lo(b->chunks[1]); + const __m128i lane3 = lasx_extracti128_hi(b->chunks[1]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane2, lane2, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 2 * 16), 0); + __lsx_vst(lane3, reinterpret_cast<__m128i *>(output + 3 * 16 - 1), 0); + } break; + case 0b11: { + __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output + 0 * 16), 0); + __lsx_vst(lasx_extracti128_lo(b->chunks[1]), + reinterpret_cast<__m128i *>(output + 2 * 16), 0); + + const __m128i lane3 = lasx_extracti128_hi(b->chunks[1]); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(lane3, lane3, sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 3 * 16), 0); + } break; + } + return 63; +} + // The caller of this function is responsible to ensure that there are 64 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char *src) { - b->chunks[0] = __lsx_vld(reinterpret_cast(src), 0); - b->chunks[1] = __lsx_vld(reinterpret_cast(src), 16); - b->chunks[2] = __lsx_vld(reinterpret_cast(src), 32); - b->chunks[3] = __lsx_vld(reinterpret_cast(src), 48); + b->chunks[0] = __lasx_xvld(reinterpret_cast(src), 0); + b->chunks[1] = __lasx_xvld(reinterpret_cast(src), 32); } // The caller of this function is responsible to ensure that there are 128 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char16_t *src) { - __m128i m1 = __lsx_vld(reinterpret_cast(src), 0); - __m128i m2 = __lsx_vld(reinterpret_cast(src), 16); - __m128i m3 = __lsx_vld(reinterpret_cast(src), 32); - __m128i m4 = __lsx_vld(reinterpret_cast(src), 48); - __m128i m5 = __lsx_vld(reinterpret_cast(src), 64); - __m128i m6 = __lsx_vld(reinterpret_cast(src), 80); - __m128i m7 = __lsx_vld(reinterpret_cast(src), 96); - __m128i m8 = __lsx_vld(reinterpret_cast(src), 112); - b->chunks[0] = __lsx_vssrlni_bu_h(m2, m1, 0); - b->chunks[1] = __lsx_vssrlni_bu_h(m4, m3, 0); - b->chunks[2] = __lsx_vssrlni_bu_h(m6, m5, 0); - b->chunks[3] = __lsx_vssrlni_bu_h(m8, m7, 0); + __m256i m1 = __lasx_xvld(reinterpret_cast(src), 0); + __m256i m2 = __lasx_xvld(reinterpret_cast(src), 32); + __m256i m3 = __lasx_xvld(reinterpret_cast(src), 64); + __m256i m4 = __lasx_xvld(reinterpret_cast(src), 96); + b->chunks[0] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m2, m1, 0), 0b11011000); + b->chunks[1] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m4, m3, 0), 0b11011000); } -static inline void base64_decode(char *out, __m128i str) { - __m128i t0 = __lsx_vor_v( - __lsx_vslli_w(str, 26), - __lsx_vslli_w(__lsx_vand_v(str, lsx_splat_u32(0x0000FF00)), 12)); - __m128i t1 = __lsx_vsrli_w(__lsx_vand_v(str, lsx_splat_u32(0x003F0000)), 2); - __m128i t2 = __lsx_vor_v(t0, t1); - __m128i t3 = __lsx_vor_v(t2, __lsx_vsrli_w(str, 16)); - const v16u8 pack_shuffle = {3, 2, 1, 7, 6, 5, 11, 10, - 9, 15, 14, 13, 0, 0, 0, 0}; - t3 = __lsx_vshuf_b(t3, t3, (__m128i)pack_shuffle); +static inline void base64_decode(char *out, __m256i str) { + __m256i t0 = __lasx_xvor_v( + __lasx_xvslli_w(str, 26), + __lasx_xvslli_w(__lasx_xvand_v(str, lasx_splat_u32(0x0000ff00)), 12)); + __m256i t1 = + __lasx_xvsrli_w(__lasx_xvand_v(str, lasx_splat_u32(0x003f0000)), 2); + __m256i t2 = __lasx_xvor_v(t0, t1); + __m256i t3 = __lasx_xvor_v(t2, __lasx_xvsrli_w(str, 16)); + __m256i pack_shuffle = ____m256i( + (__m128i)v16u8{3, 2, 1, 7, 6, 5, 11, 10, 9, 15, 14, 13, 0, 0, 0, 0}); + t3 = __lasx_xvshuf_b(t3, t3, (__m256i)pack_shuffle); // Store the output: - // we only need 12. - __lsx_vstelm_d(t3, out, 0, 0); - __lsx_vstelm_w(t3, out + 8, 0, 2); + __lsx_vst(lasx_extracti128_lo(t3), out, 0); + __lsx_vst(lasx_extracti128_hi(t3), out, 12); } // decode 64 bytes and output 48 bytes static inline void base64_decode_block(char *out, const char *src) { - base64_decode(out, __lsx_vld(reinterpret_cast(src), 0)); - base64_decode(out + 12, - __lsx_vld(reinterpret_cast(src), 16)); + base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); base64_decode(out + 24, - __lsx_vld(reinterpret_cast(src), 32)); - base64_decode(out + 36, - __lsx_vld(reinterpret_cast(src), 48)); + __lasx_xvld(reinterpret_cast(src), 32)); } + static inline void base64_decode_block_safe(char *out, const char *src) { - base64_decode_block(out, src); + base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); + alignas(32) char buffer[32]; + base64_decode(buffer, + __lasx_xvld(reinterpret_cast(src), 32)); + std::memcpy(out + 24, buffer, 24); } + static inline void base64_decode_block(char *out, block64 *b) { base64_decode(out, b->chunks[0]); - base64_decode(out + 12, b->chunks[1]); - base64_decode(out + 24, b->chunks[2]); - base64_decode(out + 36, b->chunks[3]); + base64_decode(out + 24, b->chunks[1]); } static inline void base64_decode_block_safe(char *out, block64 *b) { - base64_decode_block(out, b); + base64_decode(out, b->chunks[0]); + alignas(32) char buffer[32]; + base64_decode(buffer, b->chunks[1]); + std::memcpy(out + 24, buffer, 24); } template + typename chartype> full_result -compress_decode_base64(char *dst, const char_type *src, size_t srclen, +compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_options options, last_chunk_handling_options last_chunk_options) { const uint8_t *to_base64 = @@ -59948,15 +56612,19 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } return {SUCCESS, full_input_length, 0}; } - const char_type *const srcinit = src; + char *end_of_safe_64byte_zone = + (srclen + 3) / 4 * 3 >= 63 ? dst + (srclen + 3) / 4 * 3 - 63 : dst; + + const chartype *const srcinit = src; const char *const dstinit = dst; - const char_type *const srcend = src + srclen; + const chartype *const srcend = src + srclen; - constexpr size_t block_size = 10; + constexpr size_t block_size = 6; + static_assert(block_size >= 2, "block_size must be at least two"); char buffer[block_size * 64]; char *bufferptr = buffer; if (srclen >= 64) { - const char_type *const srcend64 = src + srclen - 64; + const chartype *const srcend64 = src + srclen - 64; while (src <= srcend64) { block64 b; load_block(&b, src); @@ -59964,48 +56632,56 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, bool error = false; uint64_t badcharmask = to_base64_mask(&b, &error); - if (badcharmask) { - if (error && !ignore_garbage) { - src -= 64; - while (src < srcend && scalar::base64::is_eight_byte(*src) && - to_base64[uint8_t(*src)] <= 64) { - src++; - } - if (src < srcend) { - // should never happen - } - return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; + if (error && !ignore_garbage) { + src -= 64; + while (src < srcend && scalar::base64::is_eight_byte(*src) && + to_base64[uint8_t(*src)] <= 64) { + src++; } + return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; } - if (badcharmask != 0) { - // optimization opportunity: check for simple masks like those made of - // continuous 1s followed by continuous 0s. And masks containing a - // single bad character. - bufferptr += compress_block(&b, badcharmask, bufferptr); - } else { - // optimization opportunity: if bufferptr == buffer and mask == 0, we - // can avoid the call to compress_block and decode directly. + if (is_power_of_two(badcharmask)) { + bufferptr += compress_block_single(&b, badcharmask, bufferptr); + } else { + bufferptr += compress_block(&b, badcharmask, bufferptr); + } + } else if (bufferptr != buffer) { copy_block(&b, bufferptr); bufferptr += 64; + } else { + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, &b); + } else { + base64_decode_block(dst, &b); + } + dst += 48; } if (bufferptr >= (block_size - 1) * 64 + buffer) { - for (size_t i = 0; i < (block_size - 1); i++) { + for (size_t i = 0; i < (block_size - 2); i++) { base64_decode_block(dst, buffer + i * 64); dst += 48; } + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, buffer + (block_size - 2) * 64); + } else { + base64_decode_block(dst, buffer + (block_size - 2) * 64); + } + dst += 48; std::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } } } + char *buffer_start = buffer; // Optimization note: if this is almost full, then it is worth our // time, otherwise, we should just decode directly. int last_block = (int)((bufferptr - buffer_start) % 64); if (last_block != 0 && srcend - src + last_block >= 64) { + while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { uint8_t val = to_base64[uint8_t(*src)]; *bufferptr = char(val); @@ -60020,7 +56696,11 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { - base64_decode_block(dst, buffer_start); + if (dst >= end_of_safe_64byte_zone) { + base64_decode_block_safe(dst, buffer_start); + } else { + base64_decode_block(dst, buffer_start); + } dst += 48; } if ((bufferptr - buffer_start) % 64 != 0) { @@ -60030,6 +56710,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lasx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 4); @@ -60042,6 +56723,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lasx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 3); @@ -60101,22 +56783,24 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, } return {SUCCESS, srclen, size_t(dst - dstinit)}; } -/* end file src/lsx/lsx_base64.cpp */ -/* begin file src/lsx/lsx_find.cpp */ +/* end file src/lasx/lasx_base64.cpp */ +/* begin file src/lasx/lasx_find.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, char character) noexcept { if (start >= end) return end; - const int step = 16; - __m128i char_vec = __lsx_vreplgr2vr_b(static_cast(character)); + const int step = 32; + __m256i char_vec = __lasx_xvreplgr2vr_b(static_cast(character)); while (end - start >= step) { - __m128i data = __lsx_vld(reinterpret_cast(start), 0); - __m128i cmp = __lsx_vseq_b(data, char_vec); - if (__lsx_bnz_v(cmp)) { - uint16_t mask = - static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + __m256i data = __lasx_xvld(reinterpret_cast(start), 0); + __m256i cmp = __lasx_xvseq_b(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); return start + trailing_zeroes(mask); } @@ -60139,15 +56823,17 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, if (start >= end) return end; - const int step = 8; - __m128i char_vec = __lsx_vreplgr2vr_h(static_cast(character)); + const int step = 16; + __m256i char_vec = __lasx_xvreplgr2vr_h(static_cast(character)); while (end - start >= step) { - __m128i data = __lsx_vld(reinterpret_cast(start), 0); - __m128i cmp = __lsx_vseq_h(data, char_vec); - if (__lsx_bnz_v(cmp)) { - uint16_t mask = - static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); + __m256i data = __lasx_xvld(reinterpret_cast(start), 0); + __m256i cmp = __lasx_xvseq_h(data, char_vec); + if (__lasx_xbnz_v(cmp)) { + __m256i res = __lasx_xvmsknz_b(cmp); + uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); + uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); + uint32_t mask = (mask0 | (mask1 << 16)); return start + trailing_zeroes(mask) / 2; } @@ -60163,16 +56849,16 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, return end; } -/* end file src/lsx/lsx_find.cpp */ +/* end file src/lasx/lasx_find.cpp */ #endif // SIMDUTF_FEATURE_BASE64 } // namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* begin file src/generic/buf_block_reader.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { // Walks through a buffer in block-sized increments, loading the last part with @@ -60203,40 +56889,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -60278,13 +56930,13 @@ simdutf_really_inline void buf_block_reader::advance() { } } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/buf_block_reader.h */ #if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_validation { @@ -60478,10 +57130,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -60504,12 +57156,12 @@ struct utf8_checker { using utf8_validation::utf8_checker; } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ /* begin file src/generic/utf8_validation/utf8_validator.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_validation { @@ -60590,33 +57242,17 @@ result generic_validate_utf8_with_errors(const char *input, size_t length) { } // namespace utf8_validation } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_validation/utf8_validator.h */ #endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_ASCII /* begin file src/generic/ascii_validation.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -60643,9 +57279,24 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/ascii_validation.h */ #endif // SIMDUTF_FEATURE_ASCII @@ -60654,7 +57305,7 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { // transcoding from UTF-8 to Latin 1 /* begin file src/generic/utf8_to_latin1/utf8_to_latin1.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_latin1 { using namespace simd; @@ -60799,10 +57450,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -60888,10 +57539,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -60967,12 +57618,12 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_latin1 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_to_latin1/utf8_to_latin1.h */ /* begin file src/generic/utf8_to_latin1/valid_utf8_to_latin1.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_latin1 { using namespace simd; @@ -61046,17 +57697,16 @@ simdutf_really_inline size_t convert_valid(const char *in, size_t size, } // namespace utf8_to_latin1 } // namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf // namespace simdutf /* end file src/generic/utf8_to_latin1/valid_utf8_to_latin1.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 - #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 // transcoding from UTF-8 to UTF-16 /* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_utf16 { @@ -61127,12 +57777,12 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace utf8_to_utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ /* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_utf16 { using namespace simd; @@ -61291,10 +57941,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -61379,10 +58029,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -61461,12 +58111,12 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ /* begin file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8 { @@ -61516,16 +58166,54 @@ simdutf_really_inline size_t utf16_length_from_utf8_bytemask(const char *in, } // namespace utf8 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + // transcoding from UTF-16 to UTF-8 +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace lasx { +namespace { +namespace utf16_to_utf8 { +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace lasx +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 // transcoding from UTF-8 to UTF-32 /* begin file src/generic/utf8_to_utf32/valid_utf8_to_utf32.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_utf32 { @@ -61564,12 +58252,12 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace utf8_to_utf32 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_to_utf32/valid_utf8_to_utf32.h */ /* begin file src/generic/utf8_to_utf32/utf8_to_utf32.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8_to_utf32 { using namespace simd; @@ -61727,10 +58415,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -61814,10 +58502,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -61884,7 +58572,7 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_utf32 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8_to_utf32/utf8_to_utf32.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -61892,7 +58580,7 @@ struct validating_transcoder { #if SIMDUTF_FEATURE_UTF8 /* begin file src/generic/utf8.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf8 { @@ -61910,8 +58598,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -61981,7 +58669,7 @@ simdutf_really_inline size_t utf16_length_from_utf8(const char *in, } // namespace utf8 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf8.h */ #endif // SIMDUTF_FEATURE_UTF8 @@ -61989,7 +58677,7 @@ simdutf_really_inline size_t utf16_length_from_utf8(const char *in, #if SIMDUTF_FEATURE_UTF16 /* begin file src/generic/utf16/count_code_points_bytemask.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { @@ -62004,16 +58692,16 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t pos = 0; size_t count = 0; - constexpr size_t max_itertions = 65535; + constexpr size_t max_iterations = 65535; const auto one = vector_u16::splat(1); const auto zero = vector_u16::zero(); - size_t itertion = 0; + size_t iteration = 0; auto counters = zero; for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(in + pos); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } @@ -62026,15 +58714,15 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, counters += t2; - itertion += 1; - if (itertion == max_itertions) { + iteration += 1; + if (iteration == max_iterations) { count += counters.sum(); counters = zero; - itertion = 0; + iteration = 0; } } - if (itertion > 0) { + if (iteration > 0) { count += counters.sum(); } @@ -62044,12 +58732,12 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf16/count_code_points_bytemask.h */ /* begin file src/generic/utf16/change_endianness.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { @@ -62070,12 +58758,12 @@ change_endianness_utf16(const char16_t *in, size_t size, char16_t *output) { } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf16/change_endianness.h */ /* begin file src/generic/utf16/utf8_length_from_utf16_bytemask.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { @@ -62103,7 +58791,7 @@ simdutf_really_inline size_t utf8_length_from_utf16_bytemask(const char16_t *in, for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -62192,7 +58880,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { for (; pos < (size - 1) / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -62213,7 +58901,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { any_surrogates = true; auto input_next = vector_u16::load(reinterpret_cast(in + pos + 1)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_next = input_next.swap_bytes(); } @@ -62271,12 +58959,12 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf16/utf8_length_from_utf16_bytemask.h */ /* begin file src/generic/utf16/utf32_length_from_utf16.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { @@ -62288,12 +58976,12 @@ simdutf_really_inline size_t utf32_length_from_utf16(const char16_t *in, } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf16/utf32_length_from_utf16.h */ /* begin file src/generic/utf16/to_well_formed.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { @@ -62310,11 +58998,11 @@ namespace utf16 { template simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; - }; using vector_u16 = simd16; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); + }; const auto lookback = vector_u16::load(in - 1); const auto block = vector_u16::load(in); @@ -62324,7 +59012,6 @@ simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { const auto lb_is_high = lb_masked == swap_if_needed(0xd800); const auto block_is_low = block_masked == swap_if_needed(0xdc00); - const auto illseq = lb_is_high ^ block_is_low; if (!illseq.is_zero()) { /* compute the cause of the illegal sequencing */ @@ -62384,7 +59071,7 @@ void to_well_formed(const char16_t *in, size_t n, char16_t *out) { } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf16/to_well_formed.h */ #endif // SIMDUTF_FEATURE_UTF16 @@ -62392,7 +59079,7 @@ void to_well_formed(const char16_t *in, size_t n, char16_t *out) { #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/validate_utf16.h */ namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf16 { /* @@ -62528,7 +59215,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, for (; pos < size / 32 * 32; pos += 32) { simd16x32 input_vec( reinterpret_cast(input + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_vec.swap_bytes(); } uint64_t matches = input_vec.lteq(uint16_t(0x7f)); @@ -62541,7 +59228,8 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, // Scalar tail while (pos < size) { - char16_t v = big_endian ? scalar::u16_swap_bytes(input[pos]) : input[pos]; + + char16_t v = scalar::utf16::swap_if_needed(input[pos]); if (v > 0x7F) { return result(error_code::TOO_LARGE, pos); } @@ -62552,7 +59240,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, } // namespace utf16 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/validate_utf16.h */ #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING @@ -62562,7 +59250,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, #include namespace simdutf { -namespace lsx { +namespace lasx { namespace { namespace utf32 { @@ -62693,16 +59381,86 @@ simdutf_really_inline size_t utf8_length_from_utf32(const char32_t *input, } // namespace utf32 } // unnamed namespace -} // namespace lsx +} // namespace lasx } // namespace simdutf /* end file src/generic/utf32.h */ #endif // SIMDUTF_FEATURE_UTF32 +#if SIMDUTF_FEATURE_BASE64 +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace lasx { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + // simd16x32::to_bitmask sets two bits per matching 16-bit lane, so the + // vectorized loop counted each unit twice. + count /= 2; + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace lasx +} // namespace simdutf +/* end file src/generic/base64lengths.h */ +#endif // SIMDUTF_FEATURE_BASE64 // // Implementation-specific overrides // namespace simdutf { -namespace lsx { +namespace lasx { #if SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused int @@ -62736,33 +59494,33 @@ implementation::detect_encodings(const char *input, #if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return lsx::utf8_validation::generic_validate_utf8(buf, len); + return lasx::utf8_validation::generic_validate_utf8(buf, len); } #endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused result implementation::validate_utf8_with_errors( const char *buf, size_t len) const noexcept { - return lsx::utf8_validation::generic_validate_utf8_with_errors(buf, len); + return lasx::utf8_validation::generic_validate_utf8_with_errors(buf, len); } #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_ASCII simdutf_warn_unused bool implementation::validate_ascii(const char *buf, size_t len) const noexcept { - return lsx::ascii_validation::generic_validate_ascii(buf, len); + return lasx::ascii_validation::generic_validate_ascii(buf, len); } simdutf_warn_unused result implementation::validate_ascii_with_errors( const char *buf, size_t len) const noexcept { - return lsx::ascii_validation::generic_validate_ascii_with_errors(buf, len); + return lasx::ascii_validation::generic_validate_ascii_with_errors(buf, len); } #endif // SIMDUTF_FEATURE_ASCII #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII simdutf_warn_unused bool implementation::validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept { - return lsx::utf16::validate_utf16_as_ascii_with_errors( + return lasx::utf16::validate_utf16_as_ascii_with_errors( buf, len) .error == SUCCESS; } @@ -62770,8 +59528,8 @@ implementation::validate_utf16le_as_ascii(const char16_t *buf, simdutf_warn_unused bool implementation::validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept { - return lsx::utf16::validate_utf16_as_ascii_with_errors(buf, - len) + return lasx::utf16::validate_utf16_as_ascii_with_errors(buf, + len) .error == SUCCESS; } #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII @@ -62784,8 +59542,7 @@ implementation::validate_utf16le(const char16_t *buf, return true; } const auto res = - lsx::utf16::validate_utf16_with_errors(buf, len); - + lasx::utf16::validate_utf16_with_errors(buf, len); if (res.is_err()) { return false; } @@ -62807,9 +59564,9 @@ implementation::validate_utf16be(const char16_t *buf, // empty input is valid. protected the implementation from nullptr. return true; } - const auto res = - lsx::utf16::validate_utf16_with_errors(buf, len); + const auto res = + lasx::utf16::validate_utf16_with_errors(buf, len); if (res.is_err()) { return false; } @@ -62828,7 +59585,7 @@ simdutf_warn_unused result implementation::validate_utf16le_with_errors( return result(error_code::SUCCESS, 0); } const result res = - lsx::utf16::validate_utf16_with_errors(buf, len); + lasx::utf16::validate_utf16_with_errors(buf, len); if (res.count != len) { const result scalar_res = scalar::utf16::validate_with_errors( @@ -62845,7 +59602,7 @@ simdutf_warn_unused result implementation::validate_utf16be_with_errors( return result(error_code::SUCCESS, 0); } const result res = - lsx::utf16::validate_utf16_with_errors(buf, len); + lasx::utf16::validate_utf16_with_errors(buf, len); if (res.count != len) { const result scalar_res = scalar::utf16::validate_with_errors(buf + res.count, @@ -62858,12 +59615,12 @@ simdutf_warn_unused result implementation::validate_utf16be_with_errors( void implementation::to_well_formed_utf16le(const char16_t *input, size_t len, char16_t *output) const noexcept { - utf16::to_well_formed(input, len, output); + return utf16::to_well_formed(input, len, output); } void implementation::to_well_formed_utf16be(const char16_t *input, size_t len, char16_t *output) const noexcept { - utf16::to_well_formed(input, len, output); + return utf16::to_well_formed(input, len, output); } #endif // SIMDUTF_FEATURE_UTF16 @@ -62874,7 +59631,7 @@ implementation::validate_utf32(const char32_t *buf, size_t len) const noexcept { // empty input is valid. protected the implementation from nullptr. return true; } - const char32_t *tail = lsx_validate_utf32le(buf, len); + const char32_t *tail = lasx_validate_utf32le(buf, len); if (tail) { return scalar::utf32::validate(tail, len - (tail - buf)); } else { @@ -62889,7 +59646,7 @@ simdutf_warn_unused result implementation::validate_utf32_with_errors( if (simdutf_unlikely(len == 0)) { return result(error_code::SUCCESS, 0); } - result res = lsx_validate_utf32le_with_errors(buf, len); + result res = lasx_validate_utf32le_with_errors(buf, len); if (res.count != len) { result scalar_res = scalar::utf32::validate_with_errors(buf + res.count, len - res.count); @@ -62904,7 +59661,7 @@ simdutf_warn_unused result implementation::validate_utf32_with_errors( simdutf_warn_unused size_t implementation::convert_latin1_to_utf8( const char *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lsx_convert_latin1_to_utf8(buf, len, utf8_output); + lasx_convert_latin1_to_utf8(buf, len, utf8_output); size_t converted_chars = ret.second - utf8_output; if (ret.first != buf + len) { @@ -62920,7 +59677,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf8( simdutf_warn_unused size_t implementation::convert_latin1_to_utf16le( const char *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lsx_convert_latin1_to_utf16le(buf, len, utf16_output); + lasx_convert_latin1_to_utf16le(buf, len, utf16_output); size_t converted_chars = ret.second - utf16_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = @@ -62934,7 +59691,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf16le( simdutf_warn_unused size_t implementation::convert_latin1_to_utf16be( const char *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lsx_convert_latin1_to_utf16be(buf, len, utf16_output); + lasx_convert_latin1_to_utf16be(buf, len, utf16_output); size_t converted_chars = ret.second - utf16_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = @@ -62950,7 +59707,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf16be( simdutf_warn_unused size_t implementation::convert_latin1_to_utf32( const char *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lsx_convert_latin1_to_utf32(buf, len, utf32_output); + lasx_convert_latin1_to_utf32(buf, len, utf32_output); size_t converted_chars = ret.second - utf32_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = scalar::latin1_to_utf32::convert( @@ -62964,19 +59721,117 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf32( #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 simdutf_warn_unused size_t implementation::convert_utf8_to_latin1( const char *buf, size_t len, char *latin1_output) const noexcept { + size_t pos = 0; + char *output_start{latin1_output}; + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)latin1_output & 0x1F) && pos < len) { + if (buf[pos] & 0x80) { + if (pos + 1 >= len) + return 0; + if ((buf[pos] & 0b11100000) == 0b11000000) { + if ((buf[pos + 1] & 0b11000000) != 0b10000000) + return 0; + uint32_t code_point = + (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); + if (code_point < 0x80 || 0xFF < code_point) { + return 0; + } + *latin1_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } else { + *latin1_output++ = char(buf[pos]); + pos++; + } + } + size_t convert_size = latin1_output - output_start; + if (pos == len) + return convert_size; utf8_to_latin1::validating_transcoder converter; - return converter.convert(buf, len, latin1_output); + size_t convert_result = + converter.convert(buf + pos, len - pos, latin1_output); + return convert_result ? convert_size + convert_result : 0; } simdutf_warn_unused result implementation::convert_utf8_to_latin1_with_errors( const char *buf, size_t len, char *latin1_output) const noexcept { + size_t pos = 0; + char *output_start{latin1_output}; + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)latin1_output & 0x1F) && pos < len) { + if (buf[pos] & 0x80) { + if ((buf[pos] & 0b11100000) == 0b11000000) { + if (pos + 1 >= len) + return result(error_code::TOO_SHORT, pos); + if ((buf[pos + 1] & 0b11000000) != 0b10000000) + return result(error_code::TOO_SHORT, pos); + uint32_t code_point = + (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); + if (code_point < 0x80) + return result(error_code::OVERLONG, pos); + if (0xFF < code_point) + return result(error_code::TOO_LARGE, pos); + *latin1_output++ = char(code_point); + pos += 2; + } else if ((buf[pos] & 0b11110000) == 0b11100000) { + return result(error_code::TOO_LARGE, pos); + } else if ((buf[pos] & 0b11111000) == 0b11110000) { + return result(error_code::TOO_LARGE, pos); + } else { + if ((buf[pos] & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + return result(error_code::HEADER_BITS, pos); + } + } else { + *latin1_output++ = char(buf[pos]); + pos++; + } + } + size_t convert_size = latin1_output - output_start; + if (pos == len) + return result(error_code::SUCCESS, convert_size); + utf8_to_latin1::validating_transcoder converter; - return converter.convert_with_errors(buf, len, latin1_output); + result res = + converter.convert_with_errors(buf + pos, len - pos, latin1_output); + return res.error ? result(res.error, res.count + pos) + : result(res.error, res.count + convert_size); } simdutf_warn_unused size_t implementation::convert_valid_utf8_to_latin1( const char *buf, size_t len, char *latin1_output) const noexcept { - return lsx::utf8_to_latin1::convert_valid(buf, len, latin1_output); + size_t pos = 0; + char *output_start{latin1_output}; + // Performance degradation when memory address is not 32-byte aligned + while (((uint64_t)latin1_output & 0x1F) && pos < len) { + if (buf[pos] & 0x80) { + if (pos + 1 >= len) + break; + if ((buf[pos] & 0b11100000) == 0b11000000) { + if ((buf[pos + 1] & 0b11000000) != 0b10000000) + return 0; + uint32_t code_point = + (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); + *latin1_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } else { + *latin1_output++ = char(buf[pos]); + pos++; + } + } + size_t convert_size = latin1_output - output_start; + if (pos == len) + return convert_size; + + size_t convert_result = + lasx::utf8_to_latin1::convert_valid(buf + pos, len - pos, latin1_output); + return convert_result ? convert_size + convert_result : 0; } #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -63042,7 +59897,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf32( simdutf_warn_unused size_t implementation::convert_utf16le_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_latin1(buf, len, latin1_output); + lasx_convert_utf16_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -63063,7 +59918,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_latin1( simdutf_warn_unused size_t implementation::convert_utf16be_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_latin1(buf, len, latin1_output); + lasx_convert_utf16_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -63085,7 +59940,7 @@ simdutf_warn_unused result implementation::convert_utf16le_to_latin1_with_errors( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_latin1_with_errors( + lasx_convert_utf16_to_latin1_with_errors( buf, len, latin1_output); if (ret.first.error) { return ret.first; @@ -63112,8 +59967,8 @@ simdutf_warn_unused result implementation::convert_utf16be_to_latin1_with_errors( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_latin1_with_errors(buf, len, - latin1_output); + lasx_convert_utf16_to_latin1_with_errors(buf, len, + latin1_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -63152,7 +60007,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_latin1( simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_utf8(buf, len, utf8_output); + lasx_convert_utf16_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -63172,7 +60027,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_utf8(buf, len, utf8_output); + lasx_convert_utf16_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -63189,60 +60044,36 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return saved_bytes; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { std::pair ret = - lsx_convert_utf16_to_utf8_with_errors(buf, len, - utf8_output); + lasx_convert_utf16_to_utf8_with_errors(buf, len, utf8_output); if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } + return full_result(ret.first.error, ret.first.count, + size_t(ret.second - utf8_output)); } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + if (ret.first.count != len) { + full_result sres = + scalar::utf16_to_utf8::convert_with_errors( + buf + ret.first.count, len - ret.first.count, ret.second, 0); + return full_result(sres.error, ret.first.count + sres.input_count, + size_t(ret.second - utf8_output) + sres.output_count); + } + return full_result(error_code::SUCCESS, len, + size_t(ret.second - utf8_output)); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished - std::pair ret = - lsx_convert_utf16_to_utf8_with_errors(buf, len, + return convert_utf16_to_utf8_with_details(buf, len, utf8_output); - if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } - } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -63263,7 +60094,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_utf8( return 0; } std::pair ret = - lsx_convert_utf32_to_utf8(buf, len, utf8_output); + lasx_convert_utf32_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -63287,7 +60118,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf8_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lsx_convert_utf32_to_utf8_with_errors(buf, len, utf8_output); + lasx_convert_utf32_to_utf8_with_errors(buf, len, utf8_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf8::convert_with_errors( buf + ret.first.count, len - ret.first.count, ret.second); @@ -63309,7 +60140,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf8_with_errors( simdutf_warn_unused size_t implementation::convert_utf16le_to_utf32( const char16_t *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_utf32(buf, len, utf32_output); + lasx_convert_utf16_to_utf32(buf, len, utf32_output); if (ret.first == nullptr) { return 0; } @@ -63329,7 +60160,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_utf32( simdutf_warn_unused size_t implementation::convert_utf16be_to_utf32( const char16_t *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lsx_convert_utf16_to_utf32(buf, len, utf32_output); + lasx_convert_utf16_to_utf32(buf, len, utf32_output); if (ret.first == nullptr) { return 0; } @@ -63351,8 +60182,8 @@ simdutf_warn_unused result implementation::convert_utf16le_to_utf32_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lsx_convert_utf16_to_utf32_with_errors(buf, len, - utf32_output); + lasx_convert_utf16_to_utf32_with_errors(buf, len, + utf32_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -63379,8 +60210,8 @@ simdutf_warn_unused result implementation::convert_utf16be_to_utf32_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lsx_convert_utf16_to_utf32_with_errors(buf, len, - utf32_output); + lasx_convert_utf16_to_utf32_with_errors(buf, len, + utf32_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -63407,7 +60238,7 @@ simdutf_warn_unused result implementation::convert_utf16be_to_utf32_with_errors( simdutf_warn_unused size_t implementation::convert_utf32_to_latin1( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf32_to_latin1(buf, len, latin1_output); + lasx_convert_utf32_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -63427,7 +60258,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_latin1( simdutf_warn_unused result implementation::convert_utf32_to_latin1_with_errors( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf32_to_latin1_with_errors(buf, len, latin1_output); + lasx_convert_utf32_to_latin1_with_errors(buf, len, latin1_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -63451,7 +60282,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_latin1_with_errors( simdutf_warn_unused size_t implementation::convert_valid_utf32_to_latin1( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lsx_convert_utf32_to_latin1(buf, len, latin1_output); + lasx_convert_utf32_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -63478,7 +60309,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf32_to_utf8( simdutf_warn_unused size_t implementation::convert_utf32_to_utf16le( const char32_t *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lsx_convert_utf32_to_utf16(buf, len, utf16_output); + lasx_convert_utf32_to_utf16(buf, len, utf16_output); if (ret.first == nullptr) { return 0; } @@ -63499,7 +60330,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_utf16le( simdutf_warn_unused size_t implementation::convert_utf32_to_utf16be( const char32_t *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lsx_convert_utf32_to_utf16(buf, len, utf16_output); + lasx_convert_utf32_to_utf16(buf, len, utf16_output); if (ret.first == nullptr) { return 0; } @@ -63521,8 +60352,8 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf16le_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lsx_convert_utf32_to_utf16_with_errors(buf, len, - utf16_output); + lasx_convert_utf32_to_utf16_with_errors(buf, len, + utf16_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf16::convert_with_errors( @@ -63545,8 +60376,8 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf16be_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lsx_convert_utf32_to_utf16_with_errors(buf, len, - utf16_output); + lasx_convert_utf32_to_utf16_with_errors(buf, len, + utf16_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf16::convert_with_errors( @@ -63606,7 +60437,23 @@ simdutf_warn_unused size_t implementation::count_utf16be( #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t implementation::count_utf8(const char *input, size_t length) const noexcept { - return utf8::count_code_points(input, length); + size_t pos = 0; + size_t count = 0; + // Performance degradation when memory address is not 32-byte aligned + while ((((uint64_t)input + pos) & 0x1F && pos < length)) { + if (input[pos++] > -65) { + count++; + } + } + __m256i v_bf = __lasx_xvldi(0xBF); // 0b10111111 + for (; pos + 32 <= length; pos += 32) { + __m256i in = __lasx_xvld(reinterpret_cast(input + pos), 0); + __m256i utf8_count = + __lasx_xvpcnt_h(__lasx_xvmskltz_b(__lasx_xvslt_b(v_bf, in))); + count = count + __lasx_xvpickve2gr_wu(utf8_count, 0) + + __lasx_xvpickve2gr_wu(utf8_count, 4); + } + return count + scalar::utf8::count_code_points(input + pos, length - pos); } #endif // SIMDUTF_FEATURE_UTF8 @@ -63680,6 +60527,26 @@ implementation::utf8_length_from_utf16be_with_replacement( endianness::BIG>(input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -63692,12 +60559,12 @@ simdutf_warn_unused size_t implementation::utf8_length_from_utf32( #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t implementation::utf16_length_from_utf32( const char32_t *input, size_t length) const noexcept { - const __m128i v_ffff = lsx_splat_u32(0x0000ffff); + __m128i v_ffff = lsx_splat_u32(0x0000ffff); size_t pos = 0; size_t count = 0; for (; pos + 4 <= length; pos += 4) { __m128i in = __lsx_vld(reinterpret_cast(input + pos), 0); - const __m128i surrogate_bytemask = __lsx_vslt_wu(v_ffff, in); + __m128i surrogate_bytemask = __lsx_vslt_wu(v_ffff, in); size_t surrogate_count = __lsx_vpickve2gr_bu( __lsx_vpcnt_b(__lsx_vmskltz_w(surrogate_bytemask)), 0); count += 4 + surrogate_count; @@ -63861,35 +60728,51 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util_find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 -} // namespace lsx +} // namespace lasx } // namespace simdutf -/* begin file src/simdutf/lsx/end.h */ +/* begin file src/simdutf/lasx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lsx/end.h */ -/* end file src/lsx/implementation.cpp */ + +#if SIMDUTF_CAN_ALWAYS_RUN_LASX +// nothing needed. +#else +SIMDUTF_UNTARGET_REGION #endif -#if SIMDUTF_IMPLEMENTATION_LASX -/* begin file src/lasx/implementation.cpp */ -/* begin file src/simdutf/lasx/begin.h */ -// redefining SIMDUTF_IMPLEMENTATION to "lasx" -// #define SIMDUTF_IMPLEMENTATION lasx +/* end file src/simdutf/lasx/end.h */ +/* end file src/lasx/implementation.cpp */ +#endif +#if SIMDUTF_IMPLEMENTATION_LSX +/* begin file src/lsx/implementation.cpp */ +/* begin file src/simdutf/lsx/begin.h */ +// redefining SIMDUTF_IMPLEMENTATION to "lsx" +// #define SIMDUTF_IMPLEMENTATION lsx #define SIMDUTF_SIMD_HAS_UNSIGNED_CMP 1 -/* end file src/simdutf/lasx/begin.h */ +/* end file src/simdutf/lsx/begin.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { -#ifndef SIMDUTF_LASX_H - #error "lasx.h must be included" +#ifndef SIMDUTF_LSX_H + #error "lsx.h must be included" #endif using namespace simd; #if SIMDUTF_FEATURE_UTF8 // convert vmskltz/vmskgez/vmsknz to // simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes index -const uint8_t lasx_1_2_utf8_bytes_mask[] = { +const uint8_t lsx_1_2_utf8_bytes_mask[] = { 0, 1, 4, 5, 16, 17, 20, 21, 64, 65, 68, 69, 80, 81, 84, 85, 2, 3, 6, 7, 18, 19, 22, 23, 66, 67, 70, 71, 82, 83, 86, 87, 8, 9, 12, 13, 24, 25, 28, 29, 72, 73, 76, 77, 88, @@ -63914,9 +60797,6 @@ const uint8_t lasx_1_2_utf8_bytes_mask[] = { simdutf_really_inline __m128i lsx_swap_bytes(__m128i vec) { return __lsx_vshuf4i_b(vec, 0b10110001); } -simdutf_really_inline __m256i lasx_swap_bytes(__m256i vec) { - return __lasx_xvshuf4i_b(vec, 0b10110001); -} #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_ASCII || SIMDUTF_FEATURE_DETECT_ENCODING || \ @@ -63987,7 +60867,7 @@ convert_utf8_1_to_2_byte_to_utf16(__m128i in, size_t shufutf8_idx) { __m128i ascii = __lsx_vand_v(perm, __lsx_vrepli_h(0x7f)); // 6 or 7 bits // 1 byte: 00000000 00000000 // 2 byte: 00000aaa aa000000 - __m128i v1f00 = lsx_splat_u16(0x1f00); + const __m128i v1f00 = lsx_splat_u16(0x1f00); __m128i composed = __lsx_vsrli_h(__lsx_vand_v(perm, v1f00), 2); // 5 bits // Combine with a shift right accumulate // 1 byte: 00000000 0bbbbbbb @@ -63999,7 +60879,7 @@ convert_utf8_1_to_2_byte_to_utf16(__m128i in, size_t shufutf8_idx) { // SIMDUTF_FEATURE_UTF32) #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/lasx/lasx_validate_utf16.cpp */ +/* begin file src/lsx/lsx_validate_utf16.cpp */ template simd8 utf16_gather_high_bytes(const simd16 in0, const simd16 in1) { @@ -64013,117 +60893,101 @@ simd8 utf16_gather_high_bytes(const simd16 in0, return simd16::pack_shifted_right<8>(in0, in1); } } -/* end file src/lasx/lasx_validate_utf16.cpp */ +/* end file src/lsx/lsx_validate_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING -/* begin file src/lasx/lasx_validate_utf32le.cpp */ -const char32_t *lasx_validate_utf32le(const char32_t *input, size_t size) { +/* begin file src/lsx/lsx_validate_utf32le.cpp */ +const char32_t *lsx_validate_utf32le(const char32_t *input, size_t size) { const char32_t *end = input + size; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)input & 0x1F) && input < end) { - uint32_t word = *input++; - if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { - return nullptr; - } - } - - __m256i offset = lasx_splat_u32(0xffff2000); - __m256i standardoffsetmax = lasx_splat_u32(0xfffff7ff); - __m256i standardmax = lasx_splat_u32(0x10ffff); - __m256i currentmax = __lasx_xvldi(0x0); - __m256i currentoffsetmax = __lasx_xvldi(0x0); + __m128i offset = lsx_splat_u32(0xffff2000); + __m128i standardoffsetmax = lsx_splat_u32(0xfffff7ff); + __m128i standardmax = lsx_splat_u32(0x10ffff); + __m128i currentmax = lsx_splat_u32(0); + __m128i currentoffsetmax = lsx_splat_u32(0); - while (input + 8 < end) { - __m256i in = __lasx_xvld(reinterpret_cast(input), 0); - currentmax = __lasx_xvmax_wu(in, currentmax); + while (input + 4 < end) { + __m128i in = __lsx_vld(reinterpret_cast(input), 0); + currentmax = __lsx_vmax_wu(in, currentmax); // 0xD8__ + 0x2000 = 0xF8__ => 0xF8__ > 0xF7FF currentoffsetmax = - __lasx_xvmax_wu(__lasx_xvadd_w(in, offset), currentoffsetmax); - input += 8; + __lsx_vmax_wu(__lsx_vadd_w(in, offset), currentoffsetmax); + + input += 4; } - __m256i is_zero = - __lasx_xvxor_v(__lasx_xvmax_wu(currentmax, standardmax), standardmax); - if (__lasx_xbnz_v(is_zero)) { + + __m128i is_zero = + __lsx_vxor_v(__lsx_vmax_wu(currentmax, standardmax), standardmax); + if (__lsx_bnz_v(is_zero)) { return nullptr; } - is_zero = __lasx_xvxor_v(__lasx_xvmax_wu(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (__lasx_xbnz_v(is_zero)) { + is_zero = __lsx_vxor_v(__lsx_vmax_wu(currentoffsetmax, standardoffsetmax), + standardoffsetmax); + if (__lsx_bnz_v(is_zero)) { return nullptr; } + return input; } -const result lasx_validate_utf32le_with_errors(const char32_t *input, - size_t size) { +const result lsx_validate_utf32le_with_errors(const char32_t *input, + size_t size) { const char32_t *start = input; const char32_t *end = input + size; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)input & 0x1F) && input < end) { - uint32_t word = *input; - if (word > 0x10FFFF) { - return result(error_code::TOO_LARGE, input - start); - } - if (word >= 0xD800 && word <= 0xDFFF) { - return result(error_code::SURROGATE, input - start); - } - input++; - } - - __m256i offset = lasx_splat_u32(0xffff2000); - __m256i standardoffsetmax = lasx_splat_u32(0xfffff7ff); - __m256i standardmax = lasx_splat_u32(0x10ffff); - __m256i currentmax = __lasx_xvldi(0x0); - __m256i currentoffsetmax = __lasx_xvldi(0x0); + __m128i offset = lsx_splat_u32(0xffff2000); + __m128i standardoffsetmax = lsx_splat_u32(0xfffff7ff); + __m128i standardmax = lsx_splat_u32(0x10ffff); + __m128i currentmax = lsx_splat_u32(0); + __m128i currentoffsetmax = lsx_splat_u32(0); - while (input + 8 < end) { - __m256i in = __lasx_xvld(reinterpret_cast(input), 0); - currentmax = __lasx_xvmax_wu(in, currentmax); + while (input + 4 < end) { + __m128i in = __lsx_vld(reinterpret_cast(input), 0); + currentmax = __lsx_vmax_wu(in, currentmax); currentoffsetmax = - __lasx_xvmax_wu(__lasx_xvadd_w(in, offset), currentoffsetmax); + __lsx_vmax_wu(__lsx_vadd_w(in, offset), currentoffsetmax); - __m256i is_zero = - __lasx_xvxor_v(__lasx_xvmax_wu(currentmax, standardmax), standardmax); - if (__lasx_xbnz_v(is_zero)) { + __m128i is_zero = + __lsx_vxor_v(__lsx_vmax_wu(currentmax, standardmax), standardmax); + if (__lsx_bnz_v(is_zero)) { return result(error_code::TOO_LARGE, input - start); } - is_zero = - __lasx_xvxor_v(__lasx_xvmax_wu(currentoffsetmax, standardoffsetmax), - standardoffsetmax); - if (__lasx_xbnz_v(is_zero)) { + + is_zero = __lsx_vxor_v(__lsx_vmax_wu(currentoffsetmax, standardoffsetmax), + standardoffsetmax); + if (__lsx_bnz_v(is_zero)) { return result(error_code::SURROGATE, input - start); } - input += 8; + + input += 4; } return result(error_code::SUCCESS, input - start); } -/* end file src/lasx/lasx_validate_utf32le.cpp */ +/* end file src/lsx/lsx_validate_utf32le.cpp */ #endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_latin1_to_utf8.cpp */ +/* begin file src/lsx/lsx_convert_latin1_to_utf8.cpp */ /* Returns a pair: the first unprocessed byte from buf and utf8_output A scalar routing should carry on the conversion of the tail. */ std::pair -lasx_convert_latin1_to_utf8(const char *latin1_input, size_t len, - char *utf8_out) { +lsx_convert_latin1_to_utf8(const char *latin1_input, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); - const size_t safety_margin = 12; const char *end = latin1_input + len; + __m128i zero = __lsx_vldi(0); // We always write 16 bytes, of which more than the first 8 bytes // are valid. A safety margin of 8 is more than sufficient. - while (end - latin1_input >= std::ptrdiff_t(16 + safety_margin)) { + while (end - latin1_input >= 16) { __m128i in8 = __lsx_vld(reinterpret_cast(latin1_input), 0); - uint32_t ascii_mask = __lsx_vpickve2gr_wu(__lsx_vmskgez_b(in8), 0); - if (ascii_mask == 0xFFFF) { + uint32_t ascii = __lsx_vpickve2gr_hu(__lsx_vmskgez_b(in8), 0); + if (ascii == 0xffff) { // ASCII fast path!!!! __lsx_vst(in8, utf8_output, 0); utf8_output += 16; latin1_input += 16; @@ -64131,76 +60995,48 @@ lasx_convert_latin1_to_utf8(const char *latin1_input, size_t len, } // We just fallback on UTF-16 code. This could be optimized/simplified // further. - __m256i in16 = __lasx_vext2xv_hu_bu(____m256i(in8)); + __m128i in16 = __lsx_vilvl_b(zero, in8); // 1. prepare 2-byte values - // input 8-bit word : [aabb|bbbb] x 16 - // expected output : [1100|00aa|10bb|bbbb] x 16 + // input 8-bit word : [aabb|bbbb] x 8 + // expected output : [1100|00aa|10bb|bbbb] x 8 // t0 = [0000|00aa|bbbb|bb00] - __m256i t0 = __lasx_xvslli_h(in16, 2); + __m128i t0 = __lsx_vslli_h(in16, 2); // t1 = [0000|00aa|0000|0000] - __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x300)); + __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x300)); // t3 = [0000|00aa|00bb|bbbb] - __m256i t2 = __lasx_xvbitsel_v(t1, in16, __lasx_xvrepli_h(0x3f)); + __m128i t2 = __lsx_vbitsel_v(t1, in16, __lsx_vrepli_h(0x3f)); // t4 = [1100|00aa|10bb|bbbb] - __m256i t3 = __lasx_xvor_v(t2, __lasx_xvreplgr2vr_h(uint16_t(0xc080))); + __m128i t3 = __lsx_vor_v(t2, __lsx_vreplgr2vr_h(uint16_t(0xc080))); // merge ASCII and 2-byte codewords - __m256i one_byte_bytemask = __lasx_xvsle_hu(in16, __lasx_xvrepli_h(0x7F)); - __m256i utf8_unpacked = __lasx_xvbitsel_v(t3, in16, one_byte_bytemask); + __m128i one_byte_bytemask = __lsx_vsle_hu(in16, __lsx_vrepli_h(0x7F)); + __m128i utf8_unpacked = __lsx_vbitsel_v(t3, in16, one_byte_bytemask); - const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[(ascii_mask & 0xFF)]][0]; - __m128i shuffle0 = __lsx_vld(row0 + 1, 0); - __m128i utf8_unpacked_lo = lasx_extracti128_lo(utf8_unpacked); - __m128i utf8_packed0 = - __lsx_vshuf_b(utf8_unpacked_lo, utf8_unpacked_lo, shuffle0); - __lsx_vst(utf8_packed0, utf8_output, 0); - utf8_output += row0[0]; + const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lsx_1_2_utf8_bytes_mask[(ascii & 0xff)]][0]; + __m128i shuffle = __lsx_vld(row + 1, 0); + __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); - const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[(ascii_mask >> 8)]][0]; - __m128i shuffle1 = __lsx_vld(row1 + 1, 0); - __m128i utf8_unpacked_hi = lasx_extracti128_hi(utf8_unpacked); - __m128i utf8_packed1 = - __lsx_vshuf_b(utf8_unpacked_hi, utf8_unpacked_hi, shuffle1); - __lsx_vst(utf8_packed1, utf8_output, 0); - utf8_output += row1[0]; + // store bytes + __lsx_vst(utf8_packed, utf8_output, 0); + // adjust pointers + latin1_input += 8; + utf8_output += row[0]; - latin1_input += 16; } // while return std::make_pair(latin1_input, reinterpret_cast(utf8_output)); } -/* end file src/lasx/lasx_convert_latin1_to_utf8.cpp */ +/* end file src/lsx/lsx_convert_latin1_to_utf8.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_latin1_to_utf16.cpp */ +/* begin file src/lsx/lsx_convert_latin1_to_utf16.cpp */ std::pair -lasx_convert_latin1_to_utf16le(const char *buf, size_t len, - char16_t *utf16_output) { +lsx_convert_latin1_to_utf16le(const char *buf, size_t len, + char16_t *utf16_output) { const char *end = buf + len; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)utf16_output & 0x1F) && buf < end) { - *utf16_output++ = uint8_t(*buf) & 0xFF; - buf++; - } - - while (end - buf >= 32) { - __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); - - __m256i inlow = __lasx_vext2xv_hu_bu(in8); - __m256i in8_high = __lasx_xvpermi_q(in8, in8, 0b00000001); - __m256i inhigh = __lasx_vext2xv_hu_bu(in8_high); - __lasx_xvst(inlow, reinterpret_cast(utf16_output), 0); - __lasx_xvst(inhigh, reinterpret_cast(utf16_output), 32); - - utf16_output += 32; - buf += 32; - } - - if (end - buf >= 16) { - __m128i zero = __lsx_vldi(0); + __m128i zero = __lsx_vldi(0); + while (end - buf >= 16) { __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); __m128i inlow = __lsx_vilvl_b(zero, in8); @@ -64211,38 +61047,20 @@ lasx_convert_latin1_to_utf16le(const char *buf, size_t len, utf16_output += 16; buf += 16; } + return std::make_pair(buf, utf16_output); } std::pair -lasx_convert_latin1_to_utf16be(const char *buf, size_t len, - char16_t *utf16_output) { +lsx_convert_latin1_to_utf16be(const char *buf, size_t len, + char16_t *utf16_output) { const char *end = buf + len; - - while (((uint64_t)utf16_output & 0x1F) && buf < end) { - *utf16_output++ = (uint16_t(*buf++) << 8); - } - - __m256i zero = __lasx_xvldi(0); - while (end - buf >= 32) { - __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); - - __m256i in8_shuf = __lasx_xvpermi_d(in8, 0b11011000); - - __m256i inlow = __lasx_xvilvl_b(in8_shuf, zero); - __m256i inhigh = __lasx_xvilvh_b(in8_shuf, zero); - __lasx_xvst(inlow, reinterpret_cast(utf16_output), 0); - __lasx_xvst(inhigh, reinterpret_cast(utf16_output), 32); - utf16_output += 32; - buf += 32; - } - - if (end - buf >= 16) { - __m128i zero_128 = __lsx_vldi(0); + __m128i zero = __lsx_vldi(0); + while (end - buf >= 16) { __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); - __m128i inlow = __lsx_vilvl_b(in8, zero_128); - __m128i inhigh = __lsx_vilvh_b(in8, zero_128); + __m128i inlow = __lsx_vilvl_b(in8, zero); + __m128i inhigh = __lsx_vilvh_b(in8, zero); __lsx_vst(inlow, reinterpret_cast(utf16_output), 0); __lsx_vst(inhigh, reinterpret_cast(utf16_output), 16); utf16_output += 16; @@ -64251,44 +61069,16 @@ lasx_convert_latin1_to_utf16be(const char *buf, size_t len, return std::make_pair(buf, utf16_output); } -/* end file src/lasx/lasx_convert_latin1_to_utf16.cpp */ +/* end file src/lsx/lsx_convert_latin1_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_latin1_to_utf32.cpp */ +/* begin file src/lsx/lsx_convert_latin1_to_utf32.cpp */ std::pair -lasx_convert_latin1_to_utf32(const char *buf, size_t len, - char32_t *utf32_output) { +lsx_convert_latin1_to_utf32(const char *buf, size_t len, + char32_t *utf32_output) { const char *end = buf + len; - // LASX requires 32-byte alignment, otherwise performance will be degraded - while (((uint64_t)utf32_output & 0x1F) && buf < end) { - *utf32_output++ = ((uint32_t)*buf) & 0xFF; - buf++; - } - - while (end - buf >= 32) { - __m256i in8 = __lasx_xvld(reinterpret_cast(buf), 0); - - __m256i in32_0 = __lasx_vext2xv_wu_bu(in8); - __lasx_xvst(in32_0, reinterpret_cast(utf32_output), 0); - - __m256i in8_1 = __lasx_xvpermi_d(in8, 0b00000001); - __m256i in32_1 = __lasx_vext2xv_wu_bu(in8_1); - __lasx_xvst(in32_1, reinterpret_cast(utf32_output), 32); - - __m256i in8_2 = __lasx_xvpermi_d(in8, 0b00000010); - __m256i in32_2 = __lasx_vext2xv_wu_bu(in8_2); - __lasx_xvst(in32_2, reinterpret_cast(utf32_output), 64); - - __m256i in8_3 = __lasx_xvpermi_d(in8, 0b00000011); - __m256i in32_3 = __lasx_vext2xv_wu_bu(in8_3); - __lasx_xvst(in32_3, reinterpret_cast(utf32_output), 96); - - utf32_output += 32; - buf += 32; - } - - if (end - buf >= 16) { + while (end - buf >= 16) { __m128i in8 = __lsx_vld(reinterpret_cast(buf), 0); __m128i zero = __lsx_vldi(0); @@ -64300,9 +61090,9 @@ lasx_convert_latin1_to_utf32(const char *buf, size_t len, __m128i in32_3 = __lsx_vilvh_h(zero, in16high); __lsx_vst(in32_0, reinterpret_cast(utf32_output), 0); - __lsx_vst(in32_1, reinterpret_cast(utf32_output), 16); - __lsx_vst(in32_2, reinterpret_cast(utf32_output), 32); - __lsx_vst(in32_3, reinterpret_cast(utf32_output), 48); + __lsx_vst(in32_1, reinterpret_cast(utf32_output + 4), 0); + __lsx_vst(in32_2, reinterpret_cast(utf32_output + 8), 0); + __lsx_vst(in32_3, reinterpret_cast(utf32_output + 12), 0); utf32_output += 16; buf += 16; @@ -64310,11 +61100,11 @@ lasx_convert_latin1_to_utf32(const char *buf, size_t len, return std::make_pair(buf, utf32_output); } -/* end file src/lasx/lasx_convert_latin1_to_utf32.cpp */ +/* end file src/lsx/lsx_convert_latin1_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -/* begin file src/lasx/lasx_convert_utf8_to_utf16.cpp */ +/* begin file src/lsx/lsx_convert_utf8_to_utf16.cpp */ // Convert up to 16 bytes from utf8 to utf16 using a mask indicating the // end of the code points. Only the least significant 12 bits of the mask // are accessed. @@ -64339,22 +61129,15 @@ size_t convert_masked_utf8_to_utf16(const char *input, // We first try a few fast paths. // The obvious first test is ASCII, which actually consumes the full 16. if ((utf8_end_of_code_point_mask & 0xFFFF) == 0xFFFF) { - __m128i zero = __lsx_vldi(0); - if (match_system(big_endian)) { - __lsx_vst(__lsx_vilvl_b(zero, in), - reinterpret_cast(utf16_output), 0); - __lsx_vst(__lsx_vilvh_b(zero, in), - reinterpret_cast(utf16_output), 16); - } else { - __lsx_vst(__lsx_vilvl_b(in, zero), - reinterpret_cast(utf16_output), 0); - __lsx_vst(__lsx_vilvh_b(in, zero), - reinterpret_cast(utf16_output), 16); - } + // We process in chunks of 16 bytes + // The routine in simd.h is reused. + simd8 temp{in}; + temp.store_ascii_as_utf16(utf16_output); utf16_output += 16; // We wrote 16 16-bit characters. return 16; // We consumed 16 bytes. } + uint64_t buffer[2]; // 3 byte sequences are the next most common, as seen in CJK, which has long // sequences of these. if (input_utf8_end_of_code_point_mask == 0x924) { @@ -64362,7 +61145,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. __m128i composed = convert_utf8_3_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } @@ -64377,13 +61160,13 @@ size_t convert_masked_utf8_to_utf16(const char *input, // UTF-16 code units. __m128i composed = convert_utf8_2_byte_to_utf16(in); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } __lsx_vst(composed, reinterpret_cast(utf16_output), 0); - utf16_output += 8; // We wrote 6 16-bit characters. - return 16; // We consumed 12 bytes. + utf16_output += 6; // We wrote 6 16-bit characters. + return 12; // We consumed 12 bytes. } /// We do not have a fast path available, or the fast path is unimportant, so @@ -64399,7 +61182,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // Convert to UTF-16 __m128i composed = convert_utf8_1_to_2_byte_to_utf16(in, idx); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } // Store @@ -64445,7 +61228,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // aaaabbbb bbcccccc composed = __lsx_vbitsel_v(highperm, composed, v0fff); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } @@ -64455,6 +61238,13 @@ size_t convert_masked_utf8_to_utf16(const char *input, } else if (idx < 209) { // THREE (3) input code-code units if (input_utf8_end_of_code_point_mask == 0x888) { + // We want to take 3 4-byte UTF-8 code units and turn them into 3 4-byte + // UTF-16 pairs. Generating surrogate pairs is a little tricky though, but + // it is easier when we can assume they are all pairs. This version does + // not use the LUT, but 4 byte sequences are less common and the overhead + // of the extra memory access is less important than the early branch + // overhead in shorter sequences. + __m128i expected_mask = (__m128i)v16u8{0xf8, 0xc0, 0xc0, 0xc0, 0xf8, 0xc0, 0xc0, 0xc0, 0xf8, 0xc0, 0xc0, 0xc0, 0x0, 0x0, 0x0, 0x0}; @@ -64464,13 +61254,6 @@ size_t convert_masked_utf8_to_utf16(const char *input, __m128i check = __lsx_vseq_b(__lsx_vand_v(in, expected_mask), expected); if (__lsx_bz_b(check)) return 12; - // We want to take 3 4-byte UTF-8 code units and turn them into 3 4-byte - // UTF-16 pairs. Generating surrogate pairs is a little tricky though, but - // it is easier when we can assume they are all pairs. This version does - // not use the LUT, but 4 byte sequences are less common and the overhead - // of the extra memory access is less important than the early branch - // overhead in shorter sequences. - // Swap byte pairs // 10dddddd 10cccccc|10bbbbbb 11110aaa // 10cccccc 10dddddd|11110aaa 10bbbbbb @@ -64488,7 +61271,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, __m128i magic = __lsx_vreplgr2vr_w(uint32_t(0xDC00E7C0)); // Generate unadjusted trail surrogate minus lowest 2 bits // xxxxxxxx xxxxxxxx|11110aaa bbbbbb00 - __m128i trail = __lsx_vbitsel_v(shift, swap, lsx_splat_u32(0x0000FF00)); + __m128i trail = __lsx_vbitsel_v(shift, swap, lsx_splat_u32(0x0000ff00)); // Insert low 2 bits of trail surrogate to magic number for later // 11011100 00000000 11100111 110000cc __m128i magic_with_low_2 = __lsx_vor_v(__lsx_vsrli_w(shift, 30), magic); @@ -64508,10 +61291,12 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 110111CC CCDDDDDD|110110AA BBBBBBCC __m128i composed = __lsx_vadd_h(blend, magic_with_low_2); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { composed = lsx_swap_bytes(composed); } - __lsx_vst(composed, reinterpret_cast(utf16_output), 0); + // __lsx_vst(composed, reinterpret_cast(utf16_output), 0); + __lsx_vst(composed, reinterpret_cast(buffer), 0); + std::memcpy(utf16_output, buffer, 12); utf16_output += 6; // We 3 32-bit surrogate pairs. return 12; // We consumed 12 bytes. } @@ -64582,7 +61367,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // 4 byte: 110110AA BBBBBBCC|110111CC CCDDDDDD __m128i selected = __lsx_vbitsel_v(composed, surrogates, is_pair); // Byte swap if necessary - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { selected = lsx_swap_bytes(selected); } // Attempting to shuffle and store would be complex, just scalarize. @@ -64611,10 +61396,10 @@ size_t convert_masked_utf8_to_utf16(const char *input, return 12; } } -/* end file src/lasx/lasx_convert_utf8_to_utf16.cpp */ +/* end file src/lsx/lsx_convert_utf8_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lasx/lasx_convert_utf8_to_utf32.cpp */ +/* begin file src/lsx/lsx_convert_utf8_to_utf32.cpp */ // Convert up to 12 bytes from utf8 to utf32 using a mask indicating the // end of the code points. Only the least significant 12 bits of the mask // are accessed. @@ -64641,19 +61426,8 @@ size_t convert_masked_utf8_to_utf32(const char *input, // We process in chunks of 16 bytes. // use fast implementation in src/simdutf/arm64/simd.h // Ideally the compiler can keep the tables in registers. - __m128i zero = __lsx_vldi(0); - __m128i in16low = __lsx_vilvl_b(zero, in); - __m128i in16high = __lsx_vilvh_b(zero, in); - __m128i in32_0 = __lsx_vilvl_h(zero, in16low); - __m128i in32_1 = __lsx_vilvh_h(zero, in16low); - __m128i in32_2 = __lsx_vilvl_h(zero, in16high); - __m128i in32_3 = __lsx_vilvh_h(zero, in16high); - - __lsx_vst(in32_0, reinterpret_cast(utf32_output), 0); - __lsx_vst(in32_1, reinterpret_cast(utf32_output), 16); - __lsx_vst(in32_2, reinterpret_cast(utf32_output), 32); - __lsx_vst(in32_3, reinterpret_cast(utf32_output), 48); - + simd8 temp{in}; + temp.store_ascii_as_utf32_tbl(utf32_out); utf32_output += 16; // We wrote 16 32-bit characters. return 16; // We consumed 16 bytes. } @@ -64682,7 +61456,7 @@ size_t convert_masked_utf8_to_utf32(const char *input, utf32_output += 6; return 12; // We consumed 12 bytes. } - // Either no fast path or an unimportant fast path. + /// Either no fast path or an unimportant fast path. const uint8_t idx = simdutf::tables::utf8_to_utf16::utf8bigindex [input_utf8_end_of_code_point_mask][0]; @@ -64805,10 +61579,10 @@ size_t convert_masked_utf8_to_utf32(const char *input, return 12; } } -/* end file src/lasx/lasx_convert_utf8_to_utf32.cpp */ +/* end file src/lsx/lsx_convert_utf8_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_utf8_to_latin1.cpp */ +/* begin file src/lsx/lsx_convert_utf8_to_latin1.cpp */ size_t convert_masked_utf8_to_latin1(const char *input, uint64_t utf8_end_of_code_point_mask, char *&latin1_output) { @@ -64877,24 +61651,27 @@ size_t convert_masked_utf8_to_latin1(const char *input, // writing 8 bytes even though we only care about the first 6 bytes. __m128i latin1_packed = __lsx_vpickev_b(__lsx_vldi(0), composed); - __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); + uint64_t buffer[2]; + // __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); + __lsx_vst(latin1_packed, reinterpret_cast(buffer), 0); + std::memcpy(latin1_output, buffer, 6); latin1_output += 6; // We wrote 6 bytes. return consumed; } -/* end file src/lasx/lasx_convert_utf8_to_latin1.cpp */ +/* end file src/lsx/lsx_convert_utf8_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_utf16_to_latin1.cpp */ +/* begin file src/lsx/lsx_convert_utf16_to_latin1.cpp */ template std::pair -lasx_convert_utf16_to_latin1(const char16_t *buf, size_t len, - char *latin1_output) { +lsx_convert_utf16_to_latin1(const char16_t *buf, size_t len, + char *latin1_output) { const char16_t *end = buf + len; while (end - buf >= 16) { __m128i in = __lsx_vld(reinterpret_cast(buf), 0); __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = lsx_swap_bytes(in); in1 = lsx_swap_bytes(in1); } @@ -64915,14 +61692,14 @@ lasx_convert_utf16_to_latin1(const char16_t *buf, size_t len, template std::pair -lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, - char *latin1_output) { +lsx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, + char *latin1_output) { const char16_t *start = buf; const char16_t *end = buf + len; while (end - buf >= 16) { __m128i in = __lsx_vld(reinterpret_cast(buf), 0); __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { in = lsx_swap_bytes(in); in1 = lsx_swap_bytes(in1); } @@ -64937,8 +61714,7 @@ lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, } else { // Let us do a scalar fallback. for (int k = 0; k < 16; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if (word <= 0xff) { *latin1_output++ = char(word); } else { @@ -64951,12 +61727,12 @@ lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), latin1_output); } -/* end file src/lasx/lasx_convert_utf16_to_latin1.cpp */ +/* end file src/lsx/lsx_convert_utf16_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 -/* begin file src/lasx/lasx_convert_utf16_to_utf8.cpp */ +#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 +/* begin file src/lsx/lsx_convert_utf16_to_utf8.cpp */ /* - The vectorized algorithm works on single LASX register i.e., it + The vectorized algorithm works on single SSE register i.e., it loads eight 16-bit code units. We consider three cases: @@ -64974,7 +61750,7 @@ lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, char) or 2) two UTF8 bytes. For this case we do only some shuffle to obtain these 2-byte - codes and finally compress the whole LASX register with a single + codes and finally compress the whole SSE register with a single shuffle. We need 256-entry lookup table to get a compression pattern @@ -64992,7 +61768,7 @@ lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, the three-UTF8-bytes case. Finally these two registers are interleaved forming eight-element - array of 32-bit values. The array spans two LASX registers. + array of 32-bit values. The array spans two SSE registers. The bytes from the registers are compressed using two shuffles. We need 256-entry lookup table to get a compression pattern @@ -65007,10 +61783,9 @@ lasx_convert_utf16_to_latin1_with_errors(const char16_t *buf, size_t len, Returns a pair: the first unprocessed byte from buf and utf8_output A scalar routing should carry on the conversion of the tail. */ - template std::pair -lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { +lsx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char16_t *end = buf + len; @@ -65018,77 +61793,82 @@ lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 - __m256i v_07ff = __lasx_xvreplgr2vr_h(uint16_t(0x7ff)); - __m256i zero = __lasx_xvldi(0); - __m128i zero_128 = __lsx_vldi(0); + __m128i v_07ff = __lsx_vreplgr2vr_h(uint16_t(0x7ff)); while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lasx_swap_bytes(in); + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lsx_swap_bytes(in); } - if (__lasx_xbnz_h(__lasx_xvslt_hu( - in, __lasx_xvrepli_h(0x7F)))) { // ASCII fast path!!!! - // 1. pack the bytes - __m256i utf8_packed = - __lasx_xvpermi_d(__lasx_xvpickev_b(in, in), 0b00001000); - // 2. store (16 bytes) - __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); - // 3. adjust pointers - buf += 16; - utf8_output += 16; - continue; // we are done for this round! + if (__lsx_bz_v( + __lsx_vslt_hu(__lsx_vrepli_h(0x7F), in))) { // ASCII fast path!!!! + // It is common enough that we have sequences of 16 consecutive ASCII + // characters. + __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); + if constexpr (!match_system(big_endian)) { + nextin = lsx_swap_bytes(nextin); + } + if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), nextin))) { + // 1. pack the bytes + // obviously suboptimal. + __m128i utf8_packed = __lsx_vpickev_b(nextin, in); + // 2. store (16 bytes) + __lsx_vst(utf8_packed, utf8_output, 0); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } else { + // 1. pack the bytes + // obviously suboptimal. + __m128i utf8_packed = __lsx_vpickev_b(in, in); + // 2. store (8 bytes) + __lsx_vst(utf8_packed, utf8_output, 0); + // 3. adjust pointers + buf += 8; + utf8_output += 8; + in = nextin; + } } - if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, in))) { + __m128i zero = __lsx_vldi(0); + if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, in))) { // 1. prepare 2-byte values - // input 16-bit word : [0000|0aaa|aabb|bbbb] x 16 - // expected output : [110a|aaaa|10bb|bbbb] x 16 + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 + // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - __m256i t0 = __lasx_xvslli_h(in, 2); + __m128i t0 = __lsx_vslli_h(in, 2); // t1 = [000a|aaaa|0000|0000] - __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); + __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - __m256i t2 = __lasx_xvand_v(in, __lasx_xvrepli_h(0x3f)); + __m128i t2 = __lsx_vand_v(in, __lsx_vrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - __m256i t3 = __lasx_xvor_v(t1, t2); + __m128i t3 = __lsx_vor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - __m256i v_c080 = __lasx_xvreplgr2vr_h(uint16_t(0xc080)); - __m256i t4 = __lasx_xvor_v(t3, v_c080); + __m128i v_c080 = __lsx_vreplgr2vr_h(uint16_t(0xc080)); + __m128i t4 = __lsx_vor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m256i one_byte_bytemask = - __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F /*0x007F*/)); - __m256i utf8_unpacked = __lasx_xvbitsel_v(t4, in, one_byte_bytemask); + __m128i one_byte_bytemask = + __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F /*0x007F*/)); + __m128i utf8_unpacked = __lsx_vbitsel_v(t4, in, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); - uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); + uint32_t m2 = __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); // 4. pack the bytes - const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m1]][0]; - __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_packed1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); - - const uint8_t *row2 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_packed2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); + const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lsx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle = __lsx_vld(row, 1); + __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); // 5. store bytes - __lsx_vst(utf8_packed1, utf8_output, 0); - utf8_output += row1[0]; - - __lsx_vst(utf8_packed2, utf8_output, 0); - utf8_output += row2[0]; - - buf += 16; + __lsx_vst(utf8_packed, utf8_output, 0); + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; continue; } - __m256i surrogates_bytemask = __lasx_xvseq_h( - __lasx_xvand_v(in, lasx_splat_u16(0xf800)), lasx_splat_u16(0xd800)); + __m128i surrogates_bytemask = __lsx_vseq_h( + __lsx_vand_v(in, lsx_splat_u16(0xf800)), lsx_splat_u16(0xd800)); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lasx_xbz_v(surrogates_bytemask)) { + if (__lsx_bz_v(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes /* In this branch we handle three cases: 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - @@ -65118,94 +61898,74 @@ lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m256i t0 = __lasx_xvpickev_b(in, in); - t0 = __lasx_xvilvl_b(t0, t0); + __m128i t0 = __lsx_vpickev_b(in, in); + t0 = __lsx_vilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|00cc|cccc] - __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); - __m256i t1 = __lasx_xvand_v(t0, v_3f7f); + __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); + __m128i t1 = __lsx_vand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); + __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m256i s0 = __lasx_xvsrli_h(in, 12); + __m128i s0 = __lsx_vsrli_h(in, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m256i s1 = __lasx_xvslli_h(in, 2); + __m128i s1 = __lsx_vslli_h(in, 2); // s1: [aabb|bbbb|cccc|cc00] => [00bb|bbbb|0000|0000] - s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); + s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3f00)); // [00bb|bbbb|0000|aaaa] - __m256i s2 = __lasx_xvor_v(s0, s1); + __m128i s2 = __lsx_vor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); - __m256i s3 = __lasx_xvor_v(s2, v_c0e0); - __m256i one_or_two_bytes_bytemask = __lasx_xvsle_hu(in, v_07ff); - __m256i m0 = - __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); - __m256i s4 = __lasx_xvxor_v(s3, m0); + __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); + __m128i s3 = __lsx_vor_v(s2, v_c0e0); + __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(in, v_07ff); + __m128i m0 = + __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); + __m128i s4 = __lsx_vxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m256i out0 = __lasx_xvilvl_h(s4, t2); - __m256i out1 = __lasx_xvilvh_h(s4, t2); + __m128i out0 = __lsx_vilvl_h(s4, t2); + __m128i out1 = __lsx_vilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m256i one_byte_bytemask = __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F)); - __m256i one_byte_bytemask_low = - __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); - __m256i one_byte_bytemask_high = - __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask = __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F)); - __m256i one_or_two_bytes_bytemask_low = - __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); - __m256i one_or_two_bytes_bytemask_high = - __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_low = + __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_high = + __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); - __m256i mask0 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_low, one_byte_bytemask_low)); - __m256i mask1 = __lasx_xvmskltz_h(__lasx_xvor_v( - one_or_two_bytes_bytemask_high, one_byte_bytemask_high)); + __m128i one_byte_bytemask_low = + __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_high = + __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + + const uint32_t mask0 = __lsx_vpickve2gr_bu( + __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_low, + one_byte_bytemask_low)), + 0); + const uint32_t mask1 = __lsx_vpickve2gr_bu( + __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_high, + one_byte_bytemask_high)), + 0); - uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); - mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); + __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); + + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - mask = __lasx_xvpickve2gr_wu(mask0, 4); - const uint8_t *row2 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); - __lsx_vst(utf8_2, utf8_output, 0); - utf8_output += row2[0]; - - mask = __lasx_xvpickve2gr_wu(mask1, 4); - const uint8_t *row3 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle3 = __lsx_vld(row3, 1); - __m128i utf8_3 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); - __lsx_vst(utf8_3, utf8_output, 0); - utf8_output += row3[0]; - - buf += 16; + buf += 8; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -65217,8 +61977,7 @@ lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -65231,9 +61990,8 @@ lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -65262,8 +62020,8 @@ lasx_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_out) { */ template std::pair -lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, - char *utf8_out) { +lsx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char16_t *start = buf; const char16_t *end = buf + len; @@ -65271,78 +62029,82 @@ lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 - - __m256i v_07ff = __lasx_xvreplgr2vr_h(uint16_t(0x7ff)); - __m256i zero = __lasx_xvldi(0); - __m128i zero_128 = __lsx_vldi(0); while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lasx_swap_bytes(in); + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lsx_swap_bytes(in); } - if (__lasx_xbnz_h(__lasx_xvslt_hu( - in, __lasx_xvrepli_h(0x7F)))) { // ASCII fast path!!!! - // 1. pack the bytes - __m256i utf8_packed = - __lasx_xvpermi_d(__lasx_xvpickev_b(in, in), 0b00001000); - // 2. store (16 bytes) - __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); - // 3. adjust pointers - buf += 16; - utf8_output += 16; - continue; // we are done for this round! + if (__lsx_bz_v( + __lsx_vslt_hu(__lsx_vrepli_h(0x7F), in))) { // ASCII fast path!!!! + // It is common enough that we have sequences of 16 consecutive ASCII + // characters. + __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); + if constexpr (!match_system(big_endian)) { + nextin = lsx_swap_bytes(nextin); + } + if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), nextin))) { + // 1. pack the bytes + // obviously suboptimal. + __m128i utf8_packed = __lsx_vpickev_b(nextin, in); + // 2. store (16 bytes) + __lsx_vst(utf8_packed, utf8_output, 0); + // 3. adjust pointers + buf += 16; + utf8_output += 16; + continue; // we are done for this round! + } else { + // 1. pack the bytes + // obviously suboptimal. + __m128i utf8_packed = __lsx_vpickev_b(in, in); + // 2. store (8 bytes) + __lsx_vst(utf8_packed, utf8_output, 0); + // 3. adjust pointers + buf += 8; + utf8_output += 8; + in = nextin; + } } - if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, in))) { + __m128i v_07ff = __lsx_vreplgr2vr_h(uint16_t(0x7ff)); + __m128i zero = __lsx_vldi(0); + if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, in))) { // 1. prepare 2-byte values - // input 16-bit word : [0000|0aaa|aabb|bbbb] x 16 - // expected output : [110a|aaaa|10bb|bbbb] x 16 + // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 + // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - __m256i t0 = __lasx_xvslli_h(in, 2); + __m128i t0 = __lsx_vslli_h(in, 2); // t1 = [000a|aaaa|0000|0000] - __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); + __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - __m256i t2 = __lasx_xvand_v(in, __lasx_xvrepli_h(0x3f)); + __m128i t2 = __lsx_vand_v(in, __lsx_vrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - __m256i t3 = __lasx_xvor_v(t1, t2); + __m128i t3 = __lsx_vor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - __m256i v_c080 = __lasx_xvreplgr2vr_h(uint16_t(0xc080)); - __m256i t4 = __lasx_xvor_v(t3, v_c080); + __m128i v_c080 = __lsx_vreplgr2vr_h(uint16_t(0xc080)); + __m128i t4 = __lsx_vor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m256i one_byte_bytemask = - __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F /*0x007F*/)); - __m256i utf8_unpacked = __lasx_xvbitsel_v(t4, in, one_byte_bytemask); + __m128i one_byte_bytemask = + __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F /*0x007F*/)); + __m128i utf8_unpacked = __lsx_vbitsel_v(t4, in, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); - uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); + uint32_t m2 = __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); // 4. pack the bytes - const uint8_t *row1 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m1]][0]; - __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_packed1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); - - const uint8_t *row2 = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_packed2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); + const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes + [lsx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle = __lsx_vld(row, 1); + __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); // 5. store bytes - __lsx_vst(utf8_packed1, utf8_output, 0); - utf8_output += row1[0]; - - __lsx_vst(utf8_packed2, utf8_output, 0); - utf8_output += row2[0]; - - buf += 16; + __lsx_vst(utf8_packed, utf8_output, 0); + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; continue; } - __m256i surrogates_bytemask = __lasx_xvseq_h( - __lasx_xvand_v(in, lasx_splat_u16(0xf800)), lasx_splat_u16(0xd800)); + __m128i surrogates_bytemask = __lsx_vseq_h( + __lsx_vand_v(in, lsx_splat_u16(0xf800)), lsx_splat_u16(0xd800)); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lasx_xbz_v(surrogates_bytemask)) { + if (__lsx_bz_v(surrogates_bytemask)) { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes /* In this branch we handle three cases: 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - @@ -65372,94 +62134,74 @@ lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m256i t0 = __lasx_xvpickev_b(in, in); - t0 = __lasx_xvilvl_b(t0, t0); + __m128i t0 = __lsx_vpickev_b(in, in); + t0 = __lsx_vilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|00cc|cccc] - __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); - __m256i t1 = __lasx_xvand_v(t0, v_3f7f); + __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); + __m128i t1 = __lsx_vand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); + __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m256i s0 = __lasx_xvsrli_h(in, 12); + __m128i s0 = __lsx_vsrli_h(in, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m256i s1 = __lasx_xvslli_h(in, 2); + __m128i s1 = __lsx_vslli_h(in, 2); // s1: [aabb|bbbb|cccc|cc00] => [00bb|bbbb|0000|0000] - s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); + s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3f00)); // [00bb|bbbb|0000|aaaa] - __m256i s2 = __lasx_xvor_v(s0, s1); + __m128i s2 = __lsx_vor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); - __m256i s3 = __lasx_xvor_v(s2, v_c0e0); - __m256i one_or_two_bytes_bytemask = __lasx_xvsle_hu(in, v_07ff); - __m256i m0 = - __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); - __m256i s4 = __lasx_xvxor_v(s3, m0); + __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); + __m128i s3 = __lsx_vor_v(s2, v_c0e0); + __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(in, v_07ff); + __m128i m0 = + __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); + __m128i s4 = __lsx_vxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m256i out0 = __lasx_xvilvl_h(s4, t2); - __m256i out1 = __lasx_xvilvh_h(s4, t2); + __m128i out0 = __lsx_vilvl_h(s4, t2); + __m128i out1 = __lsx_vilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m256i one_byte_bytemask = __lasx_xvsle_hu(in, __lasx_xvrepli_h(0x7F)); - __m256i one_byte_bytemask_low = - __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); - __m256i one_byte_bytemask_high = - __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask = __lsx_vsle_hu(in, __lsx_vrepli_h(0x7F)); - __m256i one_or_two_bytes_bytemask_low = - __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); - __m256i one_or_two_bytes_bytemask_high = - __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_low = + __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_high = + __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); - __m256i mask0 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_low, one_byte_bytemask_low)); - __m256i mask1 = __lasx_xvmskltz_h(__lasx_xvor_v( - one_or_two_bytes_bytemask_high, one_byte_bytemask_high)); + __m128i one_byte_bytemask_low = + __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_high = + __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); + + const uint32_t mask0 = __lsx_vpickve2gr_bu( + __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_low, + one_byte_bytemask_low)), + 0); + const uint32_t mask1 = __lsx_vpickve2gr_bu( + __lsx_vmskltz_h(__lsx_vor_v(one_or_two_bytes_bytemask_high, + one_byte_bytemask_high)), + 0); - uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); - mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); + __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); + + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - mask = __lasx_xvpickve2gr_wu(mask0, 4); - const uint8_t *row2 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); - __lsx_vst(utf8_2, utf8_output, 0); - utf8_output += row2[0]; - - mask = __lasx_xvpickve2gr_wu(mask1, 4); - const uint8_t *row3 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle3 = __lsx_vld(row3, 1); - __m128i utf8_3 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); - __lsx_vst(utf8_3, utf8_output, 0); - utf8_output += row3[0]; - - buf += 16; + buf += 8; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -65471,8 +62213,7 @@ lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xFF80) == 0) { *utf8_output++ = char(word); } else if ((word & 0xF800) == 0) { @@ -65485,9 +62226,8 @@ lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -65509,65 +62249,38 @@ lasx_convert_utf16_to_utf8_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf8_output)); } -/* end file src/lasx/lasx_convert_utf16_to_utf8.cpp */ -#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +/* end file src/lsx/lsx_convert_utf16_to_utf8.cpp */ +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lasx/lasx_convert_utf16_to_utf32.cpp */ +/* begin file src/lsx/lsx_convert_utf16_to_utf32.cpp */ template std::pair -lasx_convert_utf16_to_utf32(const char16_t *buf, size_t len, - char32_t *utf32_out) { +lsx_convert_utf16_to_utf32(const char16_t *buf, size_t len, + char32_t *utf32_out) { uint32_t *utf32_output = reinterpret_cast(utf32_out); const char16_t *end = buf + len; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)utf32_output & 0x1f) && buf < end) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[0]) : buf[0]; - if ((word & 0xF800) != 0xD800) { - *utf32_output++ = char32_t(word); - buf++; - } else { - if (buf + 1 >= end) { - return std::make_pair(nullptr, - reinterpret_cast(utf32_output)); - } - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[1]) : buf[1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if ((diff | diff2) > 0x3FF) { - return std::make_pair(nullptr, - reinterpret_cast(utf32_output)); - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - *utf32_output++ = char32_t(value); - buf += 2; - } - } - - __m256i v_f800 = lasx_splat_u16(0xf800); - __m256i v_d800 = lasx_splat_u16(0xd800); + __m128i zero = __lsx_vldi(0); + __m128i v_f800 = lsx_splat_u16(0xf800); + __m128i v_d800 = lsx_splat_u16(0xd800); - while (end - buf >= 16) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lasx_swap_bytes(in); + while (end - buf >= 8) { + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lsx_swap_bytes(in); } - __m256i surrogates_bytemask = - __lasx_xvseq_h(__lasx_xvand_v(in, v_f800), v_d800); + __m128i surrogates_bytemask = + __lsx_vseq_h(__lsx_vand_v(in, v_f800), v_d800); // It might seem like checking for surrogates_bitmask == 0xc000 could help. // However, it is likely an uncommon occurrence. - if (__lasx_xbz_v(surrogates_bytemask)) { + if (__lsx_bz_v(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units - __m256i in_hi = __lasx_xvpermi_q(in, in, 0b00000001); - __lasx_xvst(__lasx_vext2xv_wu_hu(in), utf32_output, 0); - __lasx_xvst(__lasx_vext2xv_wu_hu(in_hi), utf32_output, 32); - utf32_output += 16; - buf += 16; + __lsx_vst(__lsx_vilvl_h(zero, in), utf32_output, 0); + __lsx_vst(__lsx_vilvh_h(zero, in), utf32_output, 16); + utf32_output += 8; + buf += 8; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -65579,16 +62292,14 @@ lasx_convert_utf16_to_utf32(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -65614,58 +62325,31 @@ lasx_convert_utf16_to_utf32(const char16_t *buf, size_t len, */ template std::pair -lasx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, - char32_t *utf32_out) { +lsx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, + char32_t *utf32_out) { uint32_t *utf32_output = reinterpret_cast(utf32_out); const char16_t *start = buf; const char16_t *end = buf + len; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)utf32_output & 0x1f) && buf < end) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[0]) : buf[0]; - if ((word & 0xF800) != 0xD800) { - *utf32_output++ = char32_t(word); - buf++; - } else if (buf + 1 < end) { - // must be a surrogate pair - uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[1]) : buf[1]; - uint16_t diff2 = uint16_t(next_word - 0xDC00); - if ((diff | diff2) > 0x3FF) { - return std::make_pair(result(error_code::SURROGATE, buf - start), - reinterpret_cast(utf32_output)); - } - uint32_t value = (diff << 10) + diff2 + 0x10000; - *utf32_output++ = char32_t(value); - buf += 2; - } else { - return std::make_pair(result(error_code::SURROGATE, buf - start), - reinterpret_cast(utf32_output)); - } - } + __m128i zero = __lsx_vldi(0); + __m128i v_f800 = lsx_splat_u16(0xf800); + __m128i v_d800 = lsx_splat_u16(0xd800); - __m256i v_f800 = lasx_splat_u16(0xf800); - __m256i v_d800 = lasx_splat_u16(0xd800); - while (end - buf >= 16) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - if (!match_system(big_endian)) { - in = lasx_swap_bytes(in); + while (end - buf >= 8) { + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + if constexpr (!match_system(big_endian)) { + in = lsx_swap_bytes(in); } - __m256i surrogates_bytemask = - __lasx_xvseq_h(__lasx_xvand_v(in, v_f800), v_d800); - // It might seem like checking for surrogates_bitmask == 0xc000 could help. - // However, it is likely an uncommon occurrence. - if (__lasx_xbz_v(surrogates_bytemask)) { + __m128i surrogates_bytemask = + __lsx_vseq_h(__lsx_vand_v(in, v_f800), v_d800); + if (__lsx_bz_v(surrogates_bytemask)) { // case: no surrogate pairs, extend all 16-bit code units to 32-bit code // units - __m256i in_hi = __lasx_xvpermi_q(in, in, 0b00000001); - __lasx_xvst(__lasx_vext2xv_wu_hu(in), utf32_output, 0); - __lasx_xvst(__lasx_vext2xv_wu_hu(in_hi), utf32_output, 32); - utf32_output += 16; - buf += 16; + __lsx_vst(__lsx_vilvl_h(zero, in), utf32_output, 0); + __lsx_vst(__lsx_vilvh_h(zero, in), utf32_output, 16); + utf32_output += 8; + buf += 8; // surrogate pair(s) in a register } else { // Let us do a scalar fallback. @@ -65677,16 +62361,14 @@ lasx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, forward = size_t(end - buf - 1); } for (; k < forward; k++) { - uint16_t word = - !match_system(big_endian) ? scalar::u16_swap_bytes(buf[k]) : buf[k]; + uint16_t word = scalar::utf16::swap_if_needed(buf[k]); if ((word & 0xF800) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = !match_system(big_endian) - ? scalar::u16_swap_bytes(buf[k + 1]) - : buf[k + 1]; + uint16_t next_word = + scalar::utf16::swap_if_needed(buf[k + 1]); k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if ((diff | diff2) > 0x3FF) { @@ -65704,35 +62386,31 @@ lasx_convert_utf16_to_utf32_with_errors(const char16_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf32_output)); } -/* end file src/lasx/lasx_convert_utf16_to_utf32.cpp */ +/* end file src/lsx/lsx_convert_utf16_to_utf32.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 -/* begin file src/lasx/lasx_convert_utf32_to_latin1.cpp */ +/* begin file src/lsx/lsx_convert_utf32_to_latin1.cpp */ std::pair -lasx_convert_utf32_to_latin1(const char32_t *buf, size_t len, - char *latin1_output) { +lsx_convert_utf32_to_latin1(const char32_t *buf, size_t len, + char *latin1_output) { const char32_t *end = buf + len; - const __m256i shuf_mask = ____m256i( - (__m128i)v16u8{0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}); - __m256i v_ff = __lasx_xvrepli_w(0xFF); + const v16u8 shuf_mask = {0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}; + __m128i v_ff = __lsx_vrepli_w(0xFF); while (end - buf >= 16) { - __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i in2 = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i in1 = __lsx_vld(reinterpret_cast(buf), 0); + __m128i in2 = __lsx_vld(reinterpret_cast(buf), 16); - __m256i in12 = __lasx_xvor_v(in1, in2); - if (__lasx_xbz_v(__lasx_xvslt_wu(v_ff, in12))) { + __m128i in12 = __lsx_vor_v(in1, in2); + if (__lsx_bz_v(__lsx_vslt_wu(v_ff, in12))) { // 1. pack the bytes - __m256i latin1_packed_tmp = __lasx_xvshuf_b(in2, in1, shuf_mask); - latin1_packed_tmp = __lasx_xvpermi_d(latin1_packed_tmp, 0b00001000); - __m128i latin1_packed = lasx_extracti128_lo(latin1_packed_tmp); - latin1_packed = __lsx_vpermi_w(latin1_packed, latin1_packed, 0b11011000); + __m128i latin1_packed = __lsx_vshuf_b(in2, in1, (__m128i)shuf_mask); // 2. store (8 bytes) __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); // 3. adjust pointers - buf += 16; - latin1_output += 16; + buf += 8; + latin1_output += 8; } else { return std::make_pair(nullptr, reinterpret_cast(latin1_output)); } @@ -65741,34 +62419,31 @@ lasx_convert_utf32_to_latin1(const char32_t *buf, size_t len, } std::pair -lasx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, - char *latin1_output) { +lsx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, + char *latin1_output) { const char32_t *start = buf; const char32_t *end = buf + len; - const __m256i shuf_mask = ____m256i( - (__m128i)v16u8{0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}); - __m256i v_ff = __lasx_xvrepli_w(0xFF); + const v16u8 shuf_mask = {0, 4, 8, 12, 16, 20, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0}; + __m128i v_ff = __lsx_vrepli_w(0xFF); while (end - buf >= 16) { - __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i in2 = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i in1 = __lsx_vld(reinterpret_cast(buf), 0); + __m128i in2 = __lsx_vld(reinterpret_cast(buf), 16); - __m256i in12 = __lasx_xvor_v(in1, in2); - if (__lasx_xbz_v(__lasx_xvslt_wu(v_ff, in12))) { + __m128i in12 = __lsx_vor_v(in1, in2); + + if (__lsx_bz_v(__lsx_vslt_wu(v_ff, in12))) { // 1. pack the bytes - __m256i latin1_packed_tmp = __lasx_xvshuf_b(in2, in1, shuf_mask); - latin1_packed_tmp = __lasx_xvpermi_d(latin1_packed_tmp, 0b00001000); - __m128i latin1_packed = lasx_extracti128_lo(latin1_packed_tmp); - latin1_packed = __lsx_vpermi_w(latin1_packed, latin1_packed, 0b11011000); + __m128i latin1_packed = __lsx_vshuf_b(in2, in1, (__m128i)shuf_mask); // 2. store (8 bytes) __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); // 3. adjust pointers - buf += 16; - latin1_output += 16; + buf += 8; + latin1_output += 8; } else { // Let us do a scalar fallback. - for (int k = 0; k < 16; k++) { + for (int k = 0; k < 8; k++) { uint32_t word = buf[k]; if (word <= 0xff) { *latin1_output++ = char(word); @@ -65782,253 +62457,188 @@ lasx_convert_utf32_to_latin1_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), latin1_output); } -/* end file src/lasx/lasx_convert_utf32_to_latin1.cpp */ +/* end file src/lsx/lsx_convert_utf32_to_latin1.cpp */ #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lasx/lasx_convert_utf32_to_utf8.cpp */ +/* begin file src/lsx/lsx_convert_utf32_to_utf8.cpp */ std::pair -lasx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { +lsx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char32_t *end = buf + len; - // load addr align 32 - while (((uint64_t)buf & 0x1F) && buf < end) { - uint32_t word = *buf; - if ((word & 0xFFFFFF80) == 0) { - *utf8_output++ = char(word); - } else if ((word & 0xFFFFF800) == 0) { - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } else if ((word & 0xFFFF0000) == 0) { - if (word >= 0xD800 && word <= 0xDFFF) { - return std::make_pair(nullptr, reinterpret_cast(utf8_output)); - } - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } else { - if (word > 0x10FFFF) { - return std::make_pair(nullptr, reinterpret_cast(utf8_output)); - } - *utf8_output++ = char((word >> 18) | 0b11110000); - *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } - buf++; - } - - __m256i v_c080 = lasx_splat_u16(0xc080); - __m256i v_07ff = lasx_splat_u16(0x07ff); - __m256i v_dfff = lasx_splat_u16(0xdfff); - __m256i v_d800 = lasx_splat_u16(0xd800); - __m256i zero = __lasx_xvldi(0); - __m128i zero_128 = __lsx_vldi(0); - __m256i forbidden_bytemask = __lasx_xvldi(0x0); + __m128i v_c080 = lsx_splat_u16(0xc080); + __m128i v_07ff = lsx_splat_u16(0x07ff); + __m128i v_dfff = lsx_splat_u16(0xdfff); + __m128i v_d800 = lsx_splat_u16(0xd800); + __m128i forbidden_bytemask = __lsx_vldi(0x0); const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 while (end - buf > std::ptrdiff_t(16 + safety_margin)) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i nextin = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); // Check if no bits set above 16th - if (__lasx_xbz_v(__lasx_xvpickod_h(in, nextin))) { + if (__lsx_bz_v(__lsx_vpickod_h(in, nextin))) { // Pack UTF-32 to UTF-16 safely (without surrogate pairs) - // Apply UTF-16 => UTF-8 routine (lasx_convert_utf16_to_utf8.cpp) - __m256i utf16_packed = - __lasx_xvpermi_d(__lasx_xvpickev_h(nextin, in), 0b11011000); + // Apply UTF-16 => UTF-8 routine (lsx_convert_utf16_to_utf8.cpp) + __m128i utf16_packed = __lsx_vpickev_h(nextin, in); - if (__lasx_xbz_v(__lasx_xvslt_hu(__lasx_xvrepli_h(0x7F), - utf16_packed))) { // ASCII fast path!!!! + if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), + utf16_packed))) { // ASCII fast path!!!! // 1. pack the bytes // obviously suboptimal. - __m256i utf8_packed = __lasx_xvpermi_d( - __lasx_xvpickev_b(utf16_packed, utf16_packed), 0b00001000); + __m128i utf8_packed = __lsx_vpickev_b(utf16_packed, utf16_packed); // 2. store (8 bytes) - __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); + __lsx_vst(utf8_packed, utf8_output, 0); // 3. adjust pointers - buf += 16; - utf8_output += 16; + buf += 8; + utf8_output += 8; continue; // we are done for this round! } - - if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, utf16_packed))) { + __m128i zero = __lsx_vldi(0); + if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, utf16_packed))) { // 1. prepare 2-byte values // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - const __m256i t0 = __lasx_xvslli_h(utf16_packed, 2); + const __m128i t0 = __lsx_vslli_h(utf16_packed, 2); // t1 = [000a|aaaa|0000|0000] - const __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); + const __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - const __m256i t2 = __lasx_xvand_v(utf16_packed, __lasx_xvrepli_h(0x3f)); + const __m128i t2 = __lsx_vand_v(utf16_packed, __lsx_vrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - const __m256i t3 = __lasx_xvor_v(t1, t2); + const __m128i t3 = __lsx_vor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - const __m256i t4 = __lasx_xvor_v(t3, v_c080); + const __m128i t4 = __lsx_vor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m256i one_byte_bytemask = - __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F /*0x007F*/)); - __m256i utf8_unpacked = - __lasx_xvbitsel_v(t4, utf16_packed, one_byte_bytemask); + __m128i one_byte_bytemask = + __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F /*0x007F*/)); + __m128i utf8_unpacked = + __lsx_vbitsel_v(t4, utf16_packed, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); - uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); + uint32_t m2 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); // 4. pack the bytes - const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m1]][0]; - __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_packed1 = __lsx_vshuf_b( - zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); - - const uint8_t *row2 = + const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_packed2 = __lsx_vshuf_b( - zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); + [lsx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle = __lsx_vld(row, 1); + __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); // 5. store bytes - __lsx_vst(utf8_packed1, utf8_output, 0); - utf8_output += row1[0]; - - __lsx_vst(utf8_packed2, utf8_output, 0); - utf8_output += row2[0]; + __lsx_vst(utf8_packed, utf8_output, 0); - buf += 16; + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; continue; } else { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes - forbidden_bytemask = __lasx_xvor_v( - __lasx_xvand_v( - __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + forbidden_bytemask = __lsx_vor_v( + __lsx_vand_v( + __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); /* In this branch we handle three cases: - 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - - single UFT-8 byte - 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - - two UTF-8 bytes - 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - - three UTF-8 bytes + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single + UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two + UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three + UTF-8 bytes - We expand the input word (16-bit) into two code units (32-bit), thus - we have room for four bytes. However, we need five distinct bit - layouts. Note that the last byte in cases #2 and #3 is the same. + We expand the input word (16-bit) into two code units (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. - We precompute byte 1 for case #1 and the common byte for cases #2 & - #3 in register t2. + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. - We precompute byte 1 for case #3 and -- **conditionally** -- - precompute either byte 1 for case #2 or byte 2 for case #3. Note that - they differ by exactly one bit. + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. - Finally from these two code units we build proper UTF-8 sequence, - taking into account the case (i.e, the number of bytes to write). - */ + Finally from these two code units we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ /** * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: * t2 => [0ccc|cccc] [10cc|cccc] * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m256i t0 = __lasx_xvpickev_b(utf16_packed, utf16_packed); - t0 = __lasx_xvilvl_b(t0, t0); + __m128i t0 = __lsx_vpickev_b(utf16_packed, utf16_packed); + t0 = __lsx_vilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] - __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); - __m256i t1 = __lasx_xvand_v(t0, v_3f7f); + __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); + __m128i t1 = __lsx_vand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); + __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m256i s0 = __lasx_xvsrli_h(utf16_packed, 12); + __m128i s0 = __lsx_vsrli_h(utf16_packed, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m256i s1 = __lasx_xvslli_h(utf16_packed, 2); + __m128i s1 = __lsx_vslli_h(utf16_packed, 2); // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000] - s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3f00)); + s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3F00)); // [00bb|bbbb|0000|aaaa] - __m256i s2 = __lasx_xvor_v(s0, s1); + __m128i s2 = __lsx_vor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); - __m256i s3 = __lasx_xvor_v(s2, v_c0e0); - // __m256i v_07ff = vmovq_n_u16((uint16_t)0x07FF); - __m256i one_or_two_bytes_bytemask = - __lasx_xvsle_hu(utf16_packed, v_07ff); - __m256i m0 = - __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); - __m256i s4 = __lasx_xvxor_v(s3, m0); + __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); + __m128i s3 = __lsx_vor_v(s2, v_c0e0); + __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(utf16_packed, v_07ff); + __m128i m0 = + __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); + __m128i s4 = __lsx_vxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m256i out0 = __lasx_xvilvl_h(s4, t2); - __m256i out1 = __lasx_xvilvh_h(s4, t2); + __m128i out0 = __lsx_vilvl_h(s4, t2); + __m128i out1 = __lsx_vilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m256i one_byte_bytemask = - __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F)); + __m128i one_byte_bytemask = + __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F)); - __m256i one_or_two_bytes_bytemask_u16_to_u32_low = - __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); - __m256i one_or_two_bytes_bytemask_u16_to_u32_high = - __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_u16_to_u32_low = + __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_u16_to_u32_high = + __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); - __m256i one_byte_bytemask_u16_to_u32_low = - __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); - __m256i one_byte_bytemask_u16_to_u32_high = - __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_u16_to_u32_low = + __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_u16_to_u32_high = + __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); - __m256i mask0 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_low, - one_byte_bytemask_u16_to_u32_low)); - __m256i mask1 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_high, - one_byte_bytemask_u16_to_u32_high)); + const uint32_t mask0 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( + one_or_two_bytes_bytemask_u16_to_u32_low, + one_byte_bytemask_u16_to_u32_low)), + 0); + const uint32_t mask1 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( + one_or_two_bytes_bytemask_u16_to_u32_high, + one_byte_bytemask_u16_to_u32_high)), + 0); - uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); - mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); + __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); + + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - mask = __lasx_xvpickve2gr_wu(mask0, 4); - const uint8_t *row2 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); - __lsx_vst(utf8_2, utf8_output, 0); - utf8_output += row2[0]; - - mask = __lasx_xvpickve2gr_wu(mask1, 4); - const uint8_t *row3 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle3 = __lsx_vld(row3, 1); - __m128i utf8_3 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); - __lsx_vst(utf8_3, utf8_output, 0); - utf8_output += row3[0]; - - buf += 16; + buf += 8; } // At least one 32-bit word will produce a surrogate pair in UTF-16 <=> // will produce four UTF-8 bytes. @@ -66072,262 +62682,197 @@ lasx_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_out) { } // while // check for invalid input - if (__lasx_xbnz_v(forbidden_bytemask)) { + if (__lsx_bnz_v(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf8_output)); } + return std::make_pair(buf, reinterpret_cast(utf8_output)); } std::pair -lasx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, - char *utf8_out) { +lsx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, + char *utf8_out) { uint8_t *utf8_output = reinterpret_cast(utf8_out); const char32_t *start = buf; const char32_t *end = buf + len; - // load addr align 32 - while (((uint64_t)buf & 0x1F) && buf < end) { - uint32_t word = *buf; - if ((word & 0xFFFFFF80) == 0) { - *utf8_output++ = char(word); - } else if ((word & 0xFFFFF800) == 0) { - *utf8_output++ = char((word >> 6) | 0b11000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } else if ((word & 0xFFFF0000) == 0) { - if (word >= 0xD800 && word <= 0xDFFF) { - return std::make_pair(result(error_code::SURROGATE, buf - start), - reinterpret_cast(utf8_output)); - } - *utf8_output++ = char((word >> 12) | 0b11100000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } else { - if (word > 0x10FFFF) { - return std::make_pair(result(error_code::TOO_LARGE, buf - start), - reinterpret_cast(utf8_output)); - } - *utf8_output++ = char((word >> 18) | 0b11110000); - *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); - *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); - *utf8_output++ = char((word & 0b111111) | 0b10000000); - } - buf++; - } - - __m256i v_c080 = lasx_splat_u16(0xc080); - __m256i v_07ff = lasx_splat_u16(0x07ff); - __m256i v_dfff = lasx_splat_u16(0xdfff); - __m256i v_d800 = lasx_splat_u16(0xd800); - __m256i zero = __lasx_xvldi(0); - __m128i zero_128 = __lsx_vldi(0); - __m256i forbidden_bytemask = __lasx_xvldi(0x0); + __m128i v_c080 = lsx_splat_u16(0xc080); + __m128i v_07ff = lsx_splat_u16(0x07ff); + __m128i v_dfff = lsx_splat_u16(0xdfff); + __m128i v_d800 = lsx_splat_u16(0xd800); + __m128i forbidden_bytemask = __lsx_vldi(0x0); const size_t safety_margin = 12; // to avoid overruns, see issue // https://github.com/simdutf/simdutf/issues/92 while (end - buf > std::ptrdiff_t(16 + safety_margin)) { - __m256i in = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i nextin = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i in = __lsx_vld(reinterpret_cast(buf), 0); + __m128i nextin = __lsx_vld(reinterpret_cast(buf), 16); // Check if no bits set above 16th - if (__lasx_xbz_v(__lasx_xvpickod_h(in, nextin))) { + if (__lsx_bz_v(__lsx_vpickod_h(in, nextin))) { // Pack UTF-32 to UTF-16 safely (without surrogate pairs) - // Apply UTF-16 => UTF-8 routine (lasx_convert_utf16_to_utf8.cpp) - __m256i utf16_packed = - __lasx_xvpermi_d(__lasx_xvpickev_h(nextin, in), 0b11011000); + // Apply UTF-16 => UTF-8 routine (lsx_convert_utf16_to_utf8.cpp) + __m128i utf16_packed = __lsx_vpickev_h(nextin, in); - if (__lasx_xbz_v(__lasx_xvslt_hu(__lasx_xvrepli_h(0x7F), - utf16_packed))) { // ASCII fast path!!!! + if (__lsx_bz_v(__lsx_vslt_hu(__lsx_vrepli_h(0x7F), + utf16_packed))) { // ASCII fast path!!!! // 1. pack the bytes // obviously suboptimal. - __m256i utf8_packed = __lasx_xvpermi_d( - __lasx_xvpickev_b(utf16_packed, utf16_packed), 0b00001000); + __m128i utf8_packed = __lsx_vpickev_b(utf16_packed, utf16_packed); // 2. store (8 bytes) - __lsx_vst(lasx_extracti128_lo(utf8_packed), utf8_output, 0); + __lsx_vst(utf8_packed, utf8_output, 0); // 3. adjust pointers - buf += 16; - utf8_output += 16; + buf += 8; + utf8_output += 8; continue; // we are done for this round! } - - if (__lasx_xbz_v(__lasx_xvslt_hu(v_07ff, utf16_packed))) { + __m128i zero = __lsx_vldi(0); + if (__lsx_bz_v(__lsx_vslt_hu(v_07ff, utf16_packed))) { // 1. prepare 2-byte values // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 // expected output : [110a|aaaa|10bb|bbbb] x 8 // t0 = [000a|aaaa|bbbb|bb00] - const __m256i t0 = __lasx_xvslli_h(utf16_packed, 2); + const __m128i t0 = __lsx_vslli_h(utf16_packed, 2); // t1 = [000a|aaaa|0000|0000] - const __m256i t1 = __lasx_xvand_v(t0, lasx_splat_u16(0x1f00)); + const __m128i t1 = __lsx_vand_v(t0, lsx_splat_u16(0x1f00)); // t2 = [0000|0000|00bb|bbbb] - const __m256i t2 = __lasx_xvand_v(utf16_packed, __lasx_xvrepli_h(0x3f)); + const __m128i t2 = __lsx_vand_v(utf16_packed, __lsx_vrepli_h(0x3f)); // t3 = [000a|aaaa|00bb|bbbb] - const __m256i t3 = __lasx_xvor_v(t1, t2); + const __m128i t3 = __lsx_vor_v(t1, t2); // t4 = [110a|aaaa|10bb|bbbb] - const __m256i t4 = __lasx_xvor_v(t3, v_c080); + const __m128i t4 = __lsx_vor_v(t3, v_c080); // 2. merge ASCII and 2-byte codewords - __m256i one_byte_bytemask = - __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F /*0x007F*/)); - __m256i utf8_unpacked = - __lasx_xvbitsel_v(t4, utf16_packed, one_byte_bytemask); + __m128i one_byte_bytemask = + __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F /*0x007F*/)); + __m128i utf8_unpacked = + __lsx_vbitsel_v(t4, utf16_packed, one_byte_bytemask); // 3. prepare bitmask for 8-bit lookup - __m256i mask = __lasx_xvmskltz_h(one_byte_bytemask); - uint32_t m1 = __lasx_xvpickve2gr_wu(mask, 0); - uint32_t m2 = __lasx_xvpickve2gr_wu(mask, 4); + uint32_t m2 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(one_byte_bytemask), 0); // 4. pack the bytes - const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m1]][0]; - __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_packed1 = __lsx_vshuf_b( - zero_128, lasx_extracti128_lo(utf8_unpacked), shuffle1); - - const uint8_t *row2 = + const uint8_t *row = &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes - [lasx_1_2_utf8_bytes_mask[m2]][0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_packed2 = __lsx_vshuf_b( - zero_128, lasx_extracti128_hi(utf8_unpacked), shuffle2); + [lsx_1_2_utf8_bytes_mask[m2]][0]; + __m128i shuffle = __lsx_vld(row, 1); + __m128i utf8_packed = __lsx_vshuf_b(zero, utf8_unpacked, shuffle); // 5. store bytes - __lsx_vst(utf8_packed1, utf8_output, 0); - utf8_output += row1[0]; - - __lsx_vst(utf8_packed2, utf8_output, 0); - utf8_output += row2[0]; + __lsx_vst(utf8_packed, utf8_output, 0); - buf += 16; + // 6. adjust pointers + buf += 8; + utf8_output += row[0]; continue; } else { // case: code units from register produce either 1, 2 or 3 UTF-8 bytes - forbidden_bytemask = __lasx_xvor_v( - __lasx_xvand_v( - __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + forbidden_bytemask = __lsx_vor_v( + __lsx_vand_v( + __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (__lasx_xbnz_v(forbidden_bytemask)) { + if (__lsx_bnz_v(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf8_output)); } /* In this branch we handle three cases: - 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - - single UFT-8 byte - 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - - two UTF-8 bytes - 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - - three UTF-8 bytes + 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - single + UFT-8 byte + 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two + UTF-8 bytes + 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - three + UTF-8 bytes - We expand the input word (16-bit) into two code units (32-bit), thus - we have room for four bytes. However, we need five distinct bit - layouts. Note that the last byte in cases #2 and #3 is the same. + We expand the input word (16-bit) into two code units (32-bit), thus + we have room for four bytes. However, we need five distinct bit + layouts. Note that the last byte in cases #2 and #3 is the same. - We precompute byte 1 for case #1 and the common byte for cases #2 & - #3 in register t2. + We precompute byte 1 for case #1 and the common byte for cases #2 & #3 + in register t2. - We precompute byte 1 for case #3 and -- **conditionally** -- - precompute either byte 1 for case #2 or byte 2 for case #3. Note that - they differ by exactly one bit. + We precompute byte 1 for case #3 and -- **conditionally** -- precompute + either byte 1 for case #2 or byte 2 for case #3. Note that they + differ by exactly one bit. - Finally from these two code units we build proper UTF-8 sequence, - taking into account the case (i.e, the number of bytes to write). - */ + Finally from these two code units we build proper UTF-8 sequence, taking + into account the case (i.e, the number of bytes to write). + */ /** * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: * t2 => [0ccc|cccc] [10cc|cccc] * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) */ // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] - __m256i t0 = __lasx_xvpickev_b(utf16_packed, utf16_packed); - t0 = __lasx_xvilvl_b(t0, t0); + __m128i t0 = __lsx_vpickev_b(utf16_packed, utf16_packed); + t0 = __lsx_vilvl_b(t0, t0); // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] - __m256i v_3f7f = __lasx_xvreplgr2vr_h(uint16_t(0x3F7F)); - __m256i t1 = __lasx_xvand_v(t0, v_3f7f); + __m128i v_3f7f = __lsx_vreplgr2vr_h(uint16_t(0x3F7F)); + __m128i t1 = __lsx_vand_v(t0, v_3f7f); // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] - __m256i t2 = __lasx_xvor_v(t1, lasx_splat_u16(0x8000)); + __m128i t2 = __lsx_vor_v(t1, lsx_splat_u16(0x8000)); // s0: [aaaa|bbbb|bbcc|cccc] => [0000|0000|0000|aaaa] - __m256i s0 = __lasx_xvsrli_h(utf16_packed, 12); + __m128i s0 = __lsx_vsrli_h(utf16_packed, 12); // s1: [aaaa|bbbb|bbcc|cccc] => [0000|bbbb|bb00|0000] - __m256i s1 = __lasx_xvslli_h(utf16_packed, 2); + __m128i s1 = __lsx_vslli_h(utf16_packed, 2); // [0000|bbbb|bb00|0000] => [00bb|bbbb|0000|0000] - s1 = __lasx_xvand_v(s1, lasx_splat_u16(0x3F00)); + s1 = __lsx_vand_v(s1, lsx_splat_u16(0x3F00)); // [00bb|bbbb|0000|aaaa] - __m256i s2 = __lasx_xvor_v(s0, s1); + __m128i s2 = __lsx_vor_v(s0, s1); // s3: [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] - __m256i v_c0e0 = __lasx_xvreplgr2vr_h(uint16_t(0xC0E0)); - __m256i s3 = __lasx_xvor_v(s2, v_c0e0); - // __m256i v_07ff = vmovq_n_u16((uint16_t)0x07FF); - __m256i one_or_two_bytes_bytemask = - __lasx_xvsle_hu(utf16_packed, v_07ff); - __m256i m0 = - __lasx_xvandn_v(one_or_two_bytes_bytemask, lasx_splat_u16(0x4000)); - __m256i s4 = __lasx_xvxor_v(s3, m0); + __m128i v_c0e0 = __lsx_vreplgr2vr_h(uint16_t(0xC0E0)); + __m128i s3 = __lsx_vor_v(s2, v_c0e0); + // __m128i v_07ff = vmovq_n_u16((uint16_t)0x07FF); + __m128i one_or_two_bytes_bytemask = __lsx_vsle_hu(utf16_packed, v_07ff); + __m128i m0 = + __lsx_vandn_v(one_or_two_bytes_bytemask, lsx_splat_u16(0x4000)); + __m128i s4 = __lsx_vxor_v(s3, m0); // 4. expand code units 16-bit => 32-bit - __m256i out0 = __lasx_xvilvl_h(s4, t2); - __m256i out1 = __lasx_xvilvh_h(s4, t2); + __m128i out0 = __lsx_vilvl_h(s4, t2); + __m128i out1 = __lsx_vilvh_h(s4, t2); // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle - __m256i one_byte_bytemask = - __lasx_xvsle_hu(utf16_packed, __lasx_xvrepli_h(0x7F)); + __m128i one_byte_bytemask = + __lsx_vsle_hu(utf16_packed, __lsx_vrepli_h(0x7F)); - __m256i one_or_two_bytes_bytemask_u16_to_u32_low = - __lasx_xvilvl_h(one_or_two_bytes_bytemask, zero); - __m256i one_or_two_bytes_bytemask_u16_to_u32_high = - __lasx_xvilvh_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_u16_to_u32_low = + __lsx_vilvl_h(one_or_two_bytes_bytemask, zero); + __m128i one_or_two_bytes_bytemask_u16_to_u32_high = + __lsx_vilvh_h(one_or_two_bytes_bytemask, zero); - __m256i one_byte_bytemask_u16_to_u32_low = - __lasx_xvilvl_h(one_byte_bytemask, one_byte_bytemask); - __m256i one_byte_bytemask_u16_to_u32_high = - __lasx_xvilvh_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_u16_to_u32_low = + __lsx_vilvl_h(one_byte_bytemask, one_byte_bytemask); + __m128i one_byte_bytemask_u16_to_u32_high = + __lsx_vilvh_h(one_byte_bytemask, one_byte_bytemask); - __m256i mask0 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_low, - one_byte_bytemask_u16_to_u32_low)); - __m256i mask1 = __lasx_xvmskltz_h( - __lasx_xvor_v(one_or_two_bytes_bytemask_u16_to_u32_high, - one_byte_bytemask_u16_to_u32_high)); + const uint32_t mask0 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( + one_or_two_bytes_bytemask_u16_to_u32_low, + one_byte_bytemask_u16_to_u32_low)), + 0); + const uint32_t mask1 = + __lsx_vpickve2gr_bu(__lsx_vmskltz_h(__lsx_vor_v( + one_or_two_bytes_bytemask_u16_to_u32_high, + one_byte_bytemask_u16_to_u32_high)), + 0); - uint32_t mask = __lasx_xvpickve2gr_wu(mask0, 0); const uint8_t *row0 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; __m128i shuffle0 = __lsx_vld(row0, 1); - __m128i utf8_0 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out0), shuffle0); - __lsx_vst(utf8_0, utf8_output, 0); - utf8_output += row0[0]; + __m128i utf8_0 = __lsx_vshuf_b(zero, out0, shuffle0); - mask = __lasx_xvpickve2gr_wu(mask1, 0); const uint8_t *row1 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; + &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; __m128i shuffle1 = __lsx_vld(row1, 1); - __m128i utf8_1 = - __lsx_vshuf_b(zero_128, lasx_extracti128_lo(out1), shuffle1); + __m128i utf8_1 = __lsx_vshuf_b(zero, out1, shuffle1); + + __lsx_vst(utf8_0, utf8_output, 0); + utf8_output += row0[0]; __lsx_vst(utf8_1, utf8_output, 0); utf8_output += row1[0]; - mask = __lasx_xvpickve2gr_wu(mask0, 4); - const uint8_t *row2 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle2 = __lsx_vld(row2, 1); - __m128i utf8_2 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out0), shuffle2); - __lsx_vst(utf8_2, utf8_output, 0); - utf8_output += row2[0]; - - mask = __lasx_xvpickve2gr_wu(mask1, 4); - const uint8_t *row3 = - &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask & 0xFF] - [0]; - __m128i shuffle3 = __lsx_vld(row3, 1); - __m128i utf8_3 = - __lsx_vshuf_b(zero_128, lasx_extracti128_hi(out1), shuffle3); - __lsx_vst(utf8_3, utf8_output, 0); - utf8_output += row3[0]; - - buf += 16; + buf += 8; } // At least one 32-bit word will produce a surrogate pair in UTF-16 <=> // will produce four UTF-8 bytes. @@ -66375,74 +62920,41 @@ lasx_convert_utf32_to_utf8_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf8_output)); } -/* end file src/lasx/lasx_convert_utf32_to_utf8.cpp */ +/* end file src/lsx/lsx_convert_utf32_to_utf8.cpp */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -/* begin file src/lasx/lasx_convert_utf32_to_utf16.cpp */ +/* begin file src/lsx/lsx_convert_utf32_to_utf16.cpp */ template std::pair -lasx_convert_utf32_to_utf16(const char32_t *buf, size_t len, - char16_t *utf16_out) { +lsx_convert_utf32_to_utf16(const char32_t *buf, size_t len, + char16_t *utf16_out) { uint16_t *utf16_output = reinterpret_cast(utf16_out); const char32_t *end = buf + len; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)utf16_output & 0x1F) && buf < end) { - uint32_t word = *buf++; - if ((word & 0xFFFF0000) == 0) { - // will not generate a surrogate pair - if (word >= 0xD800 && word <= 0xDFFF) { - return std::make_pair(nullptr, - reinterpret_cast(utf16_output)); - } - *utf16_output++ = !match_system(big_endian) - ? char16_t(word >> 8 | word << 8) - : char16_t(word); - // buf++; - } else { - // will generate a surrogate pair - if (word > 0x10FFFF) { - return std::make_pair(nullptr, - reinterpret_cast(utf16_output)); - } - word -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); - low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - // buf++; - } - } - - __m256i forbidden_bytemask = __lasx_xvrepli_h(0); - __m256i v_d800 = lasx_splat_u16(0xd800); - __m256i v_dfff = lasx_splat_u16(0xdfff); - while (end - buf >= 16) { - __m256i in0 = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i forbidden_bytemask = __lsx_vrepli_h(0); + __m128i v_d800 = lsx_splat_u16(0xd800); + __m128i v_dfff = lsx_splat_u16(0xdfff); + while (end - buf >= 8) { + __m128i in0 = __lsx_vld(reinterpret_cast(buf), 0); + __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); // Check if no bits set above 16th - if (__lasx_xbz_v(__lasx_xvpickod_h(in1, in0))) { - __m256i utf16_packed = - __lasx_xvpermi_d(__lasx_xvpickev_h(in1, in0), 0b11011000); - forbidden_bytemask = __lasx_xvor_v( - __lasx_xvand_v( - __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + if (__lsx_bz_v(__lsx_vpickod_h(in1, in0))) { + __m128i utf16_packed = __lsx_vpickev_h(in1, in0); + forbidden_bytemask = __lsx_vor_v( + __lsx_vand_v( + __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (!match_system(big_endian)) { - utf16_packed = lasx_swap_bytes(utf16_packed); + if constexpr (!match_system(big_endian)) { + utf16_packed = lsx_swap_bytes(utf16_packed); } - __lasx_xvst(utf16_packed, utf16_output, 0); - utf16_output += 16; - buf += 16; + __lsx_vst(utf16_packed, utf16_output, 0); + utf16_output += 8; + buf += 8; } else { - size_t forward = 15; + size_t forward = 3; size_t k = 0; if (size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1); @@ -66467,7 +62979,7 @@ lasx_convert_utf32_to_utf16(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); @@ -66481,7 +62993,7 @@ lasx_convert_utf32_to_utf16(const char32_t *buf, size_t len, } // check for invalid input - if (__lasx_xbnz_v(forbidden_bytemask)) { + if (__lsx_bnz_v(forbidden_bytemask)) { return std::make_pair(nullptr, reinterpret_cast(utf16_output)); } return std::make_pair(buf, reinterpret_cast(utf16_output)); @@ -66489,72 +63001,42 @@ lasx_convert_utf32_to_utf16(const char32_t *buf, size_t len, template std::pair -lasx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, - char16_t *utf16_out) { +lsx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, + char16_t *utf16_out) { uint16_t *utf16_output = reinterpret_cast(utf16_out); const char32_t *start = buf; const char32_t *end = buf + len; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)utf16_output & 0x1F) && buf < end) { - uint32_t word = *buf++; - if ((word & 0xFFFF0000) == 0) { - // will not generate a surrogate pair - if (word >= 0xD800 && word <= 0xDFFF) { - return std::make_pair(result(error_code::SURROGATE, buf - start - 1), - reinterpret_cast(utf16_output)); - } - *utf16_output++ = !match_system(big_endian) - ? char16_t(word >> 8 | word << 8) - : char16_t(word); - } else { - // will generate a surrogate pair - if (word > 0x10FFFF) { - return std::make_pair(result(error_code::TOO_LARGE, buf - start - 1), - reinterpret_cast(utf16_output)); - } - word -= 0x10000; - uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); - uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { - high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); - low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); - } - *utf16_output++ = char16_t(high_surrogate); - *utf16_output++ = char16_t(low_surrogate); - } - } - - __m256i forbidden_bytemask = __lasx_xvrepli_h(0); - __m256i v_d800 = lasx_splat_u16(0xd800); - __m256i v_dfff = lasx_splat_u16(0xdfff); - while (end - buf >= 16) { - __m256i in0 = __lasx_xvld(reinterpret_cast(buf), 0); - __m256i in1 = __lasx_xvld(reinterpret_cast(buf), 32); + __m128i forbidden_bytemask = __lsx_vrepli_h(0); + __m128i v_d800 = lsx_splat_u16(0xd800); + __m128i v_dfff = lsx_splat_u16(0xdfff); + while (end - buf >= 8) { + __m128i in0 = __lsx_vld(reinterpret_cast(buf), 0); + __m128i in1 = __lsx_vld(reinterpret_cast(buf), 16); // Check if no bits set above 16th - if (__lasx_xbz_v(__lasx_xvpickod_h(in1, in0))) { - __m256i utf16_packed = - __lasx_xvpermi_d(__lasx_xvpickev_h(in1, in0), 0b11011000); - forbidden_bytemask = __lasx_xvor_v( - __lasx_xvand_v( - __lasx_xvsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff - __lasx_xvsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 + if (__lsx_bz_v(__lsx_vpickod_h(in1, in0))) { + __m128i utf16_packed = __lsx_vpickev_h(in1, in0); + + forbidden_bytemask = __lsx_vor_v( + __lsx_vand_v( + __lsx_vsle_h(utf16_packed, v_dfff), // utf16_packed <= 0xdfff + __lsx_vsle_h(v_d800, utf16_packed)), // utf16_packed >= 0xd800 forbidden_bytemask); - if (__lasx_xbnz_v(forbidden_bytemask)) { + if (__lsx_bnz_v(forbidden_bytemask)) { return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast(utf16_output)); } - if (!match_system(big_endian)) { - utf16_packed = lasx_swap_bytes(utf16_packed); + if constexpr (!match_system(big_endian)) { + utf16_packed = lsx_swap_bytes(utf16_packed); } - __lasx_xvst(utf16_packed, utf16_output, 0); - utf16_output += 16; - buf += 16; + __lsx_vst(utf16_packed, utf16_output, 0); + utf16_output += 8; + buf += 8; } else { - size_t forward = 15; + size_t forward = 3; size_t k = 0; if (size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1); @@ -66581,7 +63063,7 @@ lasx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); @@ -66597,10 +63079,10 @@ lasx_convert_utf32_to_utf16_with_errors(const char32_t *buf, size_t len, return std::make_pair(result(error_code::SUCCESS, buf - start), reinterpret_cast(utf16_output)); } -/* end file src/lasx/lasx_convert_utf32_to_utf16.cpp */ +/* end file src/lsx/lsx_convert_utf32_to_utf16.cpp */ #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 #if SIMDUTF_FEATURE_BASE64 -/* begin file src/lasx/lasx_base64.cpp */ +/* begin file src/lsx/lsx_base64.cpp */ /** * References and further reading: * @@ -66641,118 +63123,100 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, : "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; uint8_t *out = (uint8_t *)dst; - v32u8 shuf; - __m256i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, + v16u8 shuf; + __m128i v_fc0fc00, v_3f03f0, shift_r, shift_l, base64_tbl0, base64_tbl1, base64_tbl2, base64_tbl3; - if (srclen >= 28) { - shuf = v32u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, - 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; - - v_fc0fc00 = __lasx_xvreplgr2vr_w(uint32_t(0x0fc0fc00)); - v_3f03f0 = __lasx_xvreplgr2vr_w(uint32_t(0x003f03f0)); - shift_r = __lasx_xvreplgr2vr_w(uint32_t(0x0006000a)); - shift_l = __lasx_xvreplgr2vr_w(uint32_t(0x00080004)); - base64_tbl0 = ____m256i(__lsx_vld(lookup_tbl, 0)); - base64_tbl1 = ____m256i(__lsx_vld(lookup_tbl, 16)); - base64_tbl2 = ____m256i(__lsx_vld(lookup_tbl, 32)); - base64_tbl3 = ____m256i(__lsx_vld(lookup_tbl, 48)); + if (srclen >= 16) { + shuf = v16u8{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10}; + v_fc0fc00 = __lsx_vreplgr2vr_w(uint32_t(0x0fc0fc00)); + v_3f03f0 = __lsx_vreplgr2vr_w(uint32_t(0x003f03f0)); + shift_r = __lsx_vreplgr2vr_w(uint32_t(0x0006000a)); + shift_l = __lsx_vreplgr2vr_w(uint32_t(0x00080004)); + base64_tbl0 = __lsx_vld(lookup_tbl, 0); + base64_tbl1 = __lsx_vld(lookup_tbl, 16); + base64_tbl2 = __lsx_vld(lookup_tbl, 32); + base64_tbl3 = __lsx_vld(lookup_tbl, 48); } + size_t i = 0; - for (; i + 100 <= srclen; i += 96) { - __m128i in0_lo = + for (; i + 52 <= srclen; i += 48) { + __m128i in0 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 0); - __m128i in0_hi = + __m128i in1 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); - __m128i in1_lo = + __m128i in2 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 2); - __m128i in1_hi = + __m128i in3 = __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 3); - __m128i in2_lo = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 4); - __m128i in2_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 5); - __m128i in3_lo = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 6); - __m128i in3_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 7); - - __m256i in0 = lasx_set_q(in0_hi, in0_lo); - __m256i in1 = lasx_set_q(in1_hi, in1_lo); - __m256i in2 = lasx_set_q(in2_hi, in2_lo); - __m256i in3 = lasx_set_q(in3_hi, in3_lo); - in0 = __lasx_xvshuf_b(in0, in0, (__m256i)shuf); - in1 = __lasx_xvshuf_b(in1, in1, (__m256i)shuf); - in2 = __lasx_xvshuf_b(in2, in2, (__m256i)shuf); - in3 = __lasx_xvshuf_b(in3, in3, (__m256i)shuf); + in0 = __lsx_vshuf_b(in0, in0, (__m128i)shuf); + in1 = __lsx_vshuf_b(in1, in1, (__m128i)shuf); + in2 = __lsx_vshuf_b(in2, in2, (__m128i)shuf); + in3 = __lsx_vshuf_b(in3, in3, (__m128i)shuf); - __m256i t0_0 = __lasx_xvand_v(in0, v_fc0fc00); - __m256i t0_1 = __lasx_xvand_v(in1, v_fc0fc00); - __m256i t0_2 = __lasx_xvand_v(in2, v_fc0fc00); - __m256i t0_3 = __lasx_xvand_v(in3, v_fc0fc00); + __m128i t0_0 = __lsx_vand_v(in0, v_fc0fc00); + __m128i t0_1 = __lsx_vand_v(in1, v_fc0fc00); + __m128i t0_2 = __lsx_vand_v(in2, v_fc0fc00); + __m128i t0_3 = __lsx_vand_v(in3, v_fc0fc00); - __m256i t1_0 = __lasx_xvsrl_h(t0_0, shift_r); - __m256i t1_1 = __lasx_xvsrl_h(t0_1, shift_r); - __m256i t1_2 = __lasx_xvsrl_h(t0_2, shift_r); - __m256i t1_3 = __lasx_xvsrl_h(t0_3, shift_r); + __m128i t1_0 = __lsx_vsrl_h(t0_0, shift_r); + __m128i t1_1 = __lsx_vsrl_h(t0_1, shift_r); + __m128i t1_2 = __lsx_vsrl_h(t0_2, shift_r); + __m128i t1_3 = __lsx_vsrl_h(t0_3, shift_r); - __m256i t2_0 = __lasx_xvand_v(in0, v_3f03f0); - __m256i t2_1 = __lasx_xvand_v(in1, v_3f03f0); - __m256i t2_2 = __lasx_xvand_v(in2, v_3f03f0); - __m256i t2_3 = __lasx_xvand_v(in3, v_3f03f0); + __m128i t2_0 = __lsx_vand_v(in0, v_3f03f0); + __m128i t2_1 = __lsx_vand_v(in1, v_3f03f0); + __m128i t2_2 = __lsx_vand_v(in2, v_3f03f0); + __m128i t2_3 = __lsx_vand_v(in3, v_3f03f0); - __m256i t3_0 = __lasx_xvsll_h(t2_0, shift_l); - __m256i t3_1 = __lasx_xvsll_h(t2_1, shift_l); - __m256i t3_2 = __lasx_xvsll_h(t2_2, shift_l); - __m256i t3_3 = __lasx_xvsll_h(t2_3, shift_l); + __m128i t3_0 = __lsx_vsll_h(t2_0, shift_l); + __m128i t3_1 = __lsx_vsll_h(t2_1, shift_l); + __m128i t3_2 = __lsx_vsll_h(t2_2, shift_l); + __m128i t3_3 = __lsx_vsll_h(t2_3, shift_l); - __m256i input0 = __lasx_xvor_v(t1_0, t3_0); - __m256i input0_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input0); - __m256i input0_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input0, __lasx_xvldi(32))); - __m256i input0_mask = __lasx_xvslei_bu(input0, 31); - __m256i input0_result = - __lasx_xvbitsel_v(input0_shuf1, input0_shuf0, input0_mask); - __lasx_xvst(input0_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input0 = __lsx_vor_v(t1_0, t3_0); + __m128i input0_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input0); + __m128i input0_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input0, __lsx_vldi(32))); + __m128i input0_mask = __lsx_vslei_bu(input0, 31); + __m128i input0_result = + __lsx_vbitsel_v(input0_shuf1, input0_shuf0, input0_mask); + __lsx_vst(input0_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m256i input1 = __lasx_xvor_v(t1_1, t3_1); - __m256i input1_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input1); - __m256i input1_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input1, __lasx_xvldi(32))); - __m256i input1_mask = __lasx_xvslei_bu(input1, 31); - __m256i input1_result = - __lasx_xvbitsel_v(input1_shuf1, input1_shuf0, input1_mask); - __lasx_xvst(input1_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input1 = __lsx_vor_v(t1_1, t3_1); + __m128i input1_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input1); + __m128i input1_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input1, __lsx_vldi(32))); + __m128i input1_mask = __lsx_vslei_bu(input1, 31); + __m128i input1_result = + __lsx_vbitsel_v(input1_shuf1, input1_shuf0, input1_mask); + __lsx_vst(input1_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m256i input2 = __lasx_xvor_v(t1_2, t3_2); - __m256i input2_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input2); - __m256i input2_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input2, __lasx_xvldi(32))); - __m256i input2_mask = __lasx_xvslei_bu(input2, 31); - __m256i input2_result = - __lasx_xvbitsel_v(input2_shuf1, input2_shuf0, input2_mask); - __lasx_xvst(input2_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input2 = __lsx_vor_v(t1_2, t3_2); + __m128i input2_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input2); + __m128i input2_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input2, __lsx_vldi(32))); + __m128i input2_mask = __lsx_vslei_bu(input2, 31); + __m128i input2_result = + __lsx_vbitsel_v(input2_shuf1, input2_shuf0, input2_mask); + __lsx_vst(input2_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; - __m256i input3 = __lasx_xvor_v(t1_3, t3_3); - __m256i input3_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, input3); - __m256i input3_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(input3, __lasx_xvldi(32))); - __m256i input3_mask = __lasx_xvslei_bu(input3, 31); - __m256i input3_result = - __lasx_xvbitsel_v(input3_shuf1, input3_shuf0, input3_mask); - __lasx_xvst(input3_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i input3 = __lsx_vor_v(t1_3, t3_3); + __m128i input3_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, input3); + __m128i input3_shuf1 = __lsx_vshuf_b(base64_tbl3, base64_tbl2, + __lsx_vsub_b(input3, __lsx_vldi(32))); + __m128i input3_mask = __lsx_vslei_bu(input3, 31); + __m128i input3_result = + __lsx_vbitsel_v(input3_shuf1, input3_shuf0, input3_mask); + __lsx_vst(input3_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; } - for (; i + 28 <= srclen; i += 24) { - - __m128i in_lo = __lsx_vld(reinterpret_cast(input + i), 0); - __m128i in_hi = - __lsx_vld(reinterpret_cast(input + i), 4 * 3 * 1); + for (; i + 16 <= srclen; i += 12) { - __m256i in = lasx_set_q(in_hi, in_lo); + __m128i in = __lsx_vld(reinterpret_cast(input + i), 0); // bytes from groups A, B and C are needed in separate 32-bit lanes // in = [DDDD|CCCC|BBBB|AAAA] @@ -66766,32 +63230,33 @@ size_t encode_base64(char *dst, const char *src, size_t srclen, // [bbbbcccc|ccdddddd|aaaaaabb|bbbbcccc] // ^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ // processed bits - in = __lasx_xvshuf_b(in, in, (__m256i)shuf); + in = __lsx_vshuf_b(in, in, (__m128i)shuf); // unpacking // t0 = [0000cccc|cc000000|aaaaaa00|00000000] - __m256i t0 = __lasx_xvand_v(in, v_fc0fc00); + __m128i t0 = __lsx_vand_v(in, v_fc0fc00); // t1 = [00000000|00cccccc|00000000|00aaaaaa] // ((c >> 6), (a >> 10)) - __m256i t1 = __lasx_xvsrl_h(t0, shift_r); + __m128i t1 = __lsx_vsrl_h(t0, shift_r); // t2 = [00000000|00dddddd|000000bb|bbbb0000] - __m256i t2 = __lasx_xvand_v(in, v_3f03f0); + __m128i t2 = __lsx_vand_v(in, v_3f03f0); // t3 = [00dddddd|00000000|00bbbbbb|00000000] // ((d << 8), (b << 4)) - __m256i t3 = __lasx_xvsll_h(t2, shift_l); + __m128i t3 = __lsx_vsll_h(t2, shift_l); // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 - __m256i indices = __lasx_xvor_v(t1, t3); + __m128i indices = __lsx_vor_v(t1, t3); - __m256i indices_shuf0 = __lasx_xvshuf_b(base64_tbl1, base64_tbl0, indices); - __m256i indices_shuf1 = __lasx_xvshuf_b( - base64_tbl3, base64_tbl2, __lasx_xvsub_b(indices, __lasx_xvldi(32))); - __m256i indices_mask = __lasx_xvslei_bu(indices, 31); - __m256i indices_result = - __lasx_xvbitsel_v(indices_shuf1, indices_shuf0, indices_mask); - __lasx_xvst(indices_result, reinterpret_cast<__m256i *>(out), 0); - out += 32; + __m128i indices_shuf0 = __lsx_vshuf_b(base64_tbl1, base64_tbl0, indices); + __m128i indices_shuf1 = __lsx_vshuf_b( + base64_tbl3, base64_tbl2, __lsx_vsub_b(indices, __lsx_vldi(32))); + __m128i indices_mask = __lsx_vslei_bu(indices, 31); + __m128i indices_result = + __lsx_vbitsel_v(indices_shuf1, indices_shuf0, indices_mask); + + __lsx_vst(indices_result, reinterpret_cast<__m128i *>(out), 0); + out += 16; } return i / 3 * 4 + scalar::base64::tail_encode_base64((char *)out, src + i, @@ -66815,7 +63280,7 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { tables::base64::thintable_epi8[mask2]}; // we increment by 0x08 the second half of the mask - const v4u32 hi = {0, 0, 0x08080808, 0x08080808}; + v4u32 hi = {0, 0, 0x08080808, 0x08080808}; __m128i shufmask1 = __lsx_vadd_b((__m128i)shufmask, (__m128i)hi); // this is the version "nearly pruned" @@ -66837,111 +63302,115 @@ static inline void compress(__m128i data, uint16_t mask, char *output) { } struct block64 { - __m256i chunks[2]; + __m128i chunks[4]; }; template -static inline uint32_t to_base64_mask(__m256i *src, bool *error) { - __m256i ascii_space_tbl = - ____m256i((__m128i)v16u8{0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}); +static inline uint16_t to_base64_mask(__m128i *src, bool *error) { + const v16u8 ascii_space_tbl = {0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0}; // credit: aqrit - __m256i delta_asso; + /* + '0'(0x30)-'9'(0x39) => delta_values_index = 4 + 'A'(0x41)-'Z'(0x5a) => delta_values_index = 4/5/12(4+8) + 'a'(0x61)-'z'(0x7a) => delta_values_index = 6/7/14(6+8) + '+'(0x2b) => delta_values_index = 3 + '/'(0x2f) => delta_values_index = 2+8 = 10 + '-'(0x2d) => delta_values_index = 2+8 = 10 + '_'(0x5f) => delta_values_index = 5+8 = 13 + */ + v16u8 delta_asso; if (default_or_url) { - delta_asso = - ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x11, 0x0, 0x16}); + delta_asso = v16u8{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16}; } else { - delta_asso = - ____m256i((__m128i)v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, - 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}); + delta_asso = v16u8{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, + 0x0, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF}; } - __m256i delta_values; + v16i8 delta_values; if (default_or_url) { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), - int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0xFF), int8_t(0x11), + int8_t(0xFF), int8_t(0xBF), int8_t(0x10), int8_t(0xB9)}; } else if (base64_url) { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), - int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x11), int8_t(0xC3), + int8_t(0xBF), int8_t(0xE0), int8_t(0xB9), int8_t(0xB9)}; } else { - delta_values = ____m256i( - (__m128i)v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), - int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), - int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), - int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}); + delta_values = + v16i8{int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), + int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), + int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), + int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)}; } - __m256i check_asso; + v16u8 check_asso; if (default_or_url) { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x0E, 0x0B, 0x06}); - - } else if (base64_url) { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x06, 0x0B, 0x12}); + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06}; + } else if (base64_url) { + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x06, 0x0B, 0x12}; } else { - check_asso = ____m256i((__m128i)v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, - 0x0B, 0x0B, 0x0B, 0x0F}); + check_asso = v16u8{0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F}; } - __m256i check_values; + v16i8 check_values; if (default_or_url) { - - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), - int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}); + check_values = + v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0xA1), int8_t(0x00), int8_t(0x80), + int8_t(0x00), int8_t(0x80), int8_t(0x00), int8_t(0x80)}; } else if (base64_url) { - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), - int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}); + check_values = v16i8{int8_t(0x0), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD3), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD0), int8_t(0x80), + int8_t(0xB0), int8_t(0x80), int8_t(0x0), int8_t(0x0)}; } else { - check_values = ____m256i( - (__m128i)v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), - int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), - int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), - int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}); + check_values = + v16i8{int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), + int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), + int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), + int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)}; } - __m256i shifted = __lasx_xvsrli_b(*src, 3); - __m256i asso_index = __lasx_xvand_v(*src, __lasx_xvldi(0xF)); - __m256i delta_hash = __lasx_xvavgr_bu( - __lasx_xvshuf_b(delta_asso, delta_asso, asso_index), shifted); - __m256i check_hash = __lasx_xvavgr_bu( - __lasx_xvshuf_b(check_asso, check_asso, asso_index), shifted); + const __m128i shifted = __lsx_vsrli_b(*src, 3); + __m128i asso_index = __lsx_vand_v(*src, __lsx_vldi(0xF)); + const __m128i delta_hash = + __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)delta_asso, (__m128i)delta_asso, + (__m128i)asso_index), + shifted); + const __m128i check_hash = + __lsx_vavgr_bu(__lsx_vshuf_b((__m128i)check_asso, (__m128i)check_asso, + (__m128i)asso_index), + shifted); - __m256i out = __lasx_xvsadd_b( - __lasx_xvshuf_b(delta_values, delta_values, delta_hash), *src); - __m256i chk = __lasx_xvsadd_b( - __lasx_xvshuf_b(check_values, check_values, check_hash), *src); - __m256i chk_ltz = __lasx_xvmskltz_b(chk); - unsigned int mask = __lasx_xvpickve2gr_wu(chk_ltz, 0); - mask = mask | (__lsx_vpickve2gr_hu(lasx_extracti128_hi(chk_ltz), 0) << 16); + const __m128i out = + __lsx_vsadd_b(__lsx_vshuf_b((__m128i)delta_values, (__m128i)delta_values, + (__m128i)delta_hash), + *src); + const __m128i chk = + __lsx_vsadd_b(__lsx_vshuf_b((__m128i)check_values, (__m128i)check_values, + (__m128i)check_hash), + *src); + unsigned int mask = __lsx_vpickve2gr_hu(__lsx_vmskltz_b(chk), 0); if (mask) { - __m256i ascii_space = __lasx_xvseq_b( - __lasx_xvshuf_b(ascii_space_tbl, ascii_space_tbl, asso_index), *src); - __m256i ascii_space_ltz = __lasx_xvmskltz_b(ascii_space); - unsigned int ascii_space_mask = __lasx_xvpickve2gr_wu(ascii_space_ltz, 0); - ascii_space_mask = - ascii_space_mask | - (__lsx_vpickve2gr_hu(lasx_extracti128_hi(ascii_space_ltz), 0) << 16); - *error |= (mask != ascii_space_mask); + __m128i ascii_space = __lsx_vseq_b(__lsx_vshuf_b((__m128i)ascii_space_tbl, + (__m128i)ascii_space_tbl, + (__m128i)asso_index), + *src); + *error |= + (mask != __lsx_vpickve2gr_hu(__lsx_vmskltz_b((__m128i)ascii_space), 0)); } *src = out; - return (uint32_t)mask; + return (uint16_t)mask; } template @@ -66951,12 +63420,18 @@ static inline uint64_t to_base64_mask(block64 *b, bool *error) { to_base64_mask(&b->chunks[0], error); uint64_t m1 = to_base64_mask(&b->chunks[1], error); - return m0 | (m1 << 32); + uint64_t m2 = + to_base64_mask(&b->chunks[2], error); + uint64_t m3 = + to_base64_mask(&b->chunks[3], error); + return m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); } static inline void copy_block(block64 *b, char *output) { - __lasx_xvst(b->chunks[0], reinterpret_cast<__m256i *>(output), 0); - __lasx_xvst(b->chunks[1], reinterpret_cast<__m256i *>(output), 32); + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output), 16); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output), 32); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output), 48); } static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { @@ -66964,80 +63439,152 @@ static inline uint64_t compress_block(block64 *b, uint64_t mask, char *output) { uint64_t count = __lsx_vpickve2gr_d(__lsx_vpcnt_h(__lsx_vreplgr2vr_d(nmask)), 0); uint16_t *count_ptr = (uint16_t *)&count; - compress(lasx_extracti128_lo(b->chunks[0]), uint16_t(mask), output); - compress(lasx_extracti128_hi(b->chunks[0]), uint16_t(mask >> 16), - output + count_ptr[0]); - compress(lasx_extracti128_lo(b->chunks[1]), uint16_t(mask >> 32), + compress(b->chunks[0], uint16_t(mask), output); + compress(b->chunks[1], uint16_t(mask >> 16), output + count_ptr[0]); + compress(b->chunks[2], uint16_t(mask >> 32), output + count_ptr[0] + count_ptr[1]); - compress(lasx_extracti128_hi(b->chunks[1]), uint16_t(mask >> 48), + compress(b->chunks[3], uint16_t(mask >> 48), output + count_ptr[0] + count_ptr[1] + count_ptr[2]); return count_ones(nmask); } +template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } + +inline size_t compress_block_single(block64 *b, uint64_t mask, char *output) { + const size_t pos64 = trailing_zeroes(mask); + const int8_t pos = pos64 & 0xf; + // Predefine the index vector + const v16u8 v1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + switch (pos64 >> 4) { + case 0b00: { + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); // v1 > v0 + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[0], b->chunks[0], sh); + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16 - 1), + 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16 - 1), + 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b01: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[1], b->chunks[1], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16 - 1), + 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b10: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[2], b->chunks[2], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 2 * 16), 0); + __lsx_vst(b->chunks[3], reinterpret_cast<__m128i *>(output + 3 * 16 - 1), + 0); + } break; + + case 0b11: { + __lsx_vst(b->chunks[0], reinterpret_cast<__m128i *>(output + 0 * 16), 0); + __lsx_vst(b->chunks[1], reinterpret_cast<__m128i *>(output + 1 * 16), 0); + __lsx_vst(b->chunks[2], reinterpret_cast<__m128i *>(output + 2 * 16), 0); + + const __m128i v0 = __lsx_vreplgr2vr_b((uint8_t)(pos - 1)); + const __m128i v2 = __lsx_vslt_b(v0, (__m128i)v1); + const __m128i sh = __lsx_vsub_b((__m128i)v1, v2); + const __m128i compressed = __lsx_vshuf_b(b->chunks[3], b->chunks[3], sh); + + __lsx_vst(compressed, reinterpret_cast<__m128i *>(output + 3 * 16), 0); + } break; + } + return 63; +} + // The caller of this function is responsible to ensure that there are 64 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char *src) { - b->chunks[0] = __lasx_xvld(reinterpret_cast(src), 0); - b->chunks[1] = __lasx_xvld(reinterpret_cast(src), 32); + b->chunks[0] = __lsx_vld(reinterpret_cast(src), 0); + b->chunks[1] = __lsx_vld(reinterpret_cast(src), 16); + b->chunks[2] = __lsx_vld(reinterpret_cast(src), 32); + b->chunks[3] = __lsx_vld(reinterpret_cast(src), 48); } // The caller of this function is responsible to ensure that there are 128 bytes // available from reading at src. The data is read into a block64 structure. static inline void load_block(block64 *b, const char16_t *src) { - __m256i m1 = __lasx_xvld(reinterpret_cast(src), 0); - __m256i m2 = __lasx_xvld(reinterpret_cast(src), 32); - __m256i m3 = __lasx_xvld(reinterpret_cast(src), 64); - __m256i m4 = __lasx_xvld(reinterpret_cast(src), 96); - b->chunks[0] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m2, m1, 0), 0b11011000); - b->chunks[1] = __lasx_xvpermi_d(__lasx_xvssrlni_bu_h(m4, m3, 0), 0b11011000); + __m128i m1 = __lsx_vld(reinterpret_cast(src), 0); + __m128i m2 = __lsx_vld(reinterpret_cast(src), 16); + __m128i m3 = __lsx_vld(reinterpret_cast(src), 32); + __m128i m4 = __lsx_vld(reinterpret_cast(src), 48); + __m128i m5 = __lsx_vld(reinterpret_cast(src), 64); + __m128i m6 = __lsx_vld(reinterpret_cast(src), 80); + __m128i m7 = __lsx_vld(reinterpret_cast(src), 96); + __m128i m8 = __lsx_vld(reinterpret_cast(src), 112); + b->chunks[0] = __lsx_vssrlni_bu_h(m2, m1, 0); + b->chunks[1] = __lsx_vssrlni_bu_h(m4, m3, 0); + b->chunks[2] = __lsx_vssrlni_bu_h(m6, m5, 0); + b->chunks[3] = __lsx_vssrlni_bu_h(m8, m7, 0); } -static inline void base64_decode(char *out, __m256i str) { - __m256i t0 = __lasx_xvor_v( - __lasx_xvslli_w(str, 26), - __lasx_xvslli_w(__lasx_xvand_v(str, lasx_splat_u32(0x0000ff00)), 12)); - __m256i t1 = - __lasx_xvsrli_w(__lasx_xvand_v(str, lasx_splat_u32(0x003f0000)), 2); - __m256i t2 = __lasx_xvor_v(t0, t1); - __m256i t3 = __lasx_xvor_v(t2, __lasx_xvsrli_w(str, 16)); - __m256i pack_shuffle = ____m256i( - (__m128i)v16u8{3, 2, 1, 7, 6, 5, 11, 10, 9, 15, 14, 13, 0, 0, 0, 0}); - t3 = __lasx_xvshuf_b(t3, t3, (__m256i)pack_shuffle); +static inline void base64_decode(char *out, __m128i str) { + __m128i t0 = __lsx_vor_v( + __lsx_vslli_w(str, 26), + __lsx_vslli_w(__lsx_vand_v(str, lsx_splat_u32(0x0000FF00)), 12)); + __m128i t1 = __lsx_vsrli_w(__lsx_vand_v(str, lsx_splat_u32(0x003F0000)), 2); + __m128i t2 = __lsx_vor_v(t0, t1); + __m128i t3 = __lsx_vor_v(t2, __lsx_vsrli_w(str, 16)); + const v16u8 pack_shuffle = {3, 2, 1, 7, 6, 5, 11, 10, + 9, 15, 14, 13, 0, 0, 0, 0}; + t3 = __lsx_vshuf_b(t3, t3, (__m128i)pack_shuffle); // Store the output: - __lsx_vst(lasx_extracti128_lo(t3), out, 0); - __lsx_vst(lasx_extracti128_hi(t3), out, 12); + // we only need 12. + __lsx_vstelm_d(t3, out, 0, 0); + __lsx_vstelm_w(t3, out + 8, 0, 2); } // decode 64 bytes and output 48 bytes static inline void base64_decode_block(char *out, const char *src) { - base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); + base64_decode(out, __lsx_vld(reinterpret_cast(src), 0)); + base64_decode(out + 12, + __lsx_vld(reinterpret_cast(src), 16)); base64_decode(out + 24, - __lasx_xvld(reinterpret_cast(src), 32)); + __lsx_vld(reinterpret_cast(src), 32)); + base64_decode(out + 36, + __lsx_vld(reinterpret_cast(src), 48)); } - static inline void base64_decode_block_safe(char *out, const char *src) { - base64_decode(out, __lasx_xvld(reinterpret_cast(src), 0)); - alignas(32) char buffer[32]; - base64_decode(buffer, - __lasx_xvld(reinterpret_cast(src), 32)); - std::memcpy(out + 24, buffer, 24); + base64_decode_block(out, src); } - static inline void base64_decode_block(char *out, block64 *b) { base64_decode(out, b->chunks[0]); - base64_decode(out + 24, b->chunks[1]); + base64_decode(out + 12, b->chunks[1]); + base64_decode(out + 24, b->chunks[2]); + base64_decode(out + 36, b->chunks[3]); } static inline void base64_decode_block_safe(char *out, block64 *b) { - base64_decode(out, b->chunks[0]); - alignas(32) char buffer[32]; - base64_decode(buffer, b->chunks[1]); - std::memcpy(out + 24, buffer, 24); + base64_decode_block(out, b); } template + typename char_type> full_result -compress_decode_base64(char *dst, const chartype *src, size_t srclen, +compress_decode_base64(char *dst, const char_type *src, size_t srclen, base64_options options, last_chunk_handling_options last_chunk_options) { const uint8_t *to_base64 = @@ -67055,19 +63602,15 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } return {SUCCESS, full_input_length, 0}; } - char *end_of_safe_64byte_zone = - (srclen + 3) / 4 * 3 >= 63 ? dst + (srclen + 3) / 4 * 3 - 63 : dst; - - const chartype *const srcinit = src; + const char_type *const srcinit = src; const char *const dstinit = dst; - const chartype *const srcend = src + srclen; + const char_type *const srcend = src + srclen; - constexpr size_t block_size = 6; - static_assert(block_size >= 2, "block_size must be at least two"); + constexpr size_t block_size = 10; char buffer[block_size * 64]; char *bufferptr = buffer; if (srclen >= 64) { - const chartype *const srcend64 = src + srclen - 64; + const char_type *const srcend64 = src + srclen - 64; while (src <= srcend64) { block64 b; load_block(&b, src); @@ -67075,55 +63618,49 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, bool error = false; uint64_t badcharmask = to_base64_mask(&b, &error); - if (error && !ignore_garbage) { - src -= 64; - while (src < srcend && scalar::base64::is_eight_byte(*src) && - to_base64[uint8_t(*src)] <= 64) { - src++; + if (badcharmask) { + if (error && !ignore_garbage) { + src -= 64; + while (src < srcend && scalar::base64::is_eight_byte(*src) && + to_base64[uint8_t(*src)] <= 64) { + src++; + } + if (src < srcend) { + // should never happen + } + return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; } - return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), - size_t(dst - dstinit)}; } + if (badcharmask != 0) { - // optimization opportunity: check for simple masks like those made of - // continuous 1s followed by continuous 0s. And masks containing a - // single bad character. - bufferptr += compress_block(&b, badcharmask, bufferptr); - } else if (bufferptr != buffer) { - copy_block(&b, bufferptr); - bufferptr += 64; - } else { - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, &b); + if (is_power_of_two(badcharmask)) { + bufferptr += compress_block_single(&b, badcharmask, bufferptr); } else { - base64_decode_block(dst, &b); + bufferptr += compress_block(&b, badcharmask, bufferptr); } - dst += 48; + } else { + // optimization opportunity: if bufferptr == buffer and mask == 0, we + // can avoid the call to compress_block and decode directly. + copy_block(&b, bufferptr); + bufferptr += 64; } if (bufferptr >= (block_size - 1) * 64 + buffer) { - for (size_t i = 0; i < (block_size - 2); i++) { + for (size_t i = 0; i < (block_size - 1); i++) { base64_decode_block(dst, buffer + i * 64); dst += 48; } - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, buffer + (block_size - 2) * 64); - } else { - base64_decode_block(dst, buffer + (block_size - 2) * 64); - } - dst += 48; std::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } } } - char *buffer_start = buffer; // Optimization note: if this is almost full, then it is worth our // time, otherwise, we should just decode directly. int last_block = (int)((bufferptr - buffer_start) % 64); if (last_block != 0 && srcend - src + last_block >= 64) { - while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { uint8_t val = to_base64[uint8_t(*src)]; *bufferptr = char(val); @@ -67138,11 +63675,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { - if (dst >= end_of_safe_64byte_zone) { - base64_decode_block_safe(dst, buffer_start); - } else { - base64_decode_block(dst, buffer_start); - } + base64_decode_block(dst, buffer_start); dst += 48; } if ((bufferptr - buffer_start) % 64 != 0) { @@ -67152,6 +63685,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lsx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 4); @@ -67164,6 +63698,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) << 8; + // lsx is little-endian triple = scalar::u32_swap_bytes(triple); std::memcpy(dst, &triple, 3); @@ -67223,24 +63758,22 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, } return {SUCCESS, srclen, size_t(dst - dstinit)}; } -/* end file src/lasx/lasx_base64.cpp */ -/* begin file src/lasx/lasx_find.cpp */ +/* end file src/lsx/lsx_base64.cpp */ +/* begin file src/lsx/lsx_find.cpp */ simdutf_really_inline const char *util_find(const char *start, const char *end, char character) noexcept { if (start >= end) return end; - const int step = 32; - __m256i char_vec = __lasx_xvreplgr2vr_b(static_cast(character)); + const int step = 16; + __m128i char_vec = __lsx_vreplgr2vr_b(static_cast(character)); while (end - start >= step) { - __m256i data = __lasx_xvld(reinterpret_cast(start), 0); - __m256i cmp = __lasx_xvseq_b(data, char_vec); - if (__lasx_xbnz_v(cmp)) { - __m256i res = __lasx_xvmsknz_b(cmp); - uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); - uint32_t mask = (mask0 | (mask1 << 16)); + __m128i data = __lsx_vld(reinterpret_cast(start), 0); + __m128i cmp = __lsx_vseq_b(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); return start + trailing_zeroes(mask); } @@ -67263,17 +63796,15 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, if (start >= end) return end; - const int step = 16; - __m256i char_vec = __lasx_xvreplgr2vr_h(static_cast(character)); + const int step = 8; + __m128i char_vec = __lsx_vreplgr2vr_h(static_cast(character)); while (end - start >= step) { - __m256i data = __lasx_xvld(reinterpret_cast(start), 0); - __m256i cmp = __lasx_xvseq_h(data, char_vec); - if (__lasx_xbnz_v(cmp)) { - __m256i res = __lasx_xvmsknz_b(cmp); - uint32_t mask0 = __lasx_xvpickve2gr_wu(res, 0); - uint32_t mask1 = __lasx_xvpickve2gr_wu(res, 4); - uint32_t mask = (mask0 | (mask1 << 16)); + __m128i data = __lsx_vld(reinterpret_cast(start), 0); + __m128i cmp = __lsx_vseq_h(data, char_vec); + if (__lsx_bnz_v(cmp)) { + uint16_t mask = + static_cast(__lsx_vpickve2gr_hu(__lsx_vmsknz_b(cmp), 0)); return start + trailing_zeroes(mask) / 2; } @@ -67289,16 +63820,16 @@ simdutf_really_inline const char16_t *util_find(const char16_t *start, return end; } -/* end file src/lasx/lasx_find.cpp */ +/* end file src/lsx/lsx_find.cpp */ #endif // SIMDUTF_FEATURE_BASE64 } // namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* begin file src/generic/buf_block_reader.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { // Walks through a buffer in block-sized increments, loading the last part with @@ -67329,40 +63860,6 @@ template struct buf_block_reader { size_t idx; }; -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text_64(const uint8_t *text) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdutf_unused static char *format_input_text(const simd8x64 &in) { - static char *buf = - reinterpret_cast(malloc(sizeof(simd8x64) + 1)); - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { - buf[i] = '_'; - } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdutf_unused static char *format_mask(uint64_t mask) { - static char *buf = reinterpret_cast(malloc(64 + 1)); - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - template simdutf_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) @@ -67404,13 +63901,13 @@ simdutf_really_inline void buf_block_reader::advance() { } } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/buf_block_reader.h */ #if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_validation { @@ -67604,10 +64101,10 @@ struct utf8_checker { static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -67630,12 +64127,12 @@ struct utf8_checker { using utf8_validation::utf8_checker; } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_validation/utf8_lookup4_algorithm.h */ /* begin file src/generic/utf8_validation/utf8_validator.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_validation { @@ -67716,33 +64213,17 @@ result generic_validate_utf8_with_errors(const char *input, size_t length) { } // namespace utf8_validation } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_validation/utf8_validator.h */ #endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_ASCII /* begin file src/generic/ascii_validation.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace ascii_validation { -bool generic_validate_ascii(const char *input, size_t length) { - buf_block_reader<64> reader(reinterpret_cast(input), length); - uint8_t blocks[64]{}; - simd::simd8x64 running_or(blocks); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - running_or |= in; - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - running_or |= in; - return running_or.is_ascii(); -} - result generic_validate_ascii_with_errors(const char *input, size_t length) { buf_block_reader<64> reader(reinterpret_cast(input), length); size_t count{0}; @@ -67769,9 +64250,24 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { } } +bool generic_validate_ascii(const char *input, size_t length) { + buf_block_reader<64> reader(reinterpret_cast(input), length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + if (!in.is_ascii()) { + return false; + } + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + return in.is_ascii(); +} + } // namespace ascii_validation } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/ascii_validation.h */ #endif // SIMDUTF_FEATURE_ASCII @@ -67780,7 +64276,7 @@ result generic_validate_ascii_with_errors(const char *input, size_t length) { // transcoding from UTF-8 to Latin 1 /* begin file src/generic/utf8_to_latin1/utf8_to_latin1.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_latin1 { using namespace simd; @@ -67925,10 +64421,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -68014,10 +64510,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -68093,12 +64589,12 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_latin1 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_to_latin1/utf8_to_latin1.h */ /* begin file src/generic/utf8_to_latin1/valid_utf8_to_latin1.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_latin1 { using namespace simd; @@ -68172,16 +64668,17 @@ simdutf_really_inline size_t convert_valid(const char *in, size_t size, } // namespace utf8_to_latin1 } // namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf // namespace simdutf /* end file src/generic/utf8_to_latin1/valid_utf8_to_latin1.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 + #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 // transcoding from UTF-8 to UTF-16 /* begin file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_utf16 { @@ -68252,12 +64749,12 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace utf8_to_utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_to_utf16/valid_utf8_to_utf16.h */ /* begin file src/generic/utf8_to_utf16/utf8_to_utf16.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_utf16 { using namespace simd; @@ -68416,10 +64913,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -68504,10 +65001,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -68586,12 +65083,12 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_to_utf16/utf8_to_utf16.h */ /* begin file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8 { @@ -68641,15 +65138,55 @@ simdutf_really_inline size_t utf16_length_from_utf8_bytemask(const char *in, } // namespace utf8 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8/utf16_length_from_utf8_bytemask.h */ + // transcoding from UTF-16 to UTF-8 +/* begin file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ +// Note: no include guard on purpose. This header is included once inside each +// SIMD kernel's translation unit (and re-expanded per kernel in the +// amalgamation), matching the other generic/ transcoder headers. +namespace simdutf { +namespace lsx { +namespace { +namespace utf16_to_utf8 { + +// Substitutes U+FFFD for each unpaired surrogate. convert_with_details reports +// the bytes written alongside the input position, so the converted prefix never +// has to be re-walked. +template +simdutf_really_inline size_t convert_with_replacement_via( + ConvertWithDetails convert_with_details, const char16_t *buf, size_t len, + char *utf8_output) { + char *const start = utf8_output; + size_t pos = 0; + while (pos < len) { + full_result r = convert_with_details(buf + pos, len - pos, utf8_output); + utf8_output += r.output_count; + if (r.error != error_code::SURROGATE) { + break; + } + pos += r.input_count + 1; + utf8_output[0] = char(0xef); + utf8_output[1] = char(0xbf); + utf8_output[2] = char(0xbd); + utf8_output += 3; + } + return size_t(utf8_output - start); +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace lsx +} // namespace simdutf +/* end file src/generic/utf16_to_utf8/utf16_to_utf8_with_replacement.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 + #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 // transcoding from UTF-8 to UTF-32 /* begin file src/generic/utf8_to_utf32/valid_utf8_to_utf32.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_utf32 { @@ -68688,12 +65225,12 @@ simdutf_warn_unused size_t convert_valid(const char *input, size_t size, } // namespace utf8_to_utf32 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_to_utf32/valid_utf8_to_utf32.h */ /* begin file src/generic/utf8_to_utf32/utf8_to_utf32.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8_to_utf32 { using namespace simd; @@ -68851,10 +65388,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -68938,10 +65475,10 @@ struct validating_transcoder { (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); auto zero = simd8{uint8_t(0)}; - if (simd8x64::NUM_CHUNKS == 2) { + if constexpr (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if (simd8x64::NUM_CHUNKS == 4) { + } else if constexpr (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], zero); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -69008,7 +65545,7 @@ struct validating_transcoder { }; // struct utf8_checker } // namespace utf8_to_utf32 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8_to_utf32/utf8_to_utf32.h */ #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -69016,7 +65553,7 @@ struct validating_transcoder { #if SIMDUTF_FEATURE_UTF8 /* begin file src/generic/utf8.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf8 { @@ -69034,8 +65571,8 @@ simdutf_really_inline size_t count_code_points(const char *in, size_t size) { } #ifdef SIMDUTF_SIMD_HAS_BYTEMASK -simdutf_really_inline size_t count_code_points_bytemask(const char *in, - size_t size) { +simdutf_unused simdutf_really_inline size_t +count_code_points_bytemask(const char *in, size_t size) { using vector_i8 = simd8; using vector_u8 = simd8; using vector_u64 = simd64; @@ -69105,7 +65642,7 @@ simdutf_really_inline size_t utf16_length_from_utf8(const char *in, } // namespace utf8 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf8.h */ #endif // SIMDUTF_FEATURE_UTF8 @@ -69113,7 +65650,7 @@ simdutf_really_inline size_t utf16_length_from_utf8(const char *in, #if SIMDUTF_FEATURE_UTF16 /* begin file src/generic/utf16/count_code_points_bytemask.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { @@ -69128,16 +65665,16 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, size_t pos = 0; size_t count = 0; - constexpr size_t max_itertions = 65535; + constexpr size_t max_iterations = 65535; const auto one = vector_u16::splat(1); const auto zero = vector_u16::zero(); - size_t itertion = 0; + size_t iteration = 0; auto counters = zero; for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(in + pos); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } @@ -69150,15 +65687,15 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, counters += t2; - itertion += 1; - if (itertion == max_itertions) { + iteration += 1; + if (iteration == max_iterations) { count += counters.sum(); counters = zero; - itertion = 0; + iteration = 0; } } - if (itertion > 0) { + if (iteration > 0) { count += counters.sum(); } @@ -69168,12 +65705,12 @@ simdutf_really_inline size_t count_code_points(const char16_t *in, } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf16/count_code_points_bytemask.h */ /* begin file src/generic/utf16/change_endianness.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { @@ -69194,12 +65731,12 @@ change_endianness_utf16(const char16_t *in, size_t size, char16_t *output) { } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf16/change_endianness.h */ /* begin file src/generic/utf16/utf8_length_from_utf16_bytemask.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { @@ -69227,7 +65764,7 @@ simdutf_really_inline size_t utf8_length_from_utf16_bytemask(const char16_t *in, for (; pos < size / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -69316,7 +65853,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { for (; pos < (size - 1) / N * N; pos += N) { auto input = vector_u16::load(reinterpret_cast(in + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input = input.swap_bytes(); } // 0xd800 .. 0xdbff - low surrogate @@ -69337,7 +65874,7 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { any_surrogates = true; auto input_next = vector_u16::load(reinterpret_cast(in + pos + 1)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_next = input_next.swap_bytes(); } @@ -69395,12 +65932,12 @@ utf8_length_from_utf16_with_replacement(const char16_t *in, size_t size) { } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf16/utf8_length_from_utf16_bytemask.h */ /* begin file src/generic/utf16/utf32_length_from_utf16.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { @@ -69412,12 +65949,12 @@ simdutf_really_inline size_t utf32_length_from_utf16(const char16_t *in, } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf16/utf32_length_from_utf16.h */ /* begin file src/generic/utf16/to_well_formed.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { @@ -69434,11 +65971,11 @@ namespace utf16 { template simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { const char16_t replacement = scalar::utf16::replacement(); - auto swap_if_needed = [](uint16_t c) -> uint16_t { - return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; - }; using vector_u16 = simd16; + auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t { + return scalar::utf16::swap_if_needed(x); + }; const auto lookback = vector_u16::load(in - 1); const auto block = vector_u16::load(in); @@ -69448,7 +65985,6 @@ simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) { const auto lb_is_high = lb_masked == swap_if_needed(0xd800); const auto block_is_low = block_masked == swap_if_needed(0xdc00); - const auto illseq = lb_is_high ^ block_is_low; if (!illseq.is_zero()) { /* compute the cause of the illegal sequencing */ @@ -69508,7 +66044,7 @@ void to_well_formed(const char16_t *in, size_t n, char16_t *out) { } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf16/to_well_formed.h */ #endif // SIMDUTF_FEATURE_UTF16 @@ -69516,7 +66052,7 @@ void to_well_formed(const char16_t *in, size_t n, char16_t *out) { #if SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING /* begin file src/generic/validate_utf16.h */ namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf16 { /* @@ -69652,7 +66188,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, for (; pos < size / 32 * 32; pos += 32) { simd16x32 input_vec( reinterpret_cast(input + pos)); - if (!match_system(big_endian)) { + if constexpr (!match_system(big_endian)) { input_vec.swap_bytes(); } uint64_t matches = input_vec.lteq(uint16_t(0x7f)); @@ -69665,7 +66201,8 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, // Scalar tail while (pos < size) { - char16_t v = big_endian ? scalar::u16_swap_bytes(input[pos]) : input[pos]; + + char16_t v = scalar::utf16::swap_if_needed(input[pos]); if (v > 0x7F) { return result(error_code::TOO_LARGE, pos); } @@ -69676,7 +66213,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, } // namespace utf16 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/validate_utf16.h */ #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING @@ -69686,7 +66223,7 @@ const result validate_utf16_as_ascii_with_errors(const char16_t *input, #include namespace simdutf { -namespace lasx { +namespace lsx { namespace { namespace utf32 { @@ -69817,16 +66354,86 @@ simdutf_really_inline size_t utf8_length_from_utf32(const char32_t *input, } // namespace utf32 } // unnamed namespace -} // namespace lasx +} // namespace lsx } // namespace simdutf /* end file src/generic/utf32.h */ #endif // SIMDUTF_FEATURE_UTF32 +#if SIMDUTF_FEATURE_BASE64 +/* begin file src/generic/base64lengths.h */ +namespace simdutf { +namespace lsx { +namespace { +namespace base64_lengths { + +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 64 <= length; pos += 64) { + simd8x64 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) { + size_t pos = 0; + size_t count = 0; + for (; pos + 32 <= length; pos += 32) { + simd16x32 block(reinterpret_cast(input + pos)); + uint64_t maybe_base64 = block.gteq(33); // >= 33 which is '!' in ASCII + count += count_ones(maybe_base64); + } + // simd16x32::to_bitmask sets two bits per matching 16-bit lane, so the + // vectorized loop counted each unit twice. + count /= 2; + while (pos < length) { + count += (input[pos] > 0x20) ? 1 : 0; + pos++; + } + // Count padding at the end. + size_t padding = 0; + pos = length; + while (pos > 0 && padding < 2) { + char16_t c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +} // namespace base64_lengths +} // unnamed namespace +} // namespace lsx +} // namespace simdutf +/* end file src/generic/base64lengths.h */ +#endif // SIMDUTF_FEATURE_BASE64 // // Implementation-specific overrides // namespace simdutf { -namespace lasx { +namespace lsx { #if SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused int @@ -69860,33 +66467,33 @@ implementation::detect_encodings(const char *input, #if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING simdutf_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return lasx::utf8_validation::generic_validate_utf8(buf, len); + return lsx::utf8_validation::generic_validate_utf8(buf, len); } #endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused result implementation::validate_utf8_with_errors( const char *buf, size_t len) const noexcept { - return lasx::utf8_validation::generic_validate_utf8_with_errors(buf, len); + return lsx::utf8_validation::generic_validate_utf8_with_errors(buf, len); } #endif // SIMDUTF_FEATURE_UTF8 #if SIMDUTF_FEATURE_ASCII simdutf_warn_unused bool implementation::validate_ascii(const char *buf, size_t len) const noexcept { - return lasx::ascii_validation::generic_validate_ascii(buf, len); + return lsx::ascii_validation::generic_validate_ascii(buf, len); } simdutf_warn_unused result implementation::validate_ascii_with_errors( const char *buf, size_t len) const noexcept { - return lasx::ascii_validation::generic_validate_ascii_with_errors(buf, len); + return lsx::ascii_validation::generic_validate_ascii_with_errors(buf, len); } #endif // SIMDUTF_FEATURE_ASCII #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII simdutf_warn_unused bool implementation::validate_utf16le_as_ascii(const char16_t *buf, size_t len) const noexcept { - return lasx::utf16::validate_utf16_as_ascii_with_errors( + return lsx::utf16::validate_utf16_as_ascii_with_errors( buf, len) .error == SUCCESS; } @@ -69894,8 +66501,8 @@ implementation::validate_utf16le_as_ascii(const char16_t *buf, simdutf_warn_unused bool implementation::validate_utf16be_as_ascii(const char16_t *buf, size_t len) const noexcept { - return lasx::utf16::validate_utf16_as_ascii_with_errors(buf, - len) + return lsx::utf16::validate_utf16_as_ascii_with_errors(buf, + len) .error == SUCCESS; } #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII @@ -69908,7 +66515,8 @@ implementation::validate_utf16le(const char16_t *buf, return true; } const auto res = - lasx::utf16::validate_utf16_with_errors(buf, len); + lsx::utf16::validate_utf16_with_errors(buf, len); + if (res.is_err()) { return false; } @@ -69930,9 +66538,9 @@ implementation::validate_utf16be(const char16_t *buf, // empty input is valid. protected the implementation from nullptr. return true; } - const auto res = - lasx::utf16::validate_utf16_with_errors(buf, len); + lsx::utf16::validate_utf16_with_errors(buf, len); + if (res.is_err()) { return false; } @@ -69951,7 +66559,7 @@ simdutf_warn_unused result implementation::validate_utf16le_with_errors( return result(error_code::SUCCESS, 0); } const result res = - lasx::utf16::validate_utf16_with_errors(buf, len); + lsx::utf16::validate_utf16_with_errors(buf, len); if (res.count != len) { const result scalar_res = scalar::utf16::validate_with_errors( @@ -69968,7 +66576,7 @@ simdutf_warn_unused result implementation::validate_utf16be_with_errors( return result(error_code::SUCCESS, 0); } const result res = - lasx::utf16::validate_utf16_with_errors(buf, len); + lsx::utf16::validate_utf16_with_errors(buf, len); if (res.count != len) { const result scalar_res = scalar::utf16::validate_with_errors(buf + res.count, @@ -69981,12 +66589,12 @@ simdutf_warn_unused result implementation::validate_utf16be_with_errors( void implementation::to_well_formed_utf16le(const char16_t *input, size_t len, char16_t *output) const noexcept { - return utf16::to_well_formed(input, len, output); + utf16::to_well_formed(input, len, output); } void implementation::to_well_formed_utf16be(const char16_t *input, size_t len, char16_t *output) const noexcept { - return utf16::to_well_formed(input, len, output); + utf16::to_well_formed(input, len, output); } #endif // SIMDUTF_FEATURE_UTF16 @@ -69997,7 +66605,7 @@ implementation::validate_utf32(const char32_t *buf, size_t len) const noexcept { // empty input is valid. protected the implementation from nullptr. return true; } - const char32_t *tail = lasx_validate_utf32le(buf, len); + const char32_t *tail = lsx_validate_utf32le(buf, len); if (tail) { return scalar::utf32::validate(tail, len - (tail - buf)); } else { @@ -70012,7 +66620,7 @@ simdutf_warn_unused result implementation::validate_utf32_with_errors( if (simdutf_unlikely(len == 0)) { return result(error_code::SUCCESS, 0); } - result res = lasx_validate_utf32le_with_errors(buf, len); + result res = lsx_validate_utf32le_with_errors(buf, len); if (res.count != len) { result scalar_res = scalar::utf32::validate_with_errors(buf + res.count, len - res.count); @@ -70027,7 +66635,7 @@ simdutf_warn_unused result implementation::validate_utf32_with_errors( simdutf_warn_unused size_t implementation::convert_latin1_to_utf8( const char *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lasx_convert_latin1_to_utf8(buf, len, utf8_output); + lsx_convert_latin1_to_utf8(buf, len, utf8_output); size_t converted_chars = ret.second - utf8_output; if (ret.first != buf + len) { @@ -70043,7 +66651,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf8( simdutf_warn_unused size_t implementation::convert_latin1_to_utf16le( const char *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lasx_convert_latin1_to_utf16le(buf, len, utf16_output); + lsx_convert_latin1_to_utf16le(buf, len, utf16_output); size_t converted_chars = ret.second - utf16_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = @@ -70057,7 +66665,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf16le( simdutf_warn_unused size_t implementation::convert_latin1_to_utf16be( const char *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lasx_convert_latin1_to_utf16be(buf, len, utf16_output); + lsx_convert_latin1_to_utf16be(buf, len, utf16_output); size_t converted_chars = ret.second - utf16_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = @@ -70073,7 +66681,7 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf16be( simdutf_warn_unused size_t implementation::convert_latin1_to_utf32( const char *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lasx_convert_latin1_to_utf32(buf, len, utf32_output); + lsx_convert_latin1_to_utf32(buf, len, utf32_output); size_t converted_chars = ret.second - utf32_output; if (ret.first != buf + len) { const size_t scalar_converted_chars = scalar::latin1_to_utf32::convert( @@ -70087,117 +66695,19 @@ simdutf_warn_unused size_t implementation::convert_latin1_to_utf32( #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 simdutf_warn_unused size_t implementation::convert_utf8_to_latin1( const char *buf, size_t len, char *latin1_output) const noexcept { - size_t pos = 0; - char *output_start{latin1_output}; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)latin1_output & 0x1F) && pos < len) { - if (buf[pos] & 0x80) { - if (pos + 1 >= len) - return 0; - if ((buf[pos] & 0b11100000) == 0b11000000) { - if ((buf[pos + 1] & 0b11000000) != 0b10000000) - return 0; - uint32_t code_point = - (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); - if (code_point < 0x80 || 0xFF < code_point) { - return 0; - } - *latin1_output++ = char(code_point); - pos += 2; - } else { - return 0; - } - } else { - *latin1_output++ = char(buf[pos]); - pos++; - } - } - size_t convert_size = latin1_output - output_start; - if (pos == len) - return convert_size; utf8_to_latin1::validating_transcoder converter; - size_t convert_result = - converter.convert(buf + pos, len - pos, latin1_output); - return convert_result ? convert_size + convert_result : 0; + return converter.convert(buf, len, latin1_output); } simdutf_warn_unused result implementation::convert_utf8_to_latin1_with_errors( const char *buf, size_t len, char *latin1_output) const noexcept { - size_t pos = 0; - char *output_start{latin1_output}; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)latin1_output & 0x1F) && pos < len) { - if (buf[pos] & 0x80) { - if ((buf[pos] & 0b11100000) == 0b11000000) { - if (pos + 1 >= len) - return result(error_code::TOO_SHORT, pos); - if ((buf[pos + 1] & 0b11000000) != 0b10000000) - return result(error_code::TOO_SHORT, pos); - uint32_t code_point = - (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); - if (code_point < 0x80) - return result(error_code::OVERLONG, pos); - if (0xFF < code_point) - return result(error_code::TOO_LARGE, pos); - *latin1_output++ = char(code_point); - pos += 2; - } else if ((buf[pos] & 0b11110000) == 0b11100000) { - return result(error_code::TOO_LARGE, pos); - } else if ((buf[pos] & 0b11111000) == 0b11110000) { - return result(error_code::TOO_LARGE, pos); - } else { - if ((buf[pos] & 0b11000000) == 0b10000000) { - return result(error_code::TOO_LONG, pos); - } - return result(error_code::HEADER_BITS, pos); - } - } else { - *latin1_output++ = char(buf[pos]); - pos++; - } - } - size_t convert_size = latin1_output - output_start; - if (pos == len) - return result(error_code::SUCCESS, convert_size); - utf8_to_latin1::validating_transcoder converter; - result res = - converter.convert_with_errors(buf + pos, len - pos, latin1_output); - return res.error ? result(res.error, res.count + pos) - : result(res.error, res.count + convert_size); + return converter.convert_with_errors(buf, len, latin1_output); } simdutf_warn_unused size_t implementation::convert_valid_utf8_to_latin1( const char *buf, size_t len, char *latin1_output) const noexcept { - size_t pos = 0; - char *output_start{latin1_output}; - // Performance degradation when memory address is not 32-byte aligned - while (((uint64_t)latin1_output & 0x1F) && pos < len) { - if (buf[pos] & 0x80) { - if (pos + 1 >= len) - break; - if ((buf[pos] & 0b11100000) == 0b11000000) { - if ((buf[pos + 1] & 0b11000000) != 0b10000000) - return 0; - uint32_t code_point = - (buf[pos] & 0b00011111) << 6 | (buf[pos + 1] & 0b00111111); - *latin1_output++ = char(code_point); - pos += 2; - } else { - return 0; - } - } else { - *latin1_output++ = char(buf[pos]); - pos++; - } - } - size_t convert_size = latin1_output - output_start; - if (pos == len) - return convert_size; - - size_t convert_result = - lasx::utf8_to_latin1::convert_valid(buf + pos, len - pos, latin1_output); - return convert_result ? convert_size + convert_result : 0; + return lsx::utf8_to_latin1::convert_valid(buf, len, latin1_output); } #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -70263,7 +66773,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf8_to_utf32( simdutf_warn_unused size_t implementation::convert_utf16le_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_latin1(buf, len, latin1_output); + lsx_convert_utf16_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -70284,7 +66794,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_latin1( simdutf_warn_unused size_t implementation::convert_utf16be_to_latin1( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_latin1(buf, len, latin1_output); + lsx_convert_utf16_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -70306,7 +66816,7 @@ simdutf_warn_unused result implementation::convert_utf16le_to_latin1_with_errors( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_latin1_with_errors( + lsx_convert_utf16_to_latin1_with_errors( buf, len, latin1_output); if (ret.first.error) { return ret.first; @@ -70333,8 +66843,8 @@ simdutf_warn_unused result implementation::convert_utf16be_to_latin1_with_errors( const char16_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_latin1_with_errors(buf, len, - latin1_output); + lsx_convert_utf16_to_latin1_with_errors(buf, len, + latin1_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -70373,7 +66883,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_latin1( simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_utf8(buf, len, utf8_output); + lsx_convert_utf16_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -70393,7 +66903,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_utf8( simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( const char16_t *buf, size_t len, char *utf8_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_utf8(buf, len, utf8_output); + lsx_convert_utf16_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -70410,60 +66920,36 @@ simdutf_warn_unused size_t implementation::convert_utf16be_to_utf8( return saved_bytes; } -simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( - const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished +template +simdutf_really_inline full_result convert_utf16_to_utf8_with_details( + const char16_t *buf, size_t len, char *utf8_output) { std::pair ret = - lasx_convert_utf16_to_utf8_with_errors(buf, len, - utf8_output); + lsx_convert_utf16_to_utf8_with_errors(buf, len, utf8_output); if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } + return full_result(ret.first.error, ret.first.count, + size_t(ret.second - utf8_output)); } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + if (ret.first.count != len) { + full_result sres = + scalar::utf16_to_utf8::convert_with_errors( + buf + ret.first.count, len - ret.first.count, ret.second, 0); + return full_result(sres.error, ret.first.count + sres.input_count, + size_t(ret.second - utf8_output) + sres.output_count); + } + return full_result(error_code::SUCCESS, len, + size_t(ret.second - utf8_output)); +} + +simdutf_warn_unused result implementation::convert_utf16le_to_utf8_with_errors( + const char16_t *buf, size_t len, char *utf8_output) const noexcept { + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused result implementation::convert_utf16be_to_utf8_with_errors( const char16_t *buf, size_t len, char *utf8_output) const noexcept { - // ret.first.count is always the position in the buffer, not the number of - // code units written even if finished - std::pair ret = - lasx_convert_utf16_to_utf8_with_errors(buf, len, - utf8_output); - if (ret.first.error) { - return ret.first; - } // Can return directly since scalar fallback already found correct - // ret.first.count - if (ret.first.count != len) { // All good so far, but not finished - result scalar_res = - scalar::utf16_to_utf8::convert_with_errors( - buf + ret.first.count, len - ret.first.count, ret.second); - if (scalar_res.error) { - scalar_res.count += ret.first.count; - return scalar_res; - } else { - ret.second += scalar_res.count; - } - } - ret.first.count = - ret.second - - utf8_output; // Set count to the number of 8-bit code units written - return ret.first; + return convert_utf16_to_utf8_with_details(buf, len, + utf8_output); } simdutf_warn_unused size_t implementation::convert_valid_utf16le_to_utf8( @@ -70484,7 +66970,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_utf8( return 0; } std::pair ret = - lasx_convert_utf32_to_utf8(buf, len, utf8_output); + lsx_convert_utf32_to_utf8(buf, len, utf8_output); if (ret.first == nullptr) { return 0; } @@ -70508,7 +66994,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf8_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lasx_convert_utf32_to_utf8_with_errors(buf, len, utf8_output); + lsx_convert_utf32_to_utf8_with_errors(buf, len, utf8_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf8::convert_with_errors( buf + ret.first.count, len - ret.first.count, ret.second); @@ -70530,7 +67016,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf8_with_errors( simdutf_warn_unused size_t implementation::convert_utf16le_to_utf32( const char16_t *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_utf32(buf, len, utf32_output); + lsx_convert_utf16_to_utf32(buf, len, utf32_output); if (ret.first == nullptr) { return 0; } @@ -70550,7 +67036,7 @@ simdutf_warn_unused size_t implementation::convert_utf16le_to_utf32( simdutf_warn_unused size_t implementation::convert_utf16be_to_utf32( const char16_t *buf, size_t len, char32_t *utf32_output) const noexcept { std::pair ret = - lasx_convert_utf16_to_utf32(buf, len, utf32_output); + lsx_convert_utf16_to_utf32(buf, len, utf32_output); if (ret.first == nullptr) { return 0; } @@ -70572,8 +67058,8 @@ simdutf_warn_unused result implementation::convert_utf16le_to_utf32_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lasx_convert_utf16_to_utf32_with_errors(buf, len, - utf32_output); + lsx_convert_utf16_to_utf32_with_errors(buf, len, + utf32_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -70600,8 +67086,8 @@ simdutf_warn_unused result implementation::convert_utf16be_to_utf32_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lasx_convert_utf16_to_utf32_with_errors(buf, len, - utf32_output); + lsx_convert_utf16_to_utf32_with_errors(buf, len, + utf32_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -70628,7 +67114,7 @@ simdutf_warn_unused result implementation::convert_utf16be_to_utf32_with_errors( simdutf_warn_unused size_t implementation::convert_utf32_to_latin1( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf32_to_latin1(buf, len, latin1_output); + lsx_convert_utf32_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -70648,7 +67134,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_latin1( simdutf_warn_unused result implementation::convert_utf32_to_latin1_with_errors( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf32_to_latin1_with_errors(buf, len, latin1_output); + lsx_convert_utf32_to_latin1_with_errors(buf, len, latin1_output); if (ret.first.error) { return ret.first; } // Can return directly since scalar fallback already found correct @@ -70672,7 +67158,7 @@ simdutf_warn_unused result implementation::convert_utf32_to_latin1_with_errors( simdutf_warn_unused size_t implementation::convert_valid_utf32_to_latin1( const char32_t *buf, size_t len, char *latin1_output) const noexcept { std::pair ret = - lasx_convert_utf32_to_latin1(buf, len, latin1_output); + lsx_convert_utf32_to_latin1(buf, len, latin1_output); if (ret.first == nullptr) { return 0; } @@ -70699,7 +67185,7 @@ simdutf_warn_unused size_t implementation::convert_valid_utf32_to_utf8( simdutf_warn_unused size_t implementation::convert_utf32_to_utf16le( const char32_t *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lasx_convert_utf32_to_utf16(buf, len, utf16_output); + lsx_convert_utf32_to_utf16(buf, len, utf16_output); if (ret.first == nullptr) { return 0; } @@ -70720,7 +67206,7 @@ simdutf_warn_unused size_t implementation::convert_utf32_to_utf16le( simdutf_warn_unused size_t implementation::convert_utf32_to_utf16be( const char32_t *buf, size_t len, char16_t *utf16_output) const noexcept { std::pair ret = - lasx_convert_utf32_to_utf16(buf, len, utf16_output); + lsx_convert_utf32_to_utf16(buf, len, utf16_output); if (ret.first == nullptr) { return 0; } @@ -70742,8 +67228,8 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf16le_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lasx_convert_utf32_to_utf16_with_errors(buf, len, - utf16_output); + lsx_convert_utf32_to_utf16_with_errors(buf, len, + utf16_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf16::convert_with_errors( @@ -70766,8 +67252,8 @@ simdutf_warn_unused result implementation::convert_utf32_to_utf16be_with_errors( // ret.first.count is always the position in the buffer, not the number of // code units written even if finished std::pair ret = - lasx_convert_utf32_to_utf16_with_errors(buf, len, - utf16_output); + lsx_convert_utf32_to_utf16_with_errors(buf, len, + utf16_output); if (ret.first.count != len) { result scalar_res = scalar::utf32_to_utf16::convert_with_errors( @@ -70827,23 +67313,7 @@ simdutf_warn_unused size_t implementation::count_utf16be( #if SIMDUTF_FEATURE_UTF8 simdutf_warn_unused size_t implementation::count_utf8(const char *input, size_t length) const noexcept { - size_t pos = 0; - size_t count = 0; - // Performance degradation when memory address is not 32-byte aligned - while ((((uint64_t)input + pos) & 0x1F && pos < length)) { - if (input[pos++] > -65) { - count++; - } - } - __m256i v_bf = __lasx_xvldi(0xBF); // 0b10111111 - for (; pos + 32 <= length; pos += 32) { - __m256i in = __lasx_xvld(reinterpret_cast(input + pos), 0); - __m256i utf8_count = - __lasx_xvpcnt_h(__lasx_xvmskltz_b(__lasx_xvslt_b(v_bf, in))); - count = count + __lasx_xvpickve2gr_wu(utf8_count, 0) + - __lasx_xvpickve2gr_wu(utf8_count, 4); - } - return count + scalar::utf8::count_code_points(input + pos, length - pos); + return utf8::count_code_points(input, length); } #endif // SIMDUTF_FEATURE_UTF8 @@ -70917,6 +67387,26 @@ implementation::utf8_length_from_utf16be_with_replacement( endianness::BIG>(input, length); } +simdutf_warn_unused size_t +implementation::convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + +simdutf_warn_unused size_t +implementation::convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) const noexcept { + return utf16_to_utf8::convert_with_replacement_via( + [](const char16_t *b, size_t l, char *o) { + return convert_utf16_to_utf8_with_details(b, l, o); + }, + input, length, utf8_buffer); +} + #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -70929,12 +67419,12 @@ simdutf_warn_unused size_t implementation::utf8_length_from_utf32( #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 simdutf_warn_unused size_t implementation::utf16_length_from_utf32( const char32_t *input, size_t length) const noexcept { - __m128i v_ffff = lsx_splat_u32(0x0000ffff); + const __m128i v_ffff = lsx_splat_u32(0x0000ffff); size_t pos = 0; size_t count = 0; for (; pos + 4 <= length; pos += 4) { __m128i in = __lsx_vld(reinterpret_cast(input + pos), 0); - __m128i surrogate_bytemask = __lsx_vslt_wu(v_ffff, in); + const __m128i surrogate_bytemask = __lsx_vslt_wu(v_ffff, in); size_t surrogate_count = __lsx_vpickve2gr_bu( __lsx_vpcnt_b(__lsx_vmskltz_w(surrogate_bytemask)), 0); count += 4 + surrogate_count; @@ -71098,16 +67588,982 @@ const char16_t *implementation::find(const char16_t *start, const char16_t *end, char16_t character) const noexcept { return util_find(start, end, character); } + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} + +simdutf_warn_unused size_t implementation::binary_length_from_base64( + const char16_t *input, size_t length) const noexcept { + return base64_lengths::binary_length_from_base64(input, length); +} #endif // SIMDUTF_FEATURE_BASE64 -} // namespace lasx +} // namespace lsx } // namespace simdutf -/* begin file src/simdutf/lasx/end.h */ +/* begin file src/simdutf/lsx/end.h */ #undef SIMDUTF_SIMD_HAS_UNSIGNED_CMP -/* end file src/simdutf/lasx/end.h */ -/* end file src/lasx/implementation.cpp */ +/* end file src/simdutf/lsx/end.h */ +/* end file src/lsx/implementation.cpp */ +#endif + +/* begin file src/simdutf_c.cpp */ +/* begin file include/simdutf_c.h */ +/*** + * simdutf_c.h.h - C API for simdutf + * This is currently experimental. + * We are committed to keeping the C API, but there might be mistakes in our + * implementation. Please report any issues you find. + */ + +#ifndef SIMDUTF_C_H +#define SIMDUTF_C_H + +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t + #endif // __has_include() +#else // __has_include() + #define char16_t uint16_t + #define char32_t uint32_t +#endif // __has_include + +#ifdef __cplusplus +extern "C" { #endif +/* C-friendly subset of simdutf errors */ +typedef enum simdutf_error_code { + SIMDUTF_ERROR_SUCCESS = 0, + SIMDUTF_ERROR_HEADER_BITS, + SIMDUTF_ERROR_TOO_SHORT, + SIMDUTF_ERROR_TOO_LONG, + SIMDUTF_ERROR_OVERLONG, + SIMDUTF_ERROR_TOO_LARGE, + SIMDUTF_ERROR_SURROGATE, + SIMDUTF_ERROR_INVALID_BASE64_CHARACTER, + SIMDUTF_ERROR_BASE64_INPUT_REMAINDER, + SIMDUTF_ERROR_BASE64_EXTRA_BITS, + SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL, + SIMDUTF_ERROR_OTHER +} simdutf_error_code; + +typedef struct simdutf_result { + simdutf_error_code error; + size_t count; /* position of error or number of code units validated */ +} simdutf_result; + +typedef struct simdutf_full_result { + simdutf_error_code error; + size_t input_count; /* number of input units consumed */ + size_t output_count; /* number of output bytes written */ +} simdutf_full_result; + +typedef enum simdutf_encoding_type { + SIMDUTF_ENCODING_UNSPECIFIED = 0, + SIMDUTF_ENCODING_UTF8 = 1, + SIMDUTF_ENCODING_UTF16_LE = 2, + SIMDUTF_ENCODING_UTF16_BE = 4, + SIMDUTF_ENCODING_UTF32_LE = 8, + SIMDUTF_ENCODING_UTF32_BE = 16 +} simdutf_encoding_type; + +/* Validate UTF-8: returns true iff input is valid UTF-8 */ +bool simdutf_validate_utf8(const char *buf, size_t len); + +/* Validate UTF-8 with detailed result */ +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len); + +/* Encoding detection */ +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length); +int simdutf_detect_encodings(const char *input, size_t length); + +/* ASCII validation */ +bool simdutf_validate_ascii(const char *buf, size_t len); +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len); + +/* UTF-16 ASCII checks */ +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len); + +/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */ +bool simdutf_validate_utf16(const char16_t *buf, size_t len); +bool simdutf_validate_utf16le(const char16_t *buf, size_t len); +bool simdutf_validate_utf16be(const char16_t *buf, size_t len); +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len); +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len); + +bool simdutf_validate_utf32(const char32_t *buf, size_t len); +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len); + +/* to_well_formed UTF-16 helpers */ +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output); +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output); + +/* Counting */ +size_t simdutf_count_utf16(const char16_t *input, size_t length); +size_t simdutf_count_utf16le(const char16_t *input, size_t length); +size_t simdutf_count_utf16be(const char16_t *input, size_t length); +size_t simdutf_count_utf8(const char *input, size_t length); + +/* Length estimators */ +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length); +size_t simdutf_latin1_length_from_utf16(size_t length); +size_t simdutf_latin1_length_from_utf32(size_t length); +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length); +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf32(const char32_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length); +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length); +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length); +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length); +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length); + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output); +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf16(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output); + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output); + +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output); +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output); +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output); + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output); +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output); + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len); +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output); +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output); + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output); +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output); + +/* Convert possibly broken UTF-16 to UTF-8, replacing each unpaired surrogate + with U+FFFD (EF BF BD). These always succeed and return the number of bytes + written. Size the output buffer with the matching + simdutf_utf8_length_from_utf16*_with_replacement function. */ +size_t simdutf_convert_utf16_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output); +size_t simdutf_convert_utf16le_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output); +size_t simdutf_convert_utf16be_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output); + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output); +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output); + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output); +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output); + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output); +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output); +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output); +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output); + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output); +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output); + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output); +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output); +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output); + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output); +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output); +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output); + +/* --- Find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character); +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character); + +/* --- Base64 enums and helpers --- */ +typedef enum simdutf_base64_options { + SIMDUTF_BASE64_DEFAULT = 0, + SIMDUTF_BASE64_URL = 1, + SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2, + SIMDUTF_BASE64_URL_WITH_PADDING = 3, + SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4, + SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5, + SIMDUTF_BASE64_DEFAULT_OR_URL = 8, + SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12 +} simdutf_base64_options; + +typedef enum simdutf_last_chunk_handling_options { + SIMDUTF_LAST_CHUNK_LOOSE = 0, + SIMDUTF_LAST_CHUNK_STRICT = 1, + SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2, + SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3 +} simdutf_last_chunk_handling_options; + +/* maximal binary length estimators */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length); +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length); + +/* base64 decoding/encoding */ +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options); +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length); + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options); +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options); + +/* safe decoding that provides an in/out outlen parameter */ +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char); + +/* detailed decoding returning input_count and output_count */ +simdutf_full_result simdutf_base64_to_binary_details( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); +simdutf_full_result simdutf_base64_to_binary_details_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options); + +/* single-character base64 validation */ +bool simdutf_base64_valid(char input, simdutf_base64_options options); +bool simdutf_base64_valid_utf16(char16_t input, simdutf_base64_options options); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SIMDUTF_C_H */ +/* end file include/simdutf_c.h */ + +static simdutf_result to_c_result(const simdutf::result &r) { + simdutf_result out; + out.error = static_cast(r.error); + out.count = r.count; + return out; +} + +/* The C wrapper depends on the library features. Only expose the C API + when all relevant feature is enabled. This helps the + single-header generator to omit the C wrapper when features are + disabled. */ +// clang-format off +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 && SIMDUTF_FEATURE_ASCII && SIMDUTF_FEATURE_BASE64 && SIMDUTF_FEATURE_DETECT_ENCODING +// clang-format on +extern "C" { + +bool simdutf_validate_utf8(const char *buf, size_t len) { + return simdutf::validate_utf8(buf, len); +} + +simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len) { + return to_c_result(simdutf::validate_utf8_with_errors(buf, len)); +} + +simdutf_encoding_type simdutf_autodetect_encoding(const char *input, + size_t length) { + return static_cast( + simdutf::autodetect_encoding(input, length)); +} + +int simdutf_detect_encodings(const char *input, size_t length) { + return simdutf::detect_encodings(input, length); +} + +bool simdutf_validate_ascii(const char *buf, size_t len) { + return simdutf::validate_ascii(buf, len); +} +simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len) { + return to_c_result(simdutf::validate_ascii_with_errors(buf, len)); +} + +bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len) { + return simdutf::validate_utf16_as_ascii(buf, len); +} +bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len) { + return simdutf::validate_utf16be_as_ascii(buf, len); +} +bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len) { + return simdutf::validate_utf16le_as_ascii(buf, len); +} + +bool simdutf_validate_utf16(const char16_t *buf, size_t len) { + return simdutf::validate_utf16(buf, len); +} +bool simdutf_validate_utf16le(const char16_t *buf, size_t len) { + return simdutf::validate_utf16le(buf, len); +} +bool simdutf_validate_utf16be(const char16_t *buf, size_t len) { + return simdutf::validate_utf16be(buf, len); +} +simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf, + size_t len) { + return to_c_result(simdutf::validate_utf16_with_errors(buf, len)); +} +simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf, + size_t len) { + return to_c_result(simdutf::validate_utf16le_with_errors(buf, len)); +} +simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf, + size_t len) { + return to_c_result(simdutf::validate_utf16be_with_errors(buf, len)); +} + +bool simdutf_validate_utf32(const char32_t *buf, size_t len) { + return simdutf::validate_utf32(buf, len); +} +simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf, + size_t len) { + return to_c_result(simdutf::validate_utf32_with_errors(buf, len)); +} + +void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len, + char16_t *output) { + simdutf::to_well_formed_utf16le(input, len, output); +} +void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len, + char16_t *output) { + simdutf::to_well_formed_utf16be(input, len, output); +} +void simdutf_to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + simdutf::to_well_formed_utf16(input, len, output); +} + +size_t simdutf_count_utf16(const char16_t *input, size_t length) { + return simdutf::count_utf16(input, length); +} +size_t simdutf_count_utf16le(const char16_t *input, size_t length) { + return simdutf::count_utf16le(input, length); +} +size_t simdutf_count_utf16be(const char16_t *input, size_t length) { + return simdutf::count_utf16be(input, length); +} +size_t simdutf_count_utf8(const char *input, size_t length) { + return simdutf::count_utf8(input, length); +} + +size_t simdutf_utf8_length_from_latin1(const char *input, size_t length) { + return simdutf::utf8_length_from_latin1(input, length); +} +size_t simdutf_latin1_length_from_utf8(const char *input, size_t length) { + return simdutf::latin1_length_from_utf8(input, length); +} +size_t simdutf_latin1_length_from_utf16(size_t length) { + return simdutf::latin1_length_from_utf16(length); +} +size_t simdutf_latin1_length_from_utf32(size_t length) { + return simdutf::latin1_length_from_utf32(length); +} +size_t simdutf_utf16_length_from_utf8(const char *input, size_t length) { + return simdutf::utf16_length_from_utf8(input, length); +} +size_t simdutf_utf32_length_from_utf8(const char *input, size_t length) { + return simdutf::utf32_length_from_utf8(input, length); +} +size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length) { + return simdutf::utf8_length_from_utf16(input, length); +} +size_t simdutf_utf8_length_from_utf32(const char32_t *input, size_t length) { + return simdutf::utf8_length_from_utf32(input, length); +} +simdutf_result +simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input, + size_t length) { + return to_c_result( + simdutf::utf8_length_from_utf16_with_replacement(input, length)); +} +size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length) { + return simdutf::utf8_length_from_utf16le(input, length); +} +size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length) { + return simdutf::utf8_length_from_utf16be(input, length); +} +simdutf_result +simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input, + size_t length) { + return to_c_result( + simdutf::utf8_length_from_utf16le_with_replacement(input, length)); +} +simdutf_result +simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input, + size_t length) { + return to_c_result( + simdutf::utf8_length_from_utf16be_with_replacement(input, length)); +} + +/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */ +size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length, + char *output) { + return simdutf::convert_latin1_to_utf8(input, length, output); +} + +size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length, + char *output, size_t utf8_len) { + return simdutf::convert_latin1_to_utf8_safe(input, length, output, utf8_len); +} +size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_latin1_to_utf16le(input, length, output); +} +size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_latin1_to_utf16be(input, length, output); +} +size_t simdutf_convert_latin1_to_utf16(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_latin1_to_utf16(input, length, output); +} +size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length, + char32_t *output) { + return simdutf::convert_latin1_to_utf32(input, length, output); +} + +size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length, + char *output) { + return simdutf::convert_utf8_to_latin1(input, length, output); +} +size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_utf8_to_utf16le(input, length, output); +} +size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_utf8_to_utf16(input, length, output); +} +size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_utf8_to_utf16be(input, length, output); +} +size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length, + char32_t *output) { + return simdutf::convert_utf8_to_utf32(input, length, output); +} +simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input, + size_t length, + char *output) { + return to_c_result( + simdutf::convert_utf8_to_latin1_with_errors(input, length, output)); +} +simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input, + size_t length, + char16_t *output) { + return to_c_result( + simdutf::convert_utf8_to_utf16_with_errors(input, length, output)); +} +simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input, + size_t length, + char16_t *output) { + return to_c_result( + simdutf::convert_utf8_to_utf16le_with_errors(input, length, output)); +} +simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input, + size_t length, + char16_t *output) { + return to_c_result( + simdutf::convert_utf8_to_utf16be_with_errors(input, length, output)); +} +simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input, + size_t length, + char32_t *output) { + return to_c_result( + simdutf::convert_utf8_to_utf32_with_errors(input, length, output)); +} + +/* Conversions assuming valid input */ +size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length, + char *output) { + return simdutf::convert_valid_utf8_to_latin1(input, length, output); +} +size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_valid_utf8_to_utf16le(input, length, output); +} +size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length, + char16_t *output) { + return simdutf::convert_valid_utf8_to_utf16be(input, length, output); +} +size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length, + char32_t *output) { + return simdutf::convert_valid_utf8_to_utf32(input, length, output); +} + +/* UTF-16 -> UTF-8 and related conversions */ +size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16_to_utf8(input, length, output); +} +size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length, + char *output, size_t utf8_len) { + return simdutf::convert_utf16_to_utf8_safe(input, length, output, utf8_len); +} +size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16_to_latin1(input, length, output); +} +size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16le_to_latin1(input, length, output); +} +size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16be_to_latin1(input, length, output); +} +simdutf_result +simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf16_to_latin1_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf16le_to_latin1_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf16be_to_latin1_with_errors(input, length, output)); +} + +simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input, + size_t length, + char *output) { + return to_c_result( + simdutf::convert_utf16_to_utf8_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf16le_to_utf8_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf16be_to_utf8_with_errors(input, length, output)); +} + +size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16le_to_utf8(input, length, output); +} +size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_utf16be_to_utf8(input, length, output); +} + +size_t simdutf_convert_utf16_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output) { + return simdutf::convert_utf16_to_utf8_with_replacement(input, length, output); +} +size_t simdutf_convert_utf16le_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output) { + return simdutf::convert_utf16le_to_utf8_with_replacement(input, length, + output); +} +size_t simdutf_convert_utf16be_to_utf8_with_replacement(const char16_t *input, + size_t length, + char *output) { + return simdutf::convert_utf16be_to_utf8_with_replacement(input, length, + output); +} + +size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length, + char *output) { + return simdutf::convert_valid_utf16_to_utf8(input, length, output); +} +size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input, + size_t length, char *output) { + return simdutf::convert_valid_utf16_to_latin1(input, length, output); +} +size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input, + size_t length, char *output) { + return simdutf::convert_valid_utf16le_to_latin1(input, length, output); +} +size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input, + size_t length, char *output) { + return simdutf::convert_valid_utf16be_to_latin1(input, length, output); +} + +size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input, + size_t length, char *output) { + return simdutf::convert_valid_utf16le_to_utf8(input, length, output); +} +size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input, + size_t length, char *output) { + return simdutf::convert_valid_utf16be_to_utf8(input, length, output); +} + +/* UTF-16 <-> UTF-32 conversions */ +size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length, + char32_t *output) { + return simdutf::convert_utf16_to_utf32(input, length, output); +} +size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length, + char32_t *output) { + return simdutf::convert_utf16le_to_utf32(input, length, output); +} +size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length, + char32_t *output) { + return simdutf::convert_utf16be_to_utf32(input, length, output); +} +simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input, + size_t length, + char32_t *output) { + return to_c_result( + simdutf::convert_utf16_to_utf32_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output) { + return to_c_result( + simdutf::convert_utf16le_to_utf32_with_errors(input, length, output)); +} +simdutf_result +simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input, + size_t length, char32_t *output) { + return to_c_result( + simdutf::convert_utf16be_to_utf32_with_errors(input, length, output)); +} + +/* Valid UTF-16 conversions */ +size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input, + size_t length, char32_t *output) { + return simdutf::convert_valid_utf16_to_utf32(input, length, output); +} +size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input, + size_t length, char32_t *output) { + return simdutf::convert_valid_utf16le_to_utf32(input, length, output); +} +size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input, + size_t length, char32_t *output) { + return simdutf::convert_valid_utf16be_to_utf32(input, length, output); +} + +/* UTF-32 -> ... conversions */ +size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length, + char *output) { + return simdutf::convert_utf32_to_utf8(input, length, output); +} +simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input, + size_t length, + char *output) { + return to_c_result( + simdutf::convert_utf32_to_utf8_with_errors(input, length, output)); +} +size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length, + char *output) { + return simdutf::convert_valid_utf32_to_utf8(input, length, output); +} + +size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length, + char16_t *output) { + return simdutf::convert_utf32_to_utf16(input, length, output); +} +size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length, + char16_t *output) { + return simdutf::convert_utf32_to_utf16le(input, length, output); +} +size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length, + char16_t *output) { + return simdutf::convert_utf32_to_utf16be(input, length, output); +} +simdutf_result +simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input, + size_t length, char *output) { + return to_c_result( + simdutf::convert_utf32_to_latin1_with_errors(input, length, output)); +} + +/* --- find helpers --- */ +const char *simdutf_find(const char *start, const char *end, char character) { + return simdutf::find(start, end, character); +} +const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end, + char16_t character) { + return simdutf::find(start, end, character); +} + +/* --- base64 helpers --- */ +size_t simdutf_maximal_binary_length_from_base64(const char *input, + size_t length) { + return simdutf::maximal_binary_length_from_base64(input, length); +} +size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input, + size_t length) { + return simdutf::maximal_binary_length_from_base64(input, length); +} + +simdutf_result simdutf_base64_to_binary( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options) { + return to_c_result(simdutf::base64_to_binary( + input, length, output, static_cast(options), + static_cast(last_chunk_options))); +} +simdutf_result simdutf_base64_to_binary_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options) { + return to_c_result(simdutf::base64_to_binary( + input, length, output, static_cast(options), + static_cast(last_chunk_options))); +} + +size_t simdutf_base64_length_from_binary(size_t length, + simdutf_base64_options options) { + return simdutf::base64_length_from_binary( + length, static_cast(options)); +} +size_t simdutf_base64_length_from_binary_with_lines( + size_t length, simdutf_base64_options options, size_t line_length) { + return simdutf::base64_length_from_binary_with_lines( + length, static_cast(options), line_length); +} + +size_t simdutf_binary_to_base64(const char *input, size_t length, char *output, + simdutf_base64_options options) { + return simdutf::binary_to_base64( + input, length, output, static_cast(options)); +} +size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length, + char *output, size_t line_length, + simdutf_base64_options options) { + return simdutf::binary_to_base64_with_lines( + input, length, output, line_length, + static_cast(options)); +} + +simdutf_result simdutf_base64_to_binary_safe( + const char *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char) { + size_t local_out = outlen ? *outlen : 0; + simdutf::result r = simdutf::base64_to_binary_safe( + input, length, output, local_out, + static_cast(options), + static_cast(last_chunk_options), + decode_up_to_bad_char); + if (outlen) + *outlen = local_out; + return to_c_result(r); +} +simdutf_result simdutf_base64_to_binary_safe_utf16( + const char16_t *input, size_t length, char *output, size_t *outlen, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options, + bool decode_up_to_bad_char) { + size_t local_out = outlen ? *outlen : 0; + simdutf::result r = simdutf::base64_to_binary_safe( + input, length, output, local_out, + static_cast(options), + static_cast(last_chunk_options), + decode_up_to_bad_char); + if (outlen) + *outlen = local_out; + return to_c_result(r); +} + +static simdutf_full_result to_c_full_result(const simdutf::full_result &r) { + simdutf_full_result out; + out.error = static_cast(r.error); + out.input_count = r.input_count; + out.output_count = r.output_count; + return out; +} + +simdutf_full_result simdutf_base64_to_binary_details( + const char *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options) { + return to_c_full_result(simdutf::base64_to_binary_details( + input, length, output, static_cast(options), + static_cast(last_chunk_options))); +} +simdutf_full_result simdutf_base64_to_binary_details_utf16( + const char16_t *input, size_t length, char *output, + simdutf_base64_options options, + simdutf_last_chunk_handling_options last_chunk_options) { + return to_c_full_result(simdutf::base64_to_binary_details( + input, length, output, static_cast(options), + static_cast(last_chunk_options))); +} + +bool simdutf_base64_valid(char input, simdutf_base64_options options) { + return simdutf::base64_valid(input, + static_cast(options)); +} +bool simdutf_base64_valid_utf16(char16_t input, + simdutf_base64_options options) { + return simdutf::base64_valid(input, + static_cast(options)); +} + +} // extern "C" +// clang-format off +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 && SIMDUTF_FEATURE_ASCII && SIMDUTF_FEATURE_BASE64 && SIMDUTF_FEATURE_DETECT_ENCODING +// clang-format on +/* end file src/simdutf_c.cpp */ SIMDUTF_POP_DISABLE_WARNINGS /* end file src/simdutf.cpp */ diff --git a/deps/v8/third_party/simdutf/simdutf.h b/deps/v8/third_party/simdutf/simdutf.h index a7a27a669798..b090840f635a 100644 --- a/deps/v8/third_party/simdutf/simdutf.h +++ b/deps/v8/third_party/simdutf/simdutf.h @@ -1,4 +1,4 @@ -/* auto-generated on 2025-11-21 22:27:43 -0500. Do not edit! */ +/* auto-generated on 2026-08-18 10:18:07 -0400. Do not edit! */ /* begin file include/simdutf.h */ #ifndef SIMDUTF_H #define SIMDUTF_H @@ -20,6 +20,11 @@ #endif #endif +// C++ 26 +#if !defined(SIMDUTF_CPLUSPLUS26) && (SIMDUTF_CPLUSPLUS >= 202602L) + #define SIMDUTF_CPLUSPLUS26 1 +#endif + // C++ 23 #if !defined(SIMDUTF_CPLUSPLUS23) && (SIMDUTF_CPLUSPLUS >= 202302L) #define SIMDUTF_CPLUSPLUS23 1 @@ -45,8 +50,8 @@ #define SIMDUTF_CPLUSPLUS11 1 #endif -#ifndef SIMDUTF_CPLUSPLUS11 - #error simdutf requires a compiler compliant with the C++11 standard +#ifndef SIMDUTF_CPLUSPLUS17 + #error simdutf requires a compiler compliant with the C++17 standard #endif #endif // SIMDUTF_COMPILER_CHECK_H @@ -63,6 +68,7 @@ #include #include #include +#include #include #ifndef _WIN32 // strcasecmp, strncasecmp @@ -200,9 +206,25 @@ #elif defined(__loongarch_lp64) #if defined(__loongarch_sx) && defined(__loongarch_asx) #define SIMDUTF_IS_LSX 1 - #define SIMDUTF_IS_LASX 1 + #define SIMDUTF_IS_LASX 1 // We can always run both #elif defined(__loongarch_sx) #define SIMDUTF_IS_LSX 1 + // Adjust for runtime dispatching support. + #if defined(__GNUC__) && !defined(__clang__) && \ + !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) + #if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) + // We are ok, we will support runtime dispatch for LASX. + #else + // We disable runtime dispatch for LASX, which means that we will not be + // able to use LASX even if it is supported by the hardware. Loongson + // users should update to GCC 15 or better. + #define SIMDUTF_IMPLEMENTATION_LASX 0 + #endif + #else + // We are not using GCC, so we assume that we can support runtime dispatch + // for LASX. https://godbolt.org/z/jcMnrjYhs + #define SIMDUTF_IMPLEMENTATION_LASX 0 + #endif #endif #else // The simdutf library is designed @@ -250,7 +272,7 @@ // // We are going to use runtime dispatch. -#ifdef SIMDUTF_IS_X86_64 +#if defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang @@ -267,7 +289,7 @@ #define SIMDUTF_UNTARGET_REGION _Pragma("GCC pop_options") #endif // clang then gcc -#endif // x86 +#endif // defined(SIMDUTF_IS_X86_64) || defined(SIMDUTF_IS_LSX) // Default target region macros don't do anything. #ifndef SIMDUTF_TARGET_REGION @@ -410,6 +432,7 @@ // Sometimes logging is useful, but we want it disabled by default // and free of any logging code in release builds. #ifdef SIMDUTF_LOGGING + #include #include #define simdutf_log(msg) \ std::cout << "[" << __FUNCTION__ << "]: " << msg << std::endl \ @@ -427,6 +450,10 @@ #define simdutf_log_assert(cond, msg) #endif +#if SIMDUTF_CPLUSPLUS17 + #define simdutf_unused [[maybe_unused]] +#endif // SIMDUTF_CPLUSPLUS17 + #if defined(SIMDUTF_REGULAR_VISUAL_STUDIO) #define SIMDUTF_DEPRECATED __declspec(deprecated) @@ -434,7 +461,9 @@ #define simdutf_always_inline __forceinline // always inline, no matter what #define simdutf_never_inline __declspec(noinline) - #define simdutf_unused + #ifndef simdutf_unused + #define simdutf_unused + #endif // simdutf_unused #define simdutf_warn_unused #ifndef simdutf_likely @@ -477,8 +506,9 @@ inline __attribute__((always_inline)) // always inline, no matter what #define SIMDUTF_DEPRECATED __attribute__((deprecated)) #define simdutf_never_inline inline __attribute__((noinline)) - - #define simdutf_unused __attribute__((unused)) + #ifndef simdutf_unused + #define simdutf_unused __attribute__((unused)) + #endif // simdutf_unused #define simdutf_warn_unused __attribute__((warn_unused_result)) #ifndef simdutf_likely @@ -526,6 +556,14 @@ #endif // MSC_VER +// Will evaluate to constexpr in C++23 or later. This makes it possible to mark +// functions constexpr if the "if consteval" feature is available to use. +#if SIMDUTF_CPLUSPLUS23 + #define simdutf_constexpr23 constexpr +#else + #define simdutf_constexpr23 +#endif + #ifndef SIMDUTF_DLLIMPORTEXPORT #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio /** @@ -576,7 +614,13 @@ /* begin file include/simdutf/encoding_types.h */ #ifndef SIMDUTF_ENCODING_TYPES_H #define SIMDUTF_ENCODING_TYPES_H -#include +#include + +#if !defined(SIMDUTF_NO_STD_TEXT_ENCODING) && \ + defined(__cpp_lib_text_encoding) && __cpp_lib_text_encoding >= 202306L + #define SIMDUTF_HAS_STD_TEXT_ENCODING 1 + #include +#endif namespace simdutf { @@ -591,48 +635,176 @@ enum encoding_type { unspecified = 0 }; -enum endianness { LITTLE = 0, BIG = 1 }; - -constexpr bool match_system(endianness e) { #ifndef SIMDUTF_IS_BIG_ENDIAN #error "SIMDUTF_IS_BIG_ENDIAN needs to be defined." #endif + +enum endianness { + LITTLE = 0, + BIG = 1, + NATIVE = #if SIMDUTF_IS_BIG_ENDIAN - return e == endianness::BIG; + BIG #else - return e == endianness::LITTLE; + LITTLE #endif +}; + +simdutf_warn_unused simdutf_really_inline constexpr bool +match_system(endianness e) { + return e == endianness::NATIVE; } -std::string to_string(encoding_type bom); +simdutf_warn_unused std::string_view to_string(encoding_type bom); // Note that BOM for UTF8 is discouraged. namespace BOM { /** * Checks for a BOM. If not, returns unspecified - * @param input the string to process + * @param byte the string to process * @param length the length of the string in code units * @return the corresponding encoding */ -encoding_type check_bom(const uint8_t *byte, size_t length); -encoding_type check_bom(const char *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const uint8_t *byte, size_t length); +simdutf_warn_unused encoding_type check_bom(const char *byte, size_t length); /** * Returns the size, in bytes, of the BOM for a given encoding type. * Note that UTF8 BOM are discouraged. * @param bom the encoding type * @return the size in bytes of the corresponding BOM */ -size_t bom_byte_size(encoding_type bom); +simdutf_warn_unused size_t bom_byte_size(encoding_type bom); } // namespace BOM + +#ifdef SIMDUTF_HAS_STD_TEXT_ENCODING +/** + * Convert a simdutf encoding type to a std::text_encoding. + * + * @param enc the simdutf encoding type + * @return the corresponding std::text_encoding, or + * std::text_encoding::id::unknown for unspecified/unsupported + */ +simdutf_warn_unused constexpr std::text_encoding +to_std_encoding(encoding_type enc) noexcept { + switch (enc) { + case UTF8: + return std::text_encoding(std::text_encoding::id::UTF8); + case UTF16_LE: + return std::text_encoding(std::text_encoding::id::UTF16LE); + case UTF16_BE: + return std::text_encoding(std::text_encoding::id::UTF16BE); + case UTF32_LE: + return std::text_encoding(std::text_encoding::id::UTF32LE); + case UTF32_BE: + return std::text_encoding(std::text_encoding::id::UTF32BE); + case Latin1: + return std::text_encoding(std::text_encoding::id::ISOLatin1); + case unspecified: + default: + return std::text_encoding(std::text_encoding::id::unknown); + } +} + +/** + * Convert a std::text_encoding to a simdutf encoding type. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} + +/** + * Get the native-endian UTF-16 encoding type for this system. + * + * @return UTF16_LE on little-endian systems, UTF16_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf16_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF16_BE; + #else + return UTF16_LE; + #endif +} + +/** + * Get the native-endian UTF-32 encoding type for this system. + * + * @return UTF32_LE on little-endian systems, UTF32_BE on big-endian systems + */ +simdutf_warn_unused constexpr encoding_type native_utf32_encoding() noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return UTF32_BE; + #else + return UTF32_LE; + #endif +} + +/** + * Convert a std::text_encoding to a simdutf encoding type, + * using native endianness for UTF-16/UTF-32 without explicit endianness. + * + * When the input is std::text_encoding::id::UTF16 or UTF32 (without LE/BE + * suffix), this returns the native-endian simdutf variant. + * + * @param enc the std::text_encoding + * @return the corresponding simdutf encoding type, or + * encoding_type::unspecified if the encoding is not supported + */ +simdutf_warn_unused constexpr encoding_type +from_std_encoding_native(const std::text_encoding &enc) noexcept { + switch (enc.mib()) { + case std::text_encoding::id::UTF8: + return UTF8; + case std::text_encoding::id::UTF16: + return native_utf16_encoding(); + case std::text_encoding::id::UTF16LE: + return UTF16_LE; + case std::text_encoding::id::UTF16BE: + return UTF16_BE; + case std::text_encoding::id::UTF32: + return native_utf32_encoding(); + case std::text_encoding::id::UTF32LE: + return UTF32_LE; + case std::text_encoding::id::UTF32BE: + return UTF32_BE; + case std::text_encoding::id::ISOLatin1: + return Latin1; + default: + return unspecified; + } +} +#endif // SIMDUTF_HAS_STD_TEXT_ENCODING + } // namespace simdutf #endif /* end file include/simdutf/encoding_types.h */ /* begin file include/simdutf/error.h */ #ifndef SIMDUTF_ERROR_H #define SIMDUTF_ERROR_H +#include + namespace simdutf { enum error_code { @@ -674,7 +846,7 @@ enum error_code { OUTPUT_BUFFER_TOO_SMALL, // The provided buffer is too small. OTHER // Not related to validation/transcoding. }; -#if SIMDUTF_CPLUSPLUS17 + inline std::string_view error_to_string(error_code code) noexcept { switch (code) { case SUCCESS: @@ -703,7 +875,6 @@ inline std::string_view error_to_string(error_code code) noexcept { return "OTHER"; } } -#endif struct result { error_code error; @@ -711,17 +882,18 @@ struct result { // case of success, indicates the number of code units // validated/written. - simdutf_really_inline result() noexcept + simdutf_really_inline simdutf_constexpr23 result() noexcept : error{error_code::SUCCESS}, count{0} {} - simdutf_really_inline result(error_code err, size_t pos) noexcept + simdutf_really_inline simdutf_constexpr23 result(error_code err, + size_t pos) noexcept : error{err}, count{pos} {} - simdutf_really_inline bool is_ok() const noexcept { + simdutf_really_inline simdutf_constexpr23 bool is_ok() const noexcept { return error == error_code::SUCCESS; } - simdutf_really_inline bool is_err() const noexcept { + simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { return error != error_code::SUCCESS; } }; @@ -733,18 +905,19 @@ struct full_result { bool padding_error = false; // true if the error is due to padding, only // meaningful when error is not SUCCESS - simdutf_really_inline full_result() noexcept + simdutf_really_inline simdutf_constexpr23 full_result() noexcept : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} - simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out) noexcept + simdutf_really_inline simdutf_constexpr23 full_result(error_code err, + size_t pos_in, + size_t pos_out) noexcept : error{err}, input_count{pos_in}, output_count{pos_out} {} - simdutf_really_inline full_result(error_code err, size_t pos_in, - size_t pos_out, bool padding_err) noexcept + simdutf_really_inline simdutf_constexpr23 full_result( + error_code err, size_t pos_in, size_t pos_out, bool padding_err) noexcept : error{err}, input_count{pos_in}, output_count{pos_out}, padding_error{padding_err} {} - simdutf_really_inline operator result() const noexcept { + simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { if (error == error_code::SUCCESS) { return result{error, output_count}; } else { @@ -768,18 +941,18 @@ SIMDUTF_DISABLE_UNDESIRED_WARNINGS #define SIMDUTF_SIMDUTF_VERSION_H /** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION "7.7.0" +#define SIMDUTF_VERSION "9.1.0" namespace simdutf { enum { /** * The major version (MAJOR.minor.revision) of simdutf being used. */ - SIMDUTF_VERSION_MAJOR = 7, + SIMDUTF_VERSION_MAJOR = 9, /** * The minor version (major.MINOR.revision) of simdutf being used. */ - SIMDUTF_VERSION_MINOR = 7, + SIMDUTF_VERSION_MINOR = 1, /** * The revision (major.minor.REVISION) of simdutf being used. */ @@ -795,7 +968,6 @@ enum { #if !defined(SIMDUTF_NO_THREADS) #include #endif -#include #ifdef SIMDUTF_INTERNAL_TESTS #include #endif @@ -852,10 +1024,15 @@ POSSIBILITY OF SUCH DAMAGE. #include #if defined(_MSC_VER) #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) #include #endif +#ifdef __FILC__ + #include +#endif + // RISC-V ISA detection utilities #if SIMDUTF_IS_RISCV64 && defined(__linux__) @@ -1003,7 +1180,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; - #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + #elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -1019,6 +1197,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv() { #if defined(_MSC_VER) return _xgetbv(0); + #elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); @@ -1126,14 +1306,13 @@ static inline uint32_t detect_supported_architectures() { #endif // SIMDutf_INTERNAL_ISADETECTION_H /* end file include/simdutf/internal/isadetection.h */ +#include #if SIMDUTF_SPAN #include #include #include #include -#endif -#if SIMDUTF_CPLUSPLUS17 - #include + #include // for std::unreachable #endif // The following defines are conditionally enabled/disabled during amalgamation. // By default all features are enabled, regular code shouldn't check them. Only @@ -1144,31 +1323,199 @@ static inline uint32_t detect_supported_architectures() { // # error("Please amalgamate simdutf with UTF-16 support") // #endif // -#define SIMDUTF_FEATURE_DETECT_ENCODING 1 -#define SIMDUTF_FEATURE_ASCII 1 -#define SIMDUTF_FEATURE_LATIN1 1 -#define SIMDUTF_FEATURE_UTF8 1 -#define SIMDUTF_FEATURE_UTF16 1 -#define SIMDUTF_FEATURE_UTF32 1 -#define SIMDUTF_FEATURE_BASE64 1 +#ifndef SIMDUTF_FEATURE_DETECT_ENCODING + #define SIMDUTF_FEATURE_DETECT_ENCODING 1 +#endif +#ifndef SIMDUTF_FEATURE_ASCII + #define SIMDUTF_FEATURE_ASCII 1 +#endif +#ifndef SIMDUTF_FEATURE_LATIN1 + #define SIMDUTF_FEATURE_LATIN1 1 +#endif +#ifndef SIMDUTF_FEATURE_UTF8 + #define SIMDUTF_FEATURE_UTF8 1 +#endif +#ifndef SIMDUTF_FEATURE_UTF16 + #define SIMDUTF_FEATURE_UTF16 1 +#endif +#ifndef SIMDUTF_FEATURE_UTF32 + #define SIMDUTF_FEATURE_UTF32 1 +#endif +#ifndef SIMDUTF_FEATURE_BASE64 + #define SIMDUTF_FEATURE_BASE64 1 +#endif + +/// helpers placed in namespace detail are not a part of the public API +namespace simdutf { +namespace detail { +namespace { +// this is to avoid including just for min +constexpr std::size_t min(std::size_t a, std::size_t b) { + return a < b ? a : b; +} +template +constexpr std::size_t min(const T &a, const U &b) = delete; +} // namespace +} // namespace detail +} // namespace simdutf + +#if SIMDUTF_CPLUSPLUS23 +/* begin file include/simdutf/constexpr_ptr.h */ +#ifndef SIMDUTF_CONSTEXPR_PTR_H +#define SIMDUTF_CONSTEXPR_PTR_H + +#include namespace simdutf { +namespace detail { +/** + * The constexpr_ptr class is a workaround for reinterpret_cast not being + * allowed during constant evaluation. + */ +template + requires(sizeof(to) == sizeof(from)) +struct constexpr_ptr { + const from *p; -constexpr size_t default_line_length = - 76; ///< default line length for base64 encoding with lines + constexpr explicit constexpr_ptr(const from *ptr) noexcept : p(ptr) {} + + constexpr to operator*() const noexcept { return static_cast(*p); } + + constexpr constexpr_ptr &operator++() noexcept { + ++p; + return *this; + } + + constexpr constexpr_ptr operator++(int) noexcept { + auto old = *this; + ++p; + return old; + } + + constexpr constexpr_ptr &operator--() noexcept { + --p; + return *this; + } + + constexpr constexpr_ptr operator--(int) noexcept { + auto old = *this; + --p; + return old; + } + + constexpr constexpr_ptr &operator+=(std::ptrdiff_t n) noexcept { + p += n; + return *this; + } + + constexpr constexpr_ptr &operator-=(std::ptrdiff_t n) noexcept { + p -= n; + return *this; + } + + constexpr constexpr_ptr operator+(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p + n}; + } + + constexpr constexpr_ptr operator-(std::ptrdiff_t n) const noexcept { + return constexpr_ptr{p - n}; + } + + constexpr std::ptrdiff_t operator-(const constexpr_ptr &o) const noexcept { + return p - o.p; + } + + constexpr to operator[](std::ptrdiff_t n) const noexcept { + return static_cast(*(p + n)); + } + + // to prevent compilation errors for memcpy, even if it is never + // called during constant evaluation + constexpr operator const void *() const noexcept { return p; } +}; + +template +constexpr constexpr_ptr constexpr_cast_ptr(from *p) noexcept { + return constexpr_ptr{p}; +} + +/** + * helper type for constexpr_writeptr, so it is possible to + * do "*ptr = val;" + */ +template +struct constexpr_write_ptr_proxy { + + constexpr explicit constexpr_write_ptr_proxy(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy &operator=(SrcType v) { + *p = static_cast(v); + return *this; + } + + TargetType *p; +}; + +/** + * helper for working around reinterpret_cast not being allowed during constexpr + * evaluation. will try to act as a SrcType* but actually write to the pointer + * given in the constructor, which is of another type TargetType + */ +template struct constexpr_write_ptr { + constexpr explicit constexpr_write_ptr(TargetType *raw) : p(raw) {} + + constexpr constexpr_write_ptr_proxy operator*() const { + return constexpr_write_ptr_proxy{p}; + } + + constexpr constexpr_write_ptr_proxy + operator[](std::ptrdiff_t n) const { + return constexpr_write_ptr_proxy{p + n}; + } + + constexpr constexpr_write_ptr &operator++() { + ++p; + return *this; + } + + constexpr constexpr_write_ptr operator++(int) { + constexpr_write_ptr old = *this; + ++p; + return old; + } + + constexpr std::ptrdiff_t operator-(const constexpr_write_ptr &other) const { + return p - other.p; + } + + TargetType *p; +}; + +template +constexpr auto constexpr_cast_writeptr(TargetType *raw) { + return constexpr_write_ptr{raw}; +} + +} // namespace detail +} // namespace simdutf +#endif +/* end file include/simdutf/constexpr_ptr.h */ +#endif #if SIMDUTF_SPAN /// helpers placed in namespace detail are not a part of the public API +namespace simdutf { namespace detail { /** * matches a byte, in the many ways C++ allows. note that these * are all distinct types. */ template -concept byte_like = std::is_same_v || // - std::is_same_v || // - std::is_same_v || // - std::is_same_v; +concept byte_like = std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v || // + std::is_same_v; template concept is_byte_like = byte_like>; @@ -1201,134 +1548,3719 @@ concept output_span_of_byte_like = requires(T &t) { { *t.data() } noexcept -> is_byte_like; { *t.data() } noexcept -> is_mutable; }; -} // namespace detail -#endif -#if SIMDUTF_FEATURE_DETECT_ENCODING -/** - * Autodetect the encoding of the input, a single encoding is recommended. - * E.g., the function might return simdutf::encoding_type::UTF8, - * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or - * simdutf::encoding_type::UTF32_LE. - * - * @param input the string to analyze. - * @param length the length of the string in bytes. - * @return the detected encoding type - */ -simdutf_warn_unused simdutf::encoding_type -autodetect_encoding(const char *input, size_t length) noexcept; -simdutf_really_inline simdutf_warn_unused simdutf::encoding_type -autodetect_encoding(const uint8_t *input, size_t length) noexcept { - return autodetect_encoding(reinterpret_cast(input), length); -} - #if SIMDUTF_SPAN /** - * Autodetect the encoding of the input, a single encoding is recommended. - * E.g., the function might return simdutf::encoding_type::UTF8, - * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or - * simdutf::encoding_type::UTF32_LE. - * - * @param input the string to analyze. can be a anything span-like that has a - * data() and size() that points to character data: std::string, - * std::string_view, std::vector, std::span etc. - * @return the detected encoding type + * a pointer like object, when indexed, results in a byte like result. + * valid examples: char*, const char*, std::array + * invalid examples: int*, std::array */ -simdutf_really_inline simdutf_warn_unused simdutf::encoding_type -autodetect_encoding( - const detail::input_span_of_byte_like auto &input) noexcept { - return autodetect_encoding(reinterpret_cast(input.data()), - input.size()); -} - #endif // SIMDUTF_SPAN +template +concept indexes_into_byte_like = requires(InputPtr p) { + { std::decay_t{} } -> simdutf::detail::byte_like; +}; +template +concept indexes_into_utf16 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +template +concept indexes_into_utf32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; + +template +concept index_assignable_from_char = requires(InputPtr p, char s) { + { p[0] = s }; +}; /** - * Autodetect the possible encodings of the input in one pass. - * E.g., if the input might be UTF-16LE or UTF-8, this function returns - * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). - * - * Overridden by each implementation. - * - * @param input the string to analyze. - * @param length the length of the string in bytes. - * @return the detected encoding type + * a pointer like object that results in a uint32_t when indexed. + * valid examples: uint32_t* */ -simdutf_warn_unused int detect_encodings(const char *input, - size_t length) noexcept; -simdutf_really_inline simdutf_warn_unused int -detect_encodings(const uint8_t *input, size_t length) noexcept { - return detect_encodings(reinterpret_cast(input), length); +template +concept indexes_into_uint32 = requires(InputPtr p) { + { std::decay_t{} } -> std::same_as; +}; +} // namespace detail +} // namespace simdutf +#endif // SIMDUTF_SPAN + +// these includes are needed for constexpr support. they are +// not part of the public api. +/* begin file include/simdutf/scalar/swap_bytes.h */ +#ifndef SIMDUTF_SWAP_BYTES_H +#define SIMDUTF_SWAP_BYTES_H + +namespace simdutf { +namespace scalar { + +constexpr inline simdutf_warn_unused uint16_t +u16_swap_bytes(const uint16_t word) { + return uint16_t((word >> 8) | (word << 8)); } - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused int -detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { - return detect_encodings(reinterpret_cast(input.data()), - input.size()); + +constexpr inline simdutf_warn_unused uint32_t +u32_swap_bytes(const uint32_t word) { + return ((word >> 24) & 0xff) | // move byte 3 to byte 0 + ((word << 8) & 0xff0000) | // move byte 1 to byte 2 + ((word >> 8) & 0xff00) | // move byte 2 to byte 1 + ((word << 24) & 0xff000000); // byte 0 to byte 3 } - #endif // SIMDUTF_SPAN -#endif // SIMDUTF_FEATURE_DETECT_ENCODING -#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING -/** - * Validate the UTF-8 string. This function may be best when you expect - * the input to be almost always valid. Otherwise, consider using - * validate_utf8_with_errors. - * - * Overridden by each implementation. - * - * @param buf the UTF-8 string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid UTF-8. - */ -simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool -validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { - return validate_utf8(reinterpret_cast(input.data()), - input.size()); +namespace utf32 { +template constexpr uint32_t swap_if_needed(uint32_t c) { + return !match_system(big_endian) ? scalar::u32_swap_bytes(c) : c; } - #endif // SIMDUTF_SPAN -#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +} // namespace utf32 -#if SIMDUTF_FEATURE_UTF8 -/** - * Validate the UTF-8 string and stop on error. - * - * Overridden by each implementation. - * - * @param buf the UTF-8 string to validate. - * @param len the length of the string in bytes. - * @return a result pair struct (of type simdutf::result containing the two - * fields error and count) with an error code and either position of the error - * (in the input in code units) if any, or the number of code units validated if - * successful. - */ -simdutf_warn_unused result validate_utf8_with_errors(const char *buf, - size_t len) noexcept; - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result validate_utf8_with_errors( - const detail::input_span_of_byte_like auto &input) noexcept { - return validate_utf8_with_errors(reinterpret_cast(input.data()), - input.size()); +namespace utf16 { +template constexpr uint16_t swap_if_needed(uint16_t c) { + return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; } - #endif // SIMDUTF_SPAN -#endif // SIMDUTF_FEATURE_UTF8 +} // namespace utf16 -#if SIMDUTF_FEATURE_ASCII -/** - * Validate the ASCII string. - * - * Overridden by each implementation. - * - * @param buf the ASCII string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid ASCII. - */ +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/swap_bytes.h */ +/* begin file include/simdutf/scalar/ascii.h */ +#ifndef SIMDUTF_ASCII_H +#define SIMDUTF_ASCII_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace ascii { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + // process in blocks of 16 bytes when possible + { + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + return false; + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return false; + } + } + return true; +} +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; +#if SIMDUTF_CPLUSPLUS23 + // avoid memcpy during constant evaluation + if !consteval +#endif + { + // process in blocks of 16 bytes when possible + for (; pos + 16 <= len; pos += 16) { + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) != 0) { + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + } + } + } + + // process the tail byte-by-byte + for (; pos < len; pos++) { + if (static_cast(data[pos]) >= 0b10000000) { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +} // namespace ascii +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/ascii.h */ +/* begin file include/simdutf/scalar/atomic_util.h */ +#ifndef SIMDUTF_ATOMIC_UTIL_H +#define SIMDUTF_ATOMIC_UTIL_H +#if SIMDUTF_ATOMIC_REF + #include + #include +namespace simdutf { +namespace scalar { + +// This function is a memcpy that uses atomic operations to read from the +// source. +inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char_type"); + // We expect all 64-bit systems to be able to read 64-bit words from an + // aligned memory region atomically. You might be able to do better on + // specific systems, e.g., x64 systems can read 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte copy + auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + char *mutable_src = const_cast(bytesrc); + for (size_t j = 0; j < bytelen; ++j) { + bytedst[j] = + std::atomic_ref(mutable_src[j]).load(std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(src) % alignment; + if (offset) { + size_t to_align = detail::min(len, alignment - offset); + bbb_memcpy_atomic_read(dst, src, to_align); + src += to_align; + dst += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *src_aligned = reinterpret_cast(const_cast(src)); + const auto dst_value = + std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); + std::memcpy(dst, &dst_value, sizeof(uint64_t)); + src += alignment; + dst += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_read(dst, src, len); + } +} + +// This function is a memcpy that uses atomic operations to write to the +// destination. +inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { + static_assert(std::atomic_ref::required_alignment == sizeof(char), + "std::atomic_ref requires the same alignment as char"); + // We expect all 64-bit systems to be able to write 64-bit words to an aligned + // memory region atomically. + // You might be able to do better on specific systems, e.g., x64 systems can + // write 128-bit words atomically. + constexpr size_t alignment = sizeof(uint64_t); + + // Lambda for atomic byte-by-byte write + auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, + size_t bytelen) noexcept { + for (size_t j = 0; j < bytelen; ++j) { + std::atomic_ref(bytedst[j]) + .store(bytesrc[j], std::memory_order_relaxed); + } + }; + + // Handle unaligned start + size_t offset = reinterpret_cast(dst) % alignment; + if (offset) { + size_t to_align = detail::min(len, alignment - offset); + bbb_memcpy_atomic_write(dst, src, to_align); + dst += to_align; + src += to_align; + len -= to_align; + } + + // Process aligned 64-bit chunks + while (len >= alignment) { + auto *dst_aligned = reinterpret_cast(dst); + uint64_t src_val; + std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + std::atomic_ref(*dst_aligned) + .store(src_val, std::memory_order_relaxed); + dst += alignment; + src += alignment; + len -= alignment; + } + + // Handle remaining bytes + if (len) { + bbb_memcpy_atomic_write(dst, src, len); + } +} +} // namespace scalar +} // namespace simdutf +#endif // SIMDUTF_ATOMIC_REF +#endif // SIMDUTF_ATOMIC_UTIL_H +/* end file include/simdutf/scalar/atomic_util.h */ +/* begin file include/simdutf/scalar/latin1.h */ +#ifndef SIMDUTF_LATIN1_H +#define SIMDUTF_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1 { + +simdutf_really_inline size_t utf8_length_from_latin1(const char *buf, + size_t len) { + const uint8_t *c = reinterpret_cast(buf); + size_t answer = 0; + for (size_t i = 0; i < len; i++) { + if ((c[i] >> 7)) { + answer++; + } + } + return answer + len; +} + +} // namespace latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1.h */ +/* begin file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF16_H +#define SIMDUTF_LATIN1_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return utf16_output - start; +} + +template +inline result convert_with_errors(const char *buf, size_t len, + char16_t *utf16_output) { + const uint8_t *data = reinterpret_cast(buf); + size_t pos = 0; + char16_t *start{utf16_output}; + + while (pos < len) { + uint16_t word = + uint16_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point + *utf16_output++ = + char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); + pos++; + } + + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace latin1_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf16/latin1_to_utf16.h */ +/* begin file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF32_H +#define SIMDUTF_LATIN1_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + char32_t *start{utf32_output}; + for (size_t i = 0; i < len; i++) { + *utf32_output++ = uint8_t(data[i]); + } + return utf32_output - start; +} + +} // namespace latin1_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf32/latin1_to_utf32.h */ +/* begin file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +#ifndef SIMDUTF_LATIN1_TO_UTF8_H +#define SIMDUTF_LATIN1_TO_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace latin1_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + // const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t utf8_pos = 0; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + utf8_output[utf8_pos++] = char(data[pos]); + pos++; + } + continue; + } + } // if (pos + 16 <= len) + } // !consteval scope + + unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } + } // while + return utf8_pos; +} + +simdutf_really_inline size_t convert(const char *buf, size_t len, + char *utf8_output) { + return convert(reinterpret_cast(buf), len, + utf8_output); +} + +inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, + size_t utf8_len) { + const unsigned char *data = reinterpret_cast(buf); + size_t pos = 0; + size_t skip_pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + // try to convert the next block of 16 ASCII bytes + if (pos >= skip_pos && pos + 16 <= len && + utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, + // check that they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + utf8_pos += 16; + pos += 16; + } else { + // At least one of the next 16 bytes are not ASCII, we will process them + // one by one + skip_pos = pos + 16; + } + } else { + const auto byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_safe_constexpr(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len) { + size_t pos = 0; + size_t utf8_pos = 0; + while (pos < len && utf8_pos < utf8_len) { + const unsigned char byte = data[pos]; + if ((byte & 0x80) == 0) { // if ASCII + // will generate one UTF-8 bytes + utf8_output[utf8_pos++] = char(byte); + pos++; + } else if (utf8_pos + 2 <= utf8_len) { + // will generate two UTF-8 bytes + utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); + utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); + pos++; + } else { + break; + } + } + return utf8_pos; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 simdutf_warn_unused size_t +utf8_length_from_latin1(InputPtr input, size_t length) noexcept { + size_t answer = length; + size_t i = 0; + +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + auto pop = [](uint64_t v) { + return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * + UINT64_C(0x0101010101010101) >> + 56); + }; + for (; i + 32 <= length; i += 32) { + uint64_t v; + memcpy(&v, input + i, 8); + answer += pop(v); + memcpy(&v, input + i + 8, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 16, sizeof(v)); + answer += pop(v); + memcpy(&v, input + i + 24, sizeof(v)); + answer += pop(v); + } + for (; i + 8 <= length; i += 8) { + uint64_t v; + memcpy(&v, input + i, sizeof(v)); + answer += pop(v); + } + } // !consteval scope + for (; i + 1 <= length; i += 1) { + answer += static_cast(input[i]) >> 7; + } + return answer; +} + +} // namespace latin1_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/latin1_to_utf8/latin1_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16.h */ +#ifndef SIMDUTF_UTF16_H +#define SIMDUTF_UTF16_H + +namespace simdutf { +namespace scalar { +namespace utf16 { + +template +simdutf_warn_unused simdutf_constexpr23 bool +validate_as_ascii(const char16_t *data, size_t len) noexcept { + for (size_t pos = 0; pos < len; pos++) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if (word >= 0x80) { + return false; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 bool +validate(const char16_t *data, size_t len) noexcept { + uint64_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return false; + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return false; + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = char16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return false; + } + pos += 2; + } else { + pos++; + } + } + return true; +} + +template +inline simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(const char16_t *data, size_t len) noexcept { + size_t pos = 0; + while (pos < len) { + char16_t word = scalar::utf16::swap_if_needed(data[pos]); + if ((word & 0xF800) == 0xD800) { + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } + char16_t diff = char16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + char16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + char16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + pos += 2; + } else { + pos++; + } + } + return result(error_code::SUCCESS, pos); +} + +template +simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +template +simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter++; // ASCII + counter += static_cast( + word > + 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes + counter += static_cast((word > 0x7FF && word <= 0xD7FF) || + (word >= 0xE000)); // three-byte + } + return counter; +} + +template +simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + char16_t word = scalar::utf16::swap_if_needed(p[i]); + counter += ((word & 0xFC00) != 0xDC00); + } + return counter; +} + +simdutf_really_inline simdutf_constexpr23 void +change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) { + for (size_t i = 0; i < size; i++) { + *output++ = char16_t(input[i] >> 8 | input[i] << 8); + } +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(const char16_t *input, size_t length) { + if (length == 0) { + return 0; + } + uint16_t last_word = uint16_t(input[length - 1]); + last_word = scalar::utf16::swap_if_needed(last_word); + length -= ((last_word & 0xFC00) == 0xD800); + return length; +} + +template constexpr bool is_high_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xd800 <= c && c <= 0xdbff); +} + +template constexpr bool is_low_surrogate(char16_t c) { + c = scalar::utf16::swap_if_needed(c); + return (0xdc00 <= c && c <= 0xdfff); +} + +simdutf_unused simdutf_really_inline constexpr bool high_surrogate(char16_t c) { + return (0xd800 <= c && c <= 0xdbff); +} + +template +simdutf_constexpr23 result +utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) { + bool any_surrogates = false; + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (is_high_surrogate(p[i])) { + any_surrogates = true; + // surrogate pair + if (i + 1 < len && is_low_surrogate(p[i + 1])) { + counter += 4; + i++; // skip low surrogate + } else { + counter += 3; // unpaired high surrogate replaced by U+FFFD + } + continue; + } else if (is_low_surrogate(p[i])) { + any_surrogates = true; + counter += 3; // unpaired low surrogate replaced by U+FFFD + continue; + } + char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i]; + counter++; // at least 1 byte + counter += + static_cast(word > 0x7F); // non-ASCII is at least 2 bytes + counter += static_cast(word > 0x7FF); // three-byte + } + return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS, + counter}; +} + +// variable templates are a C++14 extension +template constexpr char16_t replacement() { + return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd; +} + +template +simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len, + char16_t *output) { + const char16_t replacement = utf16::replacement(); + bool high_surrogate_prev = false, high_surrogate, low_surrogate; + size_t i = 0; + for (; i < len; i++) { + char16_t c = input[i]; + high_surrogate = is_high_surrogate(c); + low_surrogate = is_low_surrogate(c); + if (high_surrogate_prev && !low_surrogate) { + output[i - 1] = replacement; + } + + if (!high_surrogate_prev && low_surrogate) { + output[i] = replacement; + } else { + output[i] = input[i]; + } + high_surrogate_prev = high_surrogate; + } + + /* string may not end with high surrogate */ + if (high_surrogate_prev) { + output[i - 1] = replacement; + } +} + +} // namespace utf16 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +#ifndef SIMDUTF_UTF16_TO_LATIN1_H +#define SIMDUTF_UTF16_TO_LATIN1_H + +#include // for std::memcpy + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return 0; + } + size_t pos = 0; + const auto latin_output_start = latin_output; + uint16_t word = 0; + uint16_t too_large = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + too_large |= word; + *latin_output++ = char(word & 0xFF); + pos++; + } + if ((too_large & 0xFF00) != 0) { + return 0; + } + + return latin_output - latin_output_start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr latin_output) { + if (len == 0) { + return result(error_code::SUCCESS, 0); + } + size_t pos = 0; + auto start = latin_output; + uint16_t word; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that + // they are Latin1 + uint64_t v1, v2, v3, v4; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + + if constexpr (!match_system(big_endian)) { + v1 = (v1 >> 8) | (v1 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v2 = (v2 >> 8) | (v2 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v3 = (v3 >> 8) | (v3 << (64 - 8)); + } + if constexpr (!match_system(big_endian)) { + v4 = (v4 >> 8) | (v4 << (64 - 8)); + } + + if (((v1 | v2 | v3 | v4) & 0xFF00FF00FF00FF00) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF00) == 0) { + *latin_output++ = char(word & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_LATIN1_H +#define SIMDUTF_VALID_UTF16_TO_LATIN1_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_latin1 { + +template +simdutf_constexpr23 inline size_t +convert_valid_impl(InputIterator data, size_t len, + OutputIterator latin_output) { + static_assert( + std::is_same::type, uint16_t>::value, + "must decay to uint16_t"); + size_t pos = 0; + const auto start = latin_output; + uint16_t word = 0; + + while (pos < len) { + word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + *latin_output++ = char(word); + pos++; + } + + return latin_output - start; +} + +template +simdutf_really_inline size_t convert_valid(const char16_t *buf, size_t len, + char *latin_output) { + return convert_valid_impl(reinterpret_cast(buf), + len, latin_output); +} +} // namespace utf16_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_latin1/valid_utf16_to_latin1.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +#ifndef SIMDUTF_UTF16_TO_UTF32_H +#define SIMDUTF_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + if (pos + 1 >= len) { + return result(error_code::SURROGATE, pos); + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return result(error_code::SURROGATE, pos); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF32_H +#define SIMDUTF_VALID_UTF16_TO_UTF32_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf32 { + +template +simdutf_constexpr23 size_t convert_valid(const char16_t *data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xF800) != 0xD800) { + // No surrogate pair, extend 16-bit word to 32-bit word + *utf32_output++ = char32_t(word); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + *utf32_output++ = char32_t(value); + pos += 2; + } + } + return utf32_output - start; +} + +} // namespace utf16_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf32/valid_utf16_to_utf32.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +#ifndef SIMDUTF_UTF16_TO_UTF8_H +#define SIMDUTF_UTF16_TO_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_utf16 +// FIXME constrain output as well +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + const auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + if (pos + 1 >= len) { + return 0; + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return 0; + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return 0; + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output, + size_t utf8_len = 0) { + if (check_output && utf8_len == 0) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0); + } + + size_t pos = 0; + auto start = utf8_output; + auto end = utf8_output + utf8_len; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) + v = (v >> 8) | (v << (64 - 8)); + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + if (check_output && size_t(end - utf8_output) < 1) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 2) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (check_output && size_t(end - utf8_output) < 3) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + + if (check_output && size_t(end - utf8_output) < 4) { + return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos, + utf8_output - start); + } + // must be a surrogate pair + if (pos + 1 >= len) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t diff = uint16_t(word - 0xD800); + if (diff > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 > 0x3FF) { + return full_result(error_code::SURROGATE, pos, utf8_output - start); + } + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return full_result(error_code::SUCCESS, pos, utf8_output - start); +} + +template +inline result simple_convert_with_errors(const char16_t *buf, size_t len, + char *utf8_output) { + return convert_with_errors(buf, len, utf8_output, 0); +} + +template +simdutf_constexpr23 size_t convert_with_replacement(const char16_t *data, + size_t len, + char *utf8_output) { + size_t pos = 0; + char *start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 bytes + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // surrogate range + uint16_t diff = uint16_t(word - 0xD800); + if (diff <= 0x3FF && pos + 1 < len) { + // high surrogate, check for valid pair + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + if (diff2 <= 0x3FF) { + // valid surrogate pair + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + continue; + } + } + // unpaired surrogate: replace with U+FFFD (0xEF 0xBF 0xBD) + *utf8_output++ = char(0xef); + *utf8_output++ = char(0xbf); + *utf8_output++ = char(0xbd); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF16_TO_UTF8_H +#define SIMDUTF_VALID_UTF16_TO_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf16_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf16 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 4 ASCII characters + if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if constexpr (!match_system(big_endian)) { + v = (v >> 8) | (v << (64 - 8)); + } + if ((v & 0xFF80FF80FF80FF80) == 0) { + size_t final_pos = pos + 4; + while (pos < final_pos) { + *utf8_output++ = !match_system(big_endian) + ? char(u16_swap_bytes(data[pos])) + : char(data[pos]); + pos++; + } + continue; + } + } + } + + uint16_t word = + !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; + if ((word & 0xFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xF800) != 0xD800) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // must be a surrogate pair + uint16_t diff = uint16_t(word - 0xD800); + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + uint16_t next_word = !match_system(big_endian) + ? u16_swap_bytes(data[pos + 1]) + : data[pos + 1]; + uint16_t diff2 = uint16_t(next_word - 0xDC00); + uint32_t value = (diff << 10) + diff2 + 0x10000; + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((value >> 18) | 0b11110000); + *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((value & 0b111111) | 0b10000000); + pos += 2; + } + } + return utf8_output - start; +} + +} // namespace utf16_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf16_to_utf8/valid_utf16_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32.h */ +#ifndef SIMDUTF_UTF32_H +#define SIMDUTF_UTF32_H + +namespace simdutf { +namespace scalar { +namespace utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, + size_t len) noexcept { + uint64_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { + return false; + } + } + return true; +} + +simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_uint32 +#endif +simdutf_warn_unused simdutf_constexpr23 result +validate_with_errors(InputPtr data, size_t len) noexcept { + size_t pos = 0; + for (; pos < len; pos++) { + uint32_t word = data[pos]; + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + } + return result(error_code::SUCCESS, pos); +} + +simdutf_warn_unused simdutf_really_inline result +validate_with_errors(const char32_t *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, + size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // credit: @ttsugriy for the vectorizable approach + counter++; // ASCII + counter += static_cast(p[i] > 0x7F); // two-byte + counter += static_cast(p[i] > 0x7FF); // three-byte + counter += static_cast(p[i] > 0xFFFF); // four-bytes + } + return counter; +} + +inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf32(const char32_t *p, size_t len) { + // We are not BOM aware. + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + counter++; // non-surrogate word + counter += static_cast(p[i] > 0xFFFF); // surrogate pair + } + return counter; +} + +} // namespace utf32 +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +#ifndef SIMDUTF_UTF32_TO_LATIN1_H +#define SIMDUTF_UTF32_TO_LATIN1_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +inline simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char *latin1_output) { + char *start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + uint32_t too_large = 0; + + while (pos < len) { + utf32_char = (uint32_t)data[pos]; + too_large |= utf32_char; + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } + if ((too_large & 0xFFFFFF00) != 0) { + return 0; + } + return latin1_output - start; +} + +inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, + size_t len, + char *latin1_output) { + char *start{latin1_output}; + size_t pos = 0; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are Latin1 + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t utf32_char = data[pos]; + if ((utf32_char & 0xFFFFFF00) == + 0) { // Check if the character can be represented in Latin-1 + *latin1_output++ = (char)(utf32_char & 0xFF); + pos++; + } else { + return result(error_code::TOO_LARGE, pos); + }; + } + return result(error_code::SUCCESS, latin1_output - start); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_LATIN1_H +#define SIMDUTF_VALID_UTF32_TO_LATIN1_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_latin1 { + +template +simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, + WritePtr latin1_output) { + static_assert( + std::is_same::type, uint32_t>::value, + "dereferencing the data pointer must result in a uint32_t"); + auto start = latin1_output; + uint32_t utf32_char; + size_t pos = 0; + + while (pos < len) { + utf32_char = data[pos]; + +#if SIMDUTF_CPLUSPLUS23 + // avoid using the 8 byte at a time optimization in constant evaluation + // mode. memcpy can't be used and replacing it with bitwise or gave worse + // codegen (when not during constant evaluation). + if !consteval { +#endif + if (pos + 2 <= len) { + // if it is safe to read 8 more bytes, check that they are Latin1 + uint64_t v; + std::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF00FFFFFF00) == 0) { + *latin1_output++ = char(data[pos]); + *latin1_output++ = char(data[pos + 1]); + pos += 2; + continue; + } else { + // output can not be represented in latin1 + return 0; + } + } +#if SIMDUTF_CPLUSPLUS23 + } // if ! consteval +#endif + if ((utf32_char & 0xFFFFFF00) == 0) { + *latin1_output++ = char(utf32_char); + } else { + // output can not be represented in latin1 + return 0; + } + pos++; + } + return latin1_output - start; +} + +simdutf_really_inline size_t convert_valid(const char32_t *buf, size_t len, + char *latin1_output) { + return convert_valid(reinterpret_cast(buf), len, + latin1_output); +} + +} // namespace utf32_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_latin1/valid_utf32_to_latin1.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +#ifndef SIMDUTF_UTF32_TO_UTF16_H +#define SIMDUTF_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return 0; + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return utf16_output - start; +} + +template +simdutf_constexpr23 result convert_with_errors(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + } else { + // will generate a surrogate pair + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + } + pos++; + } + return result(error_code::SUCCESS, utf16_output - start); +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF16_H +#define SIMDUTF_VALID_UTF32_TO_UTF16_H + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf16 { + +template +simdutf_constexpr23 size_t convert_valid(const char32_t *data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { + uint32_t word = data[pos]; + if ((word & 0xFFFF0000) == 0) { + // will not generate a surrogate pair + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(uint16_t(word))) + : char16_t(word); + pos++; + } else { + // will generate a surrogate pair + word -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos++; + } + } + return utf16_output - start; +} + +} // namespace utf32_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf16/valid_utf32_to_utf16.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +#ifndef SIMDUTF_UTF32_TO_UTF8_H +#define SIMDUTF_UTF32_TO_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return 0; + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return 0; + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + if (word >= 0xD800 && word <= 0xDFFF) { + return result(error_code::SURROGATE, pos); + } + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + if (word > 0x10FFFF) { + return result(error_code::TOO_LARGE, pos); + } + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return result(error_code::SUCCESS, utf8_output - start); +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +#ifndef SIMDUTF_VALID_UTF32_TO_UTF8_H +#define SIMDUTF_VALID_UTF32_TO_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf32_to_utf8 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_utf32 && + simdutf::detail::index_assignable_from_char) +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + OutputPtr utf8_output) { + size_t pos = 0; + auto start = utf8_output; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 2 ASCII characters + if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0xFFFFFF80FFFFFF80) == 0) { + *utf8_output++ = char(data[pos]); + *utf8_output++ = char(data[pos + 1]); + pos += 2; + continue; + } + } + } + + uint32_t word = data[pos]; + if ((word & 0xFFFFFF80) == 0) { + // will generate one UTF-8 bytes + *utf8_output++ = char(word); + pos++; + } else if ((word & 0xFFFFF800) == 0) { + // will generate two UTF-8 bytes + // we have 0b110XXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 6) | 0b11000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else if ((word & 0xFFFF0000) == 0) { + // will generate three UTF-8 bytes + // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 12) | 0b11100000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } else { + // will generate four UTF-8 bytes + // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX + *utf8_output++ = char((word >> 18) | 0b11110000); + *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); + *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); + *utf8_output++ = char((word & 0b111111) | 0b10000000); + pos++; + } + } + return utf8_output - start; +} + +} // namespace utf32_to_utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf32_to_utf8/valid_utf32_to_utf8.h */ +/* begin file include/simdutf/scalar/utf8.h */ +#ifndef SIMDUTF_UTF8_H +#define SIMDUTF_UTF8_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8 { + +// credit: based on code from Google Fuchsia (Apache Licensed) +template +simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, + size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + uint64_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + uint64_t next_pos; +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // check if the next 16 bytes are ascii. + next_pos = pos + 16; + if (next_pos <= len) { // if it is safe to read 16 more bytes, check + // that they are ascii + uint64_t v1{}; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2{}; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + } + + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return true; + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80) { + return false; + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if ((code_point < 0x800) || + (0xd7ff < code_point && code_point < 0xe000)) { + return false; + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return false; + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return false; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return false; + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return false; + } + } else { + // we may have a continuation + return false; + } + pos = next_pos; + } + return true; +} + +simdutf_really_inline simdutf_warn_unused bool validate(const char *buf, + size_t len) noexcept { + return validate(reinterpret_cast(buf), len); +} + +template +simdutf_constexpr23 simdutf_warn_unused result +validate_with_errors(BytePtr data, size_t len) noexcept { + static_assert( + std::is_same::type, uint8_t>::value, + "dereferencing the data pointer must result in a uint8_t"); + size_t pos = 0; + uint32_t code_point = 0; + while (pos < len) { + // check of the next 16 bytes are ascii. + size_t next_pos = pos + 16; + if (next_pos <= + len) { // if it is safe to read 16 more bytes, check that they are ascii + uint64_t v1; + std::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + pos = next_pos; + continue; + } + } + unsigned char byte = data[pos]; + + while (byte < 0b10000000) { + if (++pos == len) { + return result(error_code::SUCCESS, len); + } + byte = data[pos]; + } + + if ((byte & 0b11100000) == 0b11000000) { + next_pos = pos + 2; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + } else if ((byte & 0b11110000) == 0b11100000) { + next_pos = pos + 3; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = (byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + next_pos = pos + 4; + if (next_pos > len) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + code_point = + (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + pos = next_pos; + } + return result(error_code::SUCCESS, len); +} + +simdutf_really_inline simdutf_warn_unused result +validate_with_errors(const char *buf, size_t len) noexcept { + return validate_with_errors(reinterpret_cast(buf), len); +} + +// Finds the previous leading byte starting backward from buf and validates with +// errors from there Used to pinpoint the location of an error when an invalid +// chunk is detected We assume that the stream starts with a leading byte, and +// to check that it is the case, we ask that you pass a pointer to the start of +// the stream (start). Note that the resulting count is underflowed if an error +// is encountered in the rewinded segment. +inline simdutf_warn_unused result rewind_and_validate_with_errors( + const char *start, const char *buf, size_t len) noexcept { + // First check that we start with a leading byte + if ((*start & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, 0); + } + size_t extra_len{0}; + // A leading byte cannot be further than 4 bytes away + for (int i = 0; i < 5; i++) { + unsigned char byte = *buf; + if ((byte & 0b11000000) != 0b10000000) { + break; + } else { + buf--; + extra_len++; + } + } + + result res = validate_with_errors(buf, len + extra_len); + res.count -= extra_len; // Might underflow + return res; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + // -65 is 0b10111111, anything larger in two-complement's should start a new + // code point. + if (int8_t(data[i]) > -65) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { + size_t counter{0}; + for (size_t i = 0; i < len; i++) { + if (int8_t(data[i]) > -65) { + counter++; + } + if (uint8_t(data[i]) >= 240) { + counter++; + } + } + return counter; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8(InputPtr input, size_t length) { + if (length < 3) { + switch (length) { + case 2: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 2 bytes left + return length; + case 1: + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + return length; + case 0: + return length; + } + } + if (uint8_t(input[length - 1]) >= 0xc0) { + return length - 1; + } // 2-, 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 2]) >= 0xe0) { + return length - 2; + } // 3- and 4-byte characters with only 1 byte left + if (uint8_t(input[length - 3]) >= 0xf0) { + return length - 3; + } // 4-byte characters with only 3 bytes left + return length; +} + +} // namespace utf8 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +#ifndef SIMDUTF_UTF8_TO_LATIN1_H +#define SIMDUTF_UTF8_TO_LATIN1_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires(simdutf::detail::indexes_into_byte_like && + simdutf::detail::indexes_into_byte_like) +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + OutputPtr latin_output) { + size_t pos = 0; + auto start = latin_output; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000 .... etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80 || 0xFF < code_point) { + return 0; // We only care about the range 129-255 which is Non-ASCII + // latin1 characters. A code_point beneath 0x80 is invalid as + // it is already covered by bytes whose leading bit is zero. + } + *latin_output++ = char(code_point); + pos += 2; + } else { + return 0; + } + } + return latin_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char *latin_output) { + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 + // 1000 1000...etc + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = char(data[pos]); + pos++; + } + continue; + } + } + } + // suppose it is not an all ASCII byte sequence + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (data[pos + 1] & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if (0xFF < code_point) { + return result(error_code::TOO_LARGE, pos); + } // We only care about the range 129-255 which is Non-ASCII latin1 + // characters + *latin_output++ = char(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + return result(error_code::TOO_LARGE, pos); + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + return result(error_code::TOO_LARGE, pos); + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } + + return result(error_code::HEADER_BITS, pos); + } + } + return result(error_code::SUCCESS, latin_output - start); +} + +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char *latin1_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, latin1_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_LATIN1_H +#define SIMDUTF_VALID_UTF8_TO_LATIN1_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_latin1 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char *latin_output) { + + size_t pos = 0; + char *start{latin_output}; + + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | + v2}; // We are only interested in these bits: 1000 1000 1000 + // 1000, so it makes sense to concatenate everything + if ((v & 0x8080808080808080) == + 0) { // if NONE of these are set, e.g. all of them are zero, then + // everything is ASCII + size_t final_pos = pos + 16; + while (pos < final_pos) { + *latin_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + + // suppose it is not an all ASCII byte sequence + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *latin_output++ = char(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == + 0b11000000) { // the first three bits indicate: + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } // checks if the next byte is a valid continuation byte in UTF-8. A + // valid continuation byte starts with 10. + // range check - + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & + 0b00111111); // assembles the Unicode code point from the two bytes. + // It does this by discarding the leading 110 and 10 + // bits from the two bytes, shifting the remaining bits + // of the first byte, and then combining the results + // with a bitwise OR operation. + *latin_output++ = char(code_point); + pos += 2; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return latin_output - start; +} + +} // namespace utf8_to_latin1 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_latin1/valid_utf8_to_latin1.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +#ifndef SIMDUTF_UTF8_TO_UTF16_H +#define SIMDUTF_UTF8_TO_UTF16_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + // try to convert the next block of 16 ASCII bytes + { + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(data[pos])) + : char16_t(data[pos]); + pos++; + } + continue; + } + } + } + + uint8_t leading_byte = data[pos]; // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = + (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + if (code_point < 0x80) { + return 0; + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (data[pos + 1] & 0b00111111) << 6 | + (data[pos + 2] & 0b00111111); + if (code_point < 0x800 || (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 2] & 0b11000000) != 0b10000000) { + return 0; + } + if ((data[pos + 3] & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (data[pos + 1] & 0b00111111) << 12 | + (data[pos + 2] & 0b00111111) << 6 | + (data[pos + 3] & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + return 0; + } + } + return utf16_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + if constexpr (!match_system(big_endian)) { + code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf16_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +template +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char16_t *utf16_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + // In theory '3' would be sufficient, but sometimes the error can go back + // quite far. + size_t how_far_back = prior_bytes; + // size_t how_far_back = 3; // 3 bytes in the past + current position + // if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + result res = convert_with_errors(buf, len + extra_len, utf16_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF16_H +#define SIMDUTF_VALID_UTF8_TO_UTF16_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf16 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char16_t *utf16_output) { + size_t pos = 0; + char16_t *start{utf16_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + const char16_t byte = uint8_t(data[pos]); + *utf16_output++ = + !match_system(big_endian) ? u16_swap_bytes(byte) : byte; + pos++; + } + continue; + } + } + } + + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf16_output++ = !match_system(big_endian) + ? char16_t(u16_swap_bytes(leading_byte)) + : char16_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 1 >= len) { + break; + } // minimal bound checking + uint16_t code_point = uint16_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + if constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8, it should become + // a single UTF-16 word. + if (pos + 2 >= len) { + break; + } // minimal bound checking + uint16_t code_point = + uint16_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + if constexpr (!match_system(big_endian)) { + code_point = u16_swap_bytes(uint16_t(code_point)); + } + *utf16_output++ = char16_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_point = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + code_point -= 0x10000; + uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); + uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); + if constexpr (!match_system(big_endian)) { + high_surrogate = u16_swap_bytes(high_surrogate); + low_surrogate = u16_swap_bytes(low_surrogate); + } + *utf16_output++ = char16_t(high_surrogate); + *utf16_output++ = char16_t(low_surrogate); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf16_output - start; +} + +} // namespace utf8_to_utf16 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf16/valid_utf8_to_utf16.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +#ifndef SIMDUTF_UTF8_TO_UTF32_H +#define SIMDUTF_UTF8_TO_UTF32_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return 0; + } // minimal bound checking + if ((data[pos + 1] & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return 0; + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800 || (0xd7ff < code_point && code_point < 0xe000)) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return 0; + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return 0; + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return 0; + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff || 0x10ffff < code_point) { + return 0; + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + return 0; + } + } + return utf32_output - start; +} + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 16 ASCII bytes + if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that + // they are ascii + uint64_t v1; + ::memcpy(&v1, data + pos, sizeof(uint64_t)); + uint64_t v2; + ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + uint64_t v{v1 | v2}; + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 16; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00011111) << 6 | + (uint8_t(data[pos + 1]) & 0b00111111); + if (code_point < 0x80) { + return result(error_code::OVERLONG, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + // range check + uint32_t code_point = (leading_byte & 0b00001111) << 12 | + (uint8_t(data[pos + 1]) & 0b00111111) << 6 | + (uint8_t(data[pos + 2]) & 0b00111111); + if (code_point < 0x800) { + return result(error_code::OVERLONG, pos); + } + if (0xd7ff < code_point && code_point < 0xe000) { + return result(error_code::SURROGATE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + return result(error_code::TOO_SHORT, pos); + } // minimal bound checking + if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { + return result(error_code::TOO_SHORT, pos); + } + + // range check + uint32_t code_point = (leading_byte & 0b00000111) << 18 | + (uint8_t(data[pos + 1]) & 0b00111111) << 12 | + (uint8_t(data[pos + 2]) & 0b00111111) << 6 | + (uint8_t(data[pos + 3]) & 0b00111111); + if (code_point <= 0xffff) { + return result(error_code::OVERLONG, pos); + } + if (0x10ffff < code_point) { + return result(error_code::TOO_LARGE, pos); + } + *utf32_output++ = char32_t(code_point); + pos += 4; + } else { + // we either have too many continuation bytes or an invalid leading byte + if ((leading_byte & 0b11000000) == 0b10000000) { + return result(error_code::TOO_LONG, pos); + } else { + return result(error_code::HEADER_BITS, pos); + } + } + } + return result(error_code::SUCCESS, utf32_output - start); +} + +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and + * we have up to len input bytes left, and we encountered some error. It is + * possible that the error is at 'buf' exactly, but it could also be in the + * previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the + * current memory section and can be safely accessed. We prior_bytes to access + * safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occurred prior to 'buf', the count value + * contain in the result will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, + const char *buf, size_t len, + char32_t *utf32_output) { + size_t extra_len{0}; + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if (how_far_back > prior_bytes) { + how_far_back = prior_bytes; + } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for (size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-static_cast(i)]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if (found_leading_bytes) { + if (i > 0 && byte < 128) { + // If we had to go back and the leading byte is ascii + // then we can stop right away. + return result(error_code::TOO_LONG, 0 - i + 1); + } + buf -= i; + extra_len = i; + break; + } + } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in which is described + // in C Standard as . C Standard Section 4.1.5 defines size_t as an + // unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if (!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is + // continuation] Or we possibly have a stream that does not start with a + // leading byte. + return result(error_code::TOO_LONG, 0 - how_far_back); + } + + result res = convert_with_errors(buf, len + extra_len, utf32_output); + if (res.error) { + res.count -= extra_len; + } + return res; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/utf8_to_utf32.h */ +/* begin file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ +#ifndef SIMDUTF_VALID_UTF8_TO_UTF32_H +#define SIMDUTF_VALID_UTF8_TO_UTF32_H + +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace utf8_to_utf32 { + +template +#if SIMDUTF_CPLUSPLUS20 + requires simdutf::detail::indexes_into_byte_like +#endif +simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, + char32_t *utf32_output) { + size_t pos = 0; + char32_t *start{utf32_output}; + while (pos < len) { +#if SIMDUTF_CPLUSPLUS23 + if !consteval +#endif + { + // try to convert the next block of 8 ASCII bytes + if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that + // they are ascii + uint64_t v; + ::memcpy(&v, data + pos, sizeof(uint64_t)); + if ((v & 0x8080808080808080) == 0) { + size_t final_pos = pos + 8; + while (pos < final_pos) { + *utf32_output++ = uint8_t(data[pos]); + pos++; + } + continue; + } + } + } + auto leading_byte = uint8_t(data[pos]); // leading byte + if (leading_byte < 0b10000000) { + // converting one ASCII byte !!! + *utf32_output++ = char32_t(leading_byte); + pos++; + } else if ((leading_byte & 0b11100000) == 0b11000000) { + // We have a two-byte UTF-8 + if (pos + 1 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00011111) << 6) | + (uint8_t(data[pos + 1]) & 0b00111111)); + pos += 2; + } else if ((leading_byte & 0b11110000) == 0b11100000) { + // We have a three-byte UTF-8 + if (pos + 2 >= len) { + break; + } // minimal bound checking + *utf32_output++ = char32_t(((leading_byte & 0b00001111) << 12) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 6) | + (uint8_t(data[pos + 2]) & 0b00111111)); + pos += 3; + } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 + // we have a 4-byte UTF-8 word. + if (pos + 3 >= len) { + break; + } // minimal bound checking + uint32_t code_word = ((leading_byte & 0b00000111) << 18) | + ((uint8_t(data[pos + 1]) & 0b00111111) << 12) | + ((uint8_t(data[pos + 2]) & 0b00111111) << 6) | + (uint8_t(data[pos + 3]) & 0b00111111); + *utf32_output++ = char32_t(code_word); + pos += 4; + } else { + // we may have a continuation but we do not do error checking + return 0; + } + } + return utf32_output - start; +} + +} // namespace utf8_to_utf32 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/utf8_to_utf32/valid_utf8_to_utf32.h */ + +namespace simdutf { + +constexpr size_t default_line_length = + 76; ///< default line length for base64 encoding with lines + +#if SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const char *input, size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding(const uint8_t *input, size_t length) noexcept { + return autodetect_encoding(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +/** + * Autodetect the encoding of the input, a single encoding is recommended. + * E.g., the function might return simdutf::encoding_type::UTF8, + * simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or + * simdutf::encoding_type::UTF32_LE. + * + * @param input the string to analyze. can be a anything span-like that has a + * data() and size() that points to character data: std::string, + * std::string_view, std::vector, std::span etc. + * @return the detected encoding type + */ +simdutf_really_inline simdutf_warn_unused simdutf::encoding_type +autodetect_encoding( + const detail::input_span_of_byte_like auto &input) noexcept { + return autodetect_encoding(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN + +/** + * Autodetect the possible encodings of the input in one pass. + * E.g., if the input might be UTF-16LE or UTF-8, this function returns + * the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE). + * + * Overridden by each implementation. + * + * @param input the string to analyze. + * @param length the length of the string in bytes. + * @return the detected encoding type + */ +simdutf_warn_unused int detect_encodings(const char *input, + size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const uint8_t *input, size_t length) noexcept { + return detect_encodings(reinterpret_cast(input), length); +} + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused int +detect_encodings(const detail::input_span_of_byte_like auto &input) noexcept { + return detect_encodings(reinterpret_cast(input.data()), + input.size()); +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING +/** + * Validate the UTF-8 string. This function may be best when you expect + * the input to be almost always valid. Otherwise, consider using + * validate_utf8_with_errors. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid UTF-8. + */ +simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_constexpr23 simdutf_really_inline simdutf_warn_unused bool +validate_utf8(const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8(reinterpret_cast(input.data()), + input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 || SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_UTF8 +/** + * Validate the UTF-8 string and stop on error. + * + * Overridden by each implementation. + * + * @param buf the UTF-8 string to validate. + * @param len the length of the string in bytes. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and either position of the error + * (in the input in code units) if any, or the number of code units validated if + * successful. + */ +simdutf_warn_unused result validate_utf8_with_errors(const char *buf, + size_t len) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result +validate_utf8_with_errors( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf8_with_errors( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 + +#if SIMDUTF_FEATURE_ASCII +/** + * Validate the ASCII string. + * + * Overridden by each implementation. + * + * @param buf the ASCII string to validate. + * @param len the length of the string in bytes. + * @return true if and only if the string is valid ASCII. + */ simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { - return validate_ascii(reinterpret_cast(input.data()), - input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii(reinterpret_cast(input.data()), + input.size()); + } } #endif // SIMDUTF_SPAN @@ -1348,10 +5280,19 @@ validate_ascii(const detail::input_span_of_byte_like auto &input) noexcept { simdutf_warn_unused result validate_ascii_with_errors(const char *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result validate_ascii_with_errors( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +validate_ascii_with_errors( const detail::input_span_of_byte_like auto &input) noexcept { - return validate_ascii_with_errors( - reinterpret_cast(input.data()), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::ascii::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_ascii_with_errors( + reinterpret_cast(input.data()), input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_ASCII @@ -1371,9 +5312,17 @@ simdutf_really_inline simdutf_warn_unused result validate_ascii_with_errors( simdutf_warn_unused bool validate_utf16_as_ascii(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf16_as_ascii(std::span input) noexcept { - return validate_utf16_as_ascii(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16_as_ascii(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -1391,12 +5340,19 @@ validate_utf16_as_ascii(std::span input) noexcept { simdutf_warn_unused bool validate_utf16be_as_ascii(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf16be_as_ascii(std::span input) noexcept { - return validate_utf16be_as_ascii(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_as_ascii(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII /** * Validate the ASCII string as a UTF-16LE sequence. @@ -1411,12 +5367,21 @@ validate_utf16be_as_ascii(std::span input) noexcept { */ simdutf_warn_unused bool validate_utf16le_as_ascii(const char16_t *buf, size_t len) noexcept; -#if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf16le_as_ascii(std::span input) noexcept { - return validate_utf16le_as_ascii(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_as_ascii(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le_as_ascii(input.data(), input.size()); + } } -#endif // SIMDUTF_SPAN + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_ASCII #if SIMDUTF_FEATURE_UTF16 /** @@ -1436,9 +5401,17 @@ validate_utf16le_as_ascii(std::span input) noexcept { simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf16(std::span input) noexcept { - return validate_utf16(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 @@ -1461,9 +5434,17 @@ validate_utf16(std::span input) noexcept { simdutf_warn_unused bool validate_utf16le(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused bool validate_utf16le(std::span input) noexcept { - return validate_utf16le(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), + input.size()); + } else + #endif + { + return validate_utf16le(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 || SIMDUTF_FEATURE_DETECT_ENCODING @@ -1486,9 +5467,16 @@ validate_utf16le(std::span input) noexcept { simdutf_warn_unused bool validate_utf16be(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf16be(std::span input) noexcept { - return validate_utf16be(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate(input.data(), input.size()); + } else + #endif + { + return validate_utf16be(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -1512,9 +5500,17 @@ validate_utf16be(std::span input) noexcept { simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result validate_utf16_with_errors(std::span input) noexcept { - return validate_utf16_with_errors(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16_with_errors(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -1537,9 +5533,17 @@ validate_utf16_with_errors(std::span input) noexcept { simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result validate_utf16le_with_errors(std::span input) noexcept { - return validate_utf16le_with_errors(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors( + input.data(), input.size()); + } else + #endif + { + return validate_utf16le_with_errors(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -1562,9 +5566,17 @@ validate_utf16le_with_errors(std::span input) noexcept { simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result validate_utf16be_with_errors(std::span input) noexcept { - return validate_utf16be_with_errors(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::validate_with_errors(input.data(), + input.size()); + } else + #endif + { + return validate_utf16be_with_errors(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -1583,10 +5595,18 @@ validate_utf16be_with_errors(std::span input) noexcept { void to_well_formed_utf16le(const char16_t *input, size_t len, char16_t *output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline void +simdutf_really_inline simdutf_constexpr23 void to_well_formed_utf16le(std::span input, std::span output) noexcept { - to_well_formed_utf16le(input.data(), input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16le(input.data(), input.size(), output.data()); + } } #endif // SIMDUTF_SPAN @@ -1605,10 +5625,18 @@ to_well_formed_utf16le(std::span input, void to_well_formed_utf16be(const char16_t *input, size_t len, char16_t *output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline void +simdutf_really_inline simdutf_constexpr23 void to_well_formed_utf16be(std::span input, std::span output) noexcept { - to_well_formed_utf16be(input.data(), input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16be(input.data(), input.size(), output.data()); + } } #endif // SIMDUTF_SPAN @@ -1627,10 +5655,18 @@ to_well_formed_utf16be(std::span input, void to_well_formed_utf16(const char16_t *input, size_t len, char16_t *output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline void +simdutf_really_inline simdutf_constexpr23 void to_well_formed_utf16(std::span input, std::span output) noexcept { - to_well_formed_utf16(input.data(), input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + scalar::utf16::to_well_formed_utf16( + input.data(), input.size(), output.data()); + } else + #endif + { + to_well_formed_utf16(input.data(), input.size(), output.data()); + } } #endif // SIMDUTF_SPAN @@ -1654,9 +5690,17 @@ to_well_formed_utf16(std::span input, simdutf_warn_unused bool validate_utf32(const char32_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused bool +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 bool validate_utf32(std::span input) noexcept { - return validate_utf32(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF32 || SIMDUTF_FEATURE_DETECT_ENCODING @@ -1681,9 +5725,17 @@ validate_utf32(std::span input) noexcept { simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, size_t len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result validate_utf32_with_errors(std::span input) noexcept { - return validate_utf32_with_errors(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::validate_with_errors( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return validate_utf32_with_errors(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF32 @@ -1703,12 +5755,23 @@ simdutf_warn_unused size_t convert_latin1_to_utf8(const char *input, size_t length, char *utf8_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8( const detail::input_span_of_byte_like auto &latin1_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_latin1_to_utf8( - reinterpret_cast(latin1_input.data()), latin1_input.size(), - utf8_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert( + detail::constexpr_cast_ptr(latin1_input.data()), + latin1_input.size(), + detail::constexpr_cast_writeptr(utf8_output.data())); + } else + #endif + { + return convert_latin1_to_utf8( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -1719,6 +5782,12 @@ simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf8( * * We write as many characters as possible. * + * Using convert_latin1_to_utf8_safe instead of convert_latin1_to_utf8 comes + * with a significant penalty in some cases, being up to four times slower, + * especially on short inputs. If you have allocated the output buffer so that + * it contains utf8_length_from_latin1(input, length) bytes, then prefer + * convert_latin1_to_utf8. + * * @param input the Latin1 string to convert * @param length the length of the string in bytes * @param utf8_output the pointer to buffer that can hold conversion result @@ -1729,18 +5798,27 @@ simdutf_warn_unused size_t convert_latin1_to_utf8_safe(const char *input, size_t length, char *utf8_output, size_t utf8_len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf8_safe( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf8_safe( const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - // implementation note: outputspan is a forwarding ref to avoid copying and - // allow both lvalues and rvalues. std::span can be copied without problems, - // but std::vector should not, and this function should accept both. it will - // allow using an owning rvalue ref (example: passing a temporary std::string) - // as output, but the user will quickly find out that he has no way of getting - // the data out of the object in that case. - return convert_latin1_to_utf8_safe( - input.data(), input.size(), reinterpret_cast(utf8_output.data()), - utf8_output.size()); + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::convert_safe_constexpr( + input.data(), input.size(), utf8_output.data(), utf8_output.size()); + } else + #endif + { + return convert_latin1_to_utf8_safe( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -1753,18 +5831,27 @@ simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf8_safe( * * @param input the Latin1 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if conversion is not possible */ simdutf_warn_unused size_t convert_latin1_to_utf16le( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf16le( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf16le( const detail::input_span_of_byte_like auto &latin1_input, std::span utf16_output) noexcept { - return convert_latin1_to_utf16le( - reinterpret_cast(latin1_input.data()), latin1_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + latin1_input.data(), latin1_input.size(), utf16_output.data()); + } else + #endif + { + return convert_latin1_to_utf16le( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -1775,17 +5862,26 @@ simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf16le( * * @param input the Latin1 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if conversion is not possible */ simdutf_warn_unused size_t convert_latin1_to_utf16be( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, std::span output) noexcept { - return convert_latin1_to_utf16be(reinterpret_cast(input.data()), - input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16be( + reinterpret_cast(input.data()), input.size(), + output.data()); + } } #endif // SIMDUTF_SPAN /** @@ -1796,7 +5892,10 @@ convert_latin1_to_utf16be(const detail::input_span_of_byte_like auto &input, * @return the length of the string in Latin1 code units (char) required to * encode the UTF-16 string as Latin1 */ -simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf16(size_t length) noexcept { + return length; +} /** * Compute the number of code units that this Latin1 string would require in @@ -1806,7 +5905,10 @@ simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept; * @return the length of the string in 2-byte code units (char16_t) required to * encode the Latin1 string as UTF-16 */ -simdutf_warn_unused size_t utf16_length_from_latin1(size_t length) noexcept; +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_latin1(size_t length) noexcept { + return length; +} #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 @@ -1823,12 +5925,21 @@ simdutf_warn_unused size_t utf16_length_from_latin1(size_t length) noexcept; simdutf_warn_unused size_t convert_latin1_to_utf32( const char *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_latin1_to_utf32( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_latin1_to_utf32( const detail::input_span_of_byte_like auto &latin1_input, std::span utf32_output) noexcept { - return convert_latin1_to_utf32( - reinterpret_cast(latin1_input.data()), latin1_input.size(), - utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf32::convert( + latin1_input.data(), latin1_input.size(), utf32_output.data()); + } else + #endif + { + return convert_latin1_to_utf32( + reinterpret_cast(latin1_input.data()), + latin1_input.size(), utf32_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 @@ -1850,12 +5961,21 @@ simdutf_warn_unused size_t convert_utf8_to_latin1(const char *input, size_t length, char *latin1_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf8_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf8_to_latin1( const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&output) noexcept { - return convert_utf8_to_latin1(reinterpret_cast(input.data()), - input.size(), - reinterpret_cast(output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert(input.data(), input.size(), + output.data()); + } else + #endif + { + return convert_utf8_to_latin1(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -1870,18 +5990,26 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf8_to_latin1( * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ simdutf_warn_unused size_t convert_utf8_to_utf16( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, std::span output) noexcept { - return convert_utf8_to_utf16(reinterpret_cast(input.data()), - input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_utf8_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } } #endif // SIMDUTF_SPAN @@ -1905,11 +6033,19 @@ convert_utf8_to_utf16(const detail::input_span_of_byte_like auto &input, simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused result utf8_length_from_utf16le_with_replacement( std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::LITTLE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -1933,11 +6069,19 @@ utf8_length_from_utf16le_with_replacement( simdutf_warn_unused result utf8_length_from_utf16be_with_replacement( const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result utf8_length_from_utf16be_with_replacement( std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::BIG>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -1949,17 +6093,25 @@ utf8_length_from_utf16be_with_replacement( * * @param input the Latin1 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t. */ simdutf_warn_unused size_t convert_latin1_to_utf16( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, std::span output) noexcept { - return convert_latin1_to_utf16(reinterpret_cast(input.data()), - input.size(), output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf16::convert( + input.data(), input.size(), output.data()); + } else + #endif + { + return convert_latin1_to_utf16(reinterpret_cast(input.data()), + input.size(), output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 @@ -1973,19 +6125,27 @@ convert_latin1_to_utf16(const detail::input_span_of_byte_like auto &input, * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ simdutf_warn_unused size_t convert_utf8_to_utf16le( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, std::span utf16_output) noexcept { - return convert_utf8_to_utf16le( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -1997,19 +6157,28 @@ convert_utf8_to_utf16le(const detail::input_span_of_byte_like auto &utf8_input, * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ simdutf_warn_unused size_t convert_utf8_to_utf16be( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, std::span utf16_output) noexcept { - return convert_utf8_to_utf16be( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf16_output.data()); + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2034,13 +6203,21 @@ convert_utf8_to_utf16be(const detail::input_span_of_byte_like auto &utf8_input, simdutf_warn_unused result convert_utf8_to_latin1_with_errors( const char *input, size_t length, char *latin1_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf8_to_latin1_with_errors( const detail::input_span_of_byte_like auto &utf8_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf8_to_latin1_with_errors( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_with_errors( + utf8_input.data(), utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf8_to_latin1_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -2055,7 +6232,7 @@ convert_utf8_to_latin1_with_errors( * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of char16_t written if @@ -2064,13 +6241,21 @@ convert_utf8_to_latin1_with_errors( simdutf_warn_unused result convert_utf8_to_utf16_with_errors( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf8_to_utf16_with_errors( const detail::input_span_of_byte_like auto &utf8_input, std::span utf16_output) noexcept { - return convert_utf8_to_utf16_with_errors( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -2082,7 +6267,7 @@ convert_utf8_to_utf16_with_errors( * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of char16_t written if @@ -2091,13 +6276,21 @@ convert_utf8_to_utf16_with_errors( simdutf_warn_unused result convert_utf8_to_utf16le_with_errors( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf8_to_utf16le_with_errors( const detail::input_span_of_byte_like auto &utf8_input, std::span utf16_output) noexcept { - return convert_utf8_to_utf16le_with_errors( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16le_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -2109,7 +6302,7 @@ convert_utf8_to_utf16le_with_errors( * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of char16_t written if @@ -2118,13 +6311,21 @@ convert_utf8_to_utf16le_with_errors( simdutf_warn_unused result convert_utf8_to_utf16be_with_errors( const char *input, size_t length, char16_t *utf16_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf8_to_utf16be_with_errors( const detail::input_span_of_byte_like auto &utf8_input, std::span utf16_output) noexcept { - return convert_utf8_to_utf16be_with_errors( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf8_to_utf16be_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2138,19 +6339,27 @@ convert_utf8_to_utf16be_with_errors( * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf32_buffer the pointer to buffer that can hold conversion result + * @param utf32_output the pointer to buffer that can hold conversion result * @return the number of written char32_t; 0 if the input was not valid UTF-8 * string */ simdutf_warn_unused size_t convert_utf8_to_utf32( const char *input, size_t length, char32_t *utf32_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, std::span utf32_output) noexcept { - return convert_utf8_to_utf32( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert(utf8_input.data(), utf8_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -2162,7 +6371,7 @@ convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf32_buffer the pointer to buffer that can hold conversion result + * @param utf32_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of char32_t written if @@ -2171,13 +6380,21 @@ convert_utf8_to_utf32(const detail::input_span_of_byte_like auto &utf8_input, simdutf_warn_unused result convert_utf8_to_utf32_with_errors( const char *input, size_t length, char32_t *utf32_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf8_to_utf32_with_errors( const detail::input_span_of_byte_like auto &utf8_input, std::span utf32_output) noexcept { - return convert_utf8_to_utf32_with_errors( - reinterpret_cast(utf8_input.data()), utf8_input.size(), - utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_with_errors( + utf8_input.data(), utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf8_to_utf32_with_errors( + reinterpret_cast(utf8_input.data()), utf8_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -2205,12 +6422,21 @@ convert_utf8_to_utf32_with_errors( simdutf_warn_unused size_t convert_valid_utf8_to_latin1( const char *input, size_t length, char *latin1_output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_latin1( const detail::input_span_of_byte_like auto &valid_utf8_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_valid_utf8_to_latin1( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size(), latin1_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_latin1::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), latin1_output.data()); + } else + #endif + { + return convert_valid_utf8_to_latin1( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), latin1_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -2229,12 +6455,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_latin1( simdutf_warn_unused size_t convert_valid_utf8_to_utf16( const char *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16( const detail::input_span_of_byte_like auto &valid_utf8_input, std::span utf16_output) noexcept { - return convert_valid_utf8_to_utf16( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -2251,12 +6486,22 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16( simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( const char *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16le( const detail::input_span_of_byte_like auto &valid_utf8_input, std::span utf16_output) noexcept { - return convert_valid_utf8_to_utf16le( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size(), utf16_output.data()); + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16le( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -2273,12 +6518,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16le( simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( const char *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf16be( const detail::input_span_of_byte_like auto &valid_utf8_input, std::span utf16_output) noexcept { - return convert_valid_utf8_to_utf16be( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf16::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf16_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf16be( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2297,12 +6551,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf16be( simdutf_warn_unused size_t convert_valid_utf8_to_utf32( const char *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf32( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf8_to_utf32( const detail::input_span_of_byte_like auto &valid_utf8_input, std::span utf32_output) noexcept { - return convert_valid_utf8_to_utf32( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8_to_utf32::convert_valid( + valid_utf8_input.data(), valid_utf8_input.size(), utf32_output.data()); + } else + #endif + { + return convert_valid_utf8_to_utf32( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size(), utf32_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -2319,10 +6582,20 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf8_to_utf32( simdutf_warn_unused size_t utf8_length_from_latin1(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t utf8_length_from_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf8_length_from_latin1( const detail::input_span_of_byte_like auto &latin1_input) noexcept { - return utf8_length_from_latin1( - reinterpret_cast(latin1_input.data()), latin1_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::latin1_to_utf8::utf8_length_from_latin1(latin1_input.data(), + latin1_input.size()); + } else + #endif + { + return utf8_length_from_latin1( + reinterpret_cast(latin1_input.data()), + latin1_input.size()); + } } #endif // SIMDUTF_SPAN @@ -2342,11 +6615,20 @@ simdutf_really_inline simdutf_warn_unused size_t utf8_length_from_latin1( simdutf_warn_unused size_t latin1_length_from_utf8(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t latin1_length_from_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +latin1_length_from_utf8( const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { - return latin1_length_from_utf8( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return latin1_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_LATIN1 @@ -2369,11 +6651,20 @@ simdutf_really_inline simdutf_warn_unused size_t latin1_length_from_utf8( simdutf_warn_unused size_t utf16_length_from_utf8(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t utf16_length_from_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf16_length_from_utf8( const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { - return utf16_length_from_utf8( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::utf16_length_from_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf16_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2398,11 +6689,21 @@ simdutf_really_inline simdutf_warn_unused size_t utf16_length_from_utf8( simdutf_warn_unused size_t utf32_length_from_utf8(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t utf32_length_from_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf8( const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { - return utf32_length_from_utf8( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size()); + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return utf32_length_from_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -2427,11 +6728,20 @@ simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8(utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2446,6 +6756,11 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_utf8( * * This function is not BOM-aware. * + * Using convert_utf16_to_utf8_safe instead of convert_utf16_to_utf8 comes with + * a significant penalty in some cases, being up to three times slower, + * especially on short inputs. If you have allocated the output buffer so that + * it contains utf8_length_from_utf16(input, length) bytes, then prefer + * convert_utf16_to_utf8. * * @param input the UTF-16 string to convert * @param length the length of the string in 16-bit code units (char16_t) @@ -2458,18 +6773,34 @@ simdutf_warn_unused size_t convert_utf16_to_utf8_safe(const char16_t *input, char *utf8_output, size_t utf8_len) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_utf8_safe( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_safe( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - // implementation note: outputspan is a forwarding ref to avoid copying and - // allow both lvalues and rvalues. std::span can be copied without problems, - // but std::vector should not, and this function should accept both. it will - // allow using an owning rvalue ref (example: passing a temporary std::string) - // as output, but the user will quickly find out that he has no way of getting - // the data out of the object in that case. - return convert_utf16_to_utf8_safe( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data()), utf8_output.size()); + // implementation note: outputspan is a forwarding ref to avoid copying + // and allow both lvalues and rvalues. std::span can be copied without + // problems, but std::vector should not, and this function should accept + // both. it will allow using an owning rvalue ref (example: passing a + // temporary std::string) as output, but the user will quickly find out + // that he has no way of getting the data out of the object in that case. + #if SIMDUTF_CPLUSPLUS23 + if consteval { + const full_result r = + scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data(), + utf8_output.size()); + if (r.error != error_code::SUCCESS && + r.error != error_code::OUTPUT_BUFFER_TOO_SMALL) { + return 0; + } + return r.output_count; + } else + #endif + { + return convert_utf16_to_utf8_safe( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data()), utf8_output.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2493,12 +6824,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_utf8_safe( simdutf_warn_unused size_t convert_utf16_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_latin1( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16_to_latin1( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2521,12 +6861,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf16_to_latin1( simdutf_warn_unused size_t convert_utf16le_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16le_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_latin1( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16le_to_latin1( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2547,12 +6896,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf16le_to_latin1( simdutf_warn_unused size_t convert_utf16be_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16be_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_latin1( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16be_to_latin1( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 @@ -2576,11 +6934,21 @@ simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16le_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16le_to_utf8(utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2602,11 +6970,21 @@ simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf16be_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16be_to_utf8(utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2631,13 +7009,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf16be_to_utf8( simdutf_warn_unused result convert_utf16_to_latin1_with_errors( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16_to_latin1_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16_to_latin1_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2659,13 +7045,21 @@ convert_utf16_to_latin1_with_errors( simdutf_warn_unused result convert_utf16le_to_latin1_with_errors( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16le_to_latin1_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16le_to_latin1_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16le_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2689,13 +7083,21 @@ convert_utf16le_to_latin1_with_errors( simdutf_warn_unused result convert_utf16be_to_latin1_with_errors( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16be_to_latin1_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf16be_to_latin1_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_with_errors( + utf16_input.data(), utf16_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf16be_to_latin1_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 @@ -2721,13 +7123,21 @@ convert_utf16be_to_latin1_with_errors( simdutf_warn_unused result convert_utf16_to_utf8_with_errors( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16_to_utf8_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16_to_utf8_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2750,13 +7160,21 @@ convert_utf16_to_utf8_with_errors( simdutf_warn_unused result convert_utf16le_to_utf8_with_errors( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16le_to_utf8_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16le_to_utf8_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2779,20 +7197,135 @@ convert_utf16le_to_utf8_with_errors( simdutf_warn_unused result convert_utf16be_to_utf8_with_errors( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16be_to_utf8_with_errors( std::span utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf16be_to_utf8_with_errors( - utf16_input.data(), utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_errors( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16LE string into UTF-8 string, replacing + * unpaired surrogates with the Unicode replacement character U+FFFD. + * + * This function always succeeds: unpaired surrogates are replaced with + * U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD). + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units + */ +simdutf_warn_unused size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16le_to_utf8_with_replacement( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_replacement( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16le_to_utf8_with_replacement( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16BE string into UTF-8 string, replacing + * unpaired surrogates with the Unicode replacement character U+FFFD. + * + * This function always succeeds: unpaired surrogates are replaced with + * U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD). + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units + */ +simdutf_warn_unused size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16be_to_utf8_with_replacement( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_replacement( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16be_to_utf8_with_replacement( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert possibly broken UTF-16 string (native endianness) into UTF-8 string, + * replacing unpaired surrogates with the Unicode replacement character U+FFFD. + * + * This function always succeeds: unpaired surrogates are replaced with + * U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte code units (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units + */ +simdutf_warn_unused size_t convert_utf16_to_utf8_with_replacement( + const char16_t *input, size_t length, char *utf8_buffer) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf16_to_utf8_with_replacement( + std::span utf16_input, + detail::output_span_of_byte_like auto &&utf8_output) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_with_replacement( + utf16_input.data(), utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf16_to_utf8_with_replacement( + utf16_input.data(), utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 +#if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 /** * Using native endianness, convert valid UTF-16 string into UTF-8 string. * - * This function assumes that the input string is valid UTF-16LE. + * This function assumes that the input string is valid UTF-16. * * This function is not BOM-aware. * @@ -2805,12 +7338,21 @@ convert_utf16be_to_utf8_with_errors( simdutf_warn_unused size_t convert_valid_utf16_to_utf8( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_utf8( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_valid_utf16_to_utf8( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2838,12 +7380,23 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16_to_utf8( simdutf_warn_unused size_t convert_valid_utf16_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16_to_latin1( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_valid_utf16_to_latin1( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2869,13 +7422,23 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16_to_latin1( simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t convert_valid_utf16le_to_latin1( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_valid_utf16le_to_latin1( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16le_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2901,13 +7464,23 @@ convert_valid_utf16le_to_latin1( simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( const char16_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t convert_valid_utf16be_to_latin1( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_valid_utf16be_to_latin1( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_latin1::convert_valid_impl( + detail::constexpr_cast_ptr(valid_utf16_input.data()), + valid_utf16_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } else + #endif + { + return convert_valid_utf16be_to_latin1( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 @@ -2916,8 +7489,7 @@ convert_valid_utf16be_to_latin1( /** * Convert valid UTF-16LE string into UTF-8 string. * - * This function assumes that the input string is valid UTF-16LE and that it can - * be represented as Latin1. + * This function assumes that the input string is valid UTF-16LE * * This function is not BOM-aware. * @@ -2930,12 +7502,21 @@ convert_valid_utf16be_to_latin1( simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16le_to_utf8( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_valid_utf16le_to_utf8( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -2955,12 +7536,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16le_to_utf8( simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( const char16_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf16be_to_utf8( std::span valid_utf16_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_valid_utf16be_to_utf8( - valid_utf16_input.data(), valid_utf16_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf8::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf8( + valid_utf16_input.data(), valid_utf16_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -2984,11 +7574,20 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf16be_to_utf8( simdutf_warn_unused size_t convert_utf16_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf16_to_utf32(std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), - utf32_output.data()); + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3009,11 +7608,19 @@ convert_utf16_to_utf32(std::span utf16_input, simdutf_warn_unused size_t convert_utf16le_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf16le_to_utf32(std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), - utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3034,11 +7641,19 @@ convert_utf16le_to_utf32(std::span utf16_input, simdutf_warn_unused size_t convert_utf16be_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf16be_to_utf32(std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), - utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32(utf16_input.data(), utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3062,11 +7677,19 @@ convert_utf16be_to_utf32(std::span utf16_input, simdutf_warn_unused result convert_utf16_to_utf32_with_errors( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16_to_utf32_with_errors(std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16_to_utf32_with_errors( - utf16_input.data(), utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3089,12 +7712,20 @@ convert_utf16_to_utf32_with_errors(std::span utf16_input, simdutf_warn_unused result convert_utf16le_to_utf32_with_errors( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16le_to_utf32_with_errors( std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16le_to_utf32_with_errors( - utf16_input.data(), utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16le_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3117,12 +7748,20 @@ convert_utf16le_to_utf32_with_errors( simdutf_warn_unused result convert_utf16be_to_utf32_with_errors( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf16be_to_utf32_with_errors( std::span utf16_input, std::span utf32_output) noexcept { - return convert_utf16be_to_utf32_with_errors( - utf16_input.data(), utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } else + #endif + { + return convert_utf16be_to_utf32_with_errors( + utf16_input.data(), utf16_input.size(), utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3143,11 +7782,21 @@ convert_utf16be_to_utf32_with_errors( simdutf_warn_unused size_t convert_valid_utf16_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf16_to_utf32(std::span valid_utf16_input, std::span utf32_output) noexcept { - return convert_valid_utf16_to_utf32( - valid_utf16_input.data(), valid_utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3167,11 +7816,21 @@ convert_valid_utf16_to_utf32(std::span valid_utf16_input, simdutf_warn_unused size_t convert_valid_utf16le_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf16le_to_utf32(std::span valid_utf16_input, std::span utf32_output) noexcept { - return convert_valid_utf16le_to_utf32( - valid_utf16_input.data(), valid_utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16le_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3191,32 +7850,25 @@ convert_valid_utf16le_to_utf32(std::span valid_utf16_input, simdutf_warn_unused size_t convert_valid_utf16be_to_utf32( const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf16be_to_utf32(std::span valid_utf16_input, std::span utf32_output) noexcept { - return convert_valid_utf16be_to_utf32( - valid_utf16_input.data(), valid_utf16_input.size(), utf32_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16_to_utf32::convert_valid( + valid_utf16_input.data(), valid_utf16_input.size(), + utf32_output.data()); + } else + #endif + { + return convert_valid_utf16be_to_utf32(valid_utf16_input.data(), + valid_utf16_input.size(), + utf32_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 -#if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 -/** - * Compute the number of bytes that this UTF-16LE/BE string would require in - * Latin1 format. - * - * This function does not validate the input. It is acceptable to pass invalid - * UTF-16 strings but in such cases the result is implementation defined. - * - * This function is not BOM-aware. - * - * @param length the length of the string in 2-byte code units (char16_t) - * @return the number of bytes required to encode the UTF-16LE string as Latin1 - */ -simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept; - -#endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_LATIN1 - #if SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 /** * Using native endianness; Compute the number of bytes that this UTF-16 @@ -3232,10 +7884,18 @@ simdutf_warn_unused size_t latin1_length_from_utf16(size_t length) noexcept; simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t utf8_length_from_utf16(std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3260,11 +7920,19 @@ utf8_length_from_utf16(std::span valid_utf16_input) noexcept { simdutf_warn_unused result utf8_length_from_utf16_with_replacement( const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result utf8_length_from_utf16_with_replacement( std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16_with_replacement< + endianness::NATIVE>(valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16_with_replacement(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3282,10 +7950,18 @@ utf8_length_from_utf16_with_replacement( simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t utf8_length_from_utf16le(std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16le(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3303,10 +7979,18 @@ utf8_length_from_utf16le(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t utf8_length_from_utf16be(std::span valid_utf16_input) noexcept { - return utf8_length_from_utf16be(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf8_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf8_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF16 @@ -3329,11 +8013,20 @@ simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf32_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_utf8( std::span utf32_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8(utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -3356,13 +8049,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf32_to_utf8( simdutf_warn_unused result convert_utf32_to_utf8_with_errors( const char32_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf32_to_utf8_with_errors( std::span utf32_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_utf32_to_utf8_with_errors( - utf32_input.data(), utf32_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_utf32_to_utf8_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN @@ -3382,12 +8083,21 @@ convert_utf32_to_utf8_with_errors( simdutf_warn_unused size_t convert_valid_utf32_to_utf8( const char32_t *input, size_t length, char *utf8_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf32_to_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_valid_utf32_to_utf8( std::span valid_utf32_input, detail::output_span_of_byte_like auto &&utf8_output) noexcept { - return convert_valid_utf32_to_utf8( - valid_utf32_input.data(), valid_utf32_input.size(), - reinterpret_cast(utf8_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf8::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), utf8_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf8( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(utf8_output.data())); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -3410,11 +8120,19 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf32_to_utf8( simdutf_warn_unused size_t convert_utf32_to_utf16( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf32_to_utf16(std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3434,11 +8152,19 @@ convert_utf32_to_utf16(std::span utf32_input, simdutf_warn_unused size_t convert_utf32_to_utf16le( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf32_to_utf16le(std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 @@ -3461,12 +8187,21 @@ convert_utf32_to_utf16le(std::span utf32_input, simdutf_warn_unused size_t convert_utf32_to_latin1( const char32_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_utf32_to_latin1( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +convert_utf32_to_latin1( std::span utf32_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf32_to_latin1( - utf32_input.data(), utf32_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -3490,13 +8225,21 @@ simdutf_really_inline simdutf_warn_unused size_t convert_utf32_to_latin1( simdutf_warn_unused result convert_utf32_to_latin1_with_errors( const char32_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf32_to_latin1_with_errors( std::span utf32_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_utf32_to_latin1_with_errors( - utf32_input.data(), utf32_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_with_errors( + utf32_input.data(), utf32_input.size(), latin1_output.data()); + } else + #endif + { + return convert_utf32_to_latin1_with_errors( + utf32_input.data(), utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -3523,12 +8266,23 @@ convert_utf32_to_latin1_with_errors( simdutf_warn_unused size_t convert_valid_utf32_to_latin1( const char32_t *input, size_t length, char *latin1_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf32_to_latin1( +simdutf_really_inline simdutf_constexpr23 simdutf_warn_unused size_t +convert_valid_utf32_to_latin1( std::span valid_utf32_input, detail::output_span_of_byte_like auto &&latin1_output) noexcept { - return convert_valid_utf32_to_latin1( - valid_utf32_input.data(), valid_utf32_input.size(), - reinterpret_cast(latin1_output.data())); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_latin1::convert_valid( + detail::constexpr_cast_ptr(valid_utf32_input.data()), + valid_utf32_input.size(), + detail::constexpr_cast_writeptr(latin1_output.data())); + } + #endif + { + return convert_valid_utf32_to_latin1( + valid_utf32_input.data(), valid_utf32_input.size(), + reinterpret_cast(latin1_output.data())); + } } #endif // SIMDUTF_SPAN @@ -3544,7 +8298,10 @@ simdutf_really_inline simdutf_warn_unused size_t convert_valid_utf32_to_latin1( * @param length the length of the string in 4-byte code units (char32_t) * @return the number of bytes required to encode the UTF-32 string as Latin1 */ -simdutf_warn_unused size_t latin1_length_from_utf32(size_t length) noexcept; +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +latin1_length_from_utf32(size_t length) noexcept { + return length; +} /** * Compute the number of bytes that this Latin1 string would require in UTF-32 @@ -3554,7 +8311,10 @@ simdutf_warn_unused size_t latin1_length_from_utf32(size_t length) noexcept; * @return the length of the string in 4-byte code units (char32_t) required to * encode the Latin1 string as UTF-32 */ -simdutf_warn_unused size_t utf32_length_from_latin1(size_t length) noexcept; +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t +utf32_length_from_latin1(size_t length) noexcept { + return length; +} #endif // SIMDUTF_FEATURE_UTF32 && SIMDUTF_FEATURE_LATIN1 #if SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 @@ -3574,11 +8334,19 @@ simdutf_warn_unused size_t utf32_length_from_latin1(size_t length) noexcept; simdutf_warn_unused size_t convert_utf32_to_utf16be( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_utf32_to_utf16be(std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be(utf32_input.data(), utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3602,11 +8370,19 @@ convert_utf32_to_utf16be(std::span utf32_input, simdutf_warn_unused result convert_utf32_to_utf16_with_errors( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf32_to_utf16_with_errors(std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16_with_errors( - utf32_input.data(), utf32_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3629,12 +8405,20 @@ convert_utf32_to_utf16_with_errors(std::span utf32_input, simdutf_warn_unused result convert_utf32_to_utf16le_with_errors( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf32_to_utf16le_with_errors( std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16le_with_errors( - utf32_input.data(), utf32_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16le_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3657,12 +8441,20 @@ convert_utf32_to_utf16le_with_errors( simdutf_warn_unused result convert_utf32_to_utf16be_with_errors( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result convert_utf32_to_utf16be_with_errors( std::span utf32_input, std::span utf16_output) noexcept { - return convert_utf32_to_utf16be_with_errors( - utf32_input.data(), utf32_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } else + #endif + { + return convert_utf32_to_utf16be_with_errors( + utf32_input.data(), utf32_input.size(), utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3682,11 +8474,22 @@ convert_utf32_to_utf16be_with_errors( simdutf_warn_unused size_t convert_valid_utf32_to_utf16( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf32_to_utf16(std::span valid_utf32_input, std::span utf16_output) noexcept { - return convert_valid_utf32_to_utf16( - valid_utf32_input.data(), valid_utf32_input.size(), utf16_output.data()); + + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3706,11 +8509,21 @@ convert_valid_utf32_to_utf16(std::span valid_utf32_input, simdutf_warn_unused size_t convert_valid_utf32_to_utf16le( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf32_to_utf16le(std::span valid_utf32_input, std::span utf16_output) noexcept { - return convert_valid_utf32_to_utf16le( - valid_utf32_input.data(), valid_utf32_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16le(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN @@ -3730,11 +8543,21 @@ convert_valid_utf32_to_utf16le(std::span valid_utf32_input, simdutf_warn_unused size_t convert_valid_utf32_to_utf16be( const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t convert_valid_utf32_to_utf16be(std::span valid_utf32_input, std::span utf16_output) noexcept { - return convert_valid_utf32_to_utf16be( - valid_utf32_input.data(), valid_utf32_input.size(), utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32_to_utf16::convert_valid( + valid_utf32_input.data(), valid_utf32_input.size(), + utf16_output.data()); + } else + #endif + { + return convert_valid_utf32_to_utf16be(valid_utf32_input.data(), + valid_utf32_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 @@ -3756,11 +8579,19 @@ convert_valid_utf32_to_utf16be(std::span valid_utf32_input, void change_endianness_utf16(const char16_t *input, size_t length, char16_t *output) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline void +simdutf_really_inline simdutf_constexpr23 void change_endianness_utf16(std::span utf16_input, std::span utf16_output) noexcept { - return change_endianness_utf16(utf16_input.data(), utf16_input.size(), - utf16_output.data()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::change_endianness_utf16( + utf16_input.data(), utf16_input.size(), utf16_output.data()); + } else + #endif + { + return change_endianness_utf16(utf16_input.data(), utf16_input.size(), + utf16_output.data()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 @@ -3780,10 +8611,18 @@ change_endianness_utf16(std::span utf16_input, simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t utf8_length_from_utf32(std::span valid_utf32_input) noexcept { - return utf8_length_from_utf32(valid_utf32_input.data(), - valid_utf32_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf8_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 && SIMDUTF_FEATURE_UTF32 @@ -3803,10 +8642,18 @@ utf8_length_from_utf32(std::span valid_utf32_input) noexcept { simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t utf16_length_from_utf32(std::span valid_utf32_input) noexcept { - return utf16_length_from_utf32(valid_utf32_input.data(), - valid_utf32_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf32::utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } else + #endif + { + return utf16_length_from_utf32(valid_utf32_input.data(), + valid_utf32_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3828,10 +8675,18 @@ utf16_length_from_utf32(std::span valid_utf32_input) noexcept { simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t utf32_length_from_utf16(std::span valid_utf16_input) noexcept { - return utf32_length_from_utf16(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3853,10 +8708,19 @@ utf32_length_from_utf16(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t utf32_length_from_utf16le( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16le( std::span valid_utf16_input) noexcept { - return utf32_length_from_utf16le(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3878,10 +8742,19 @@ simdutf_really_inline simdutf_warn_unused size_t utf32_length_from_utf16le( simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t utf32_length_from_utf16be( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +utf32_length_from_utf16be( std::span valid_utf16_input) noexcept { - return utf32_length_from_utf16be(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::utf32_length_from_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return utf32_length_from_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 && SIMDUTF_FEATURE_UTF32 @@ -3904,9 +8777,17 @@ simdutf_really_inline simdutf_warn_unused size_t utf32_length_from_utf16be( simdutf_warn_unused size_t count_utf16(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf16(std::span valid_utf16_input) noexcept { - return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3927,9 +8808,17 @@ count_utf16(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t count_utf16le(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf16le(std::span valid_utf16_input) noexcept { - return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16le(valid_utf16_input.data(), valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3950,9 +8839,17 @@ count_utf16le(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t count_utf16be(const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf16be(std::span valid_utf16_input) noexcept { - return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::count_code_points( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return count_utf16be(valid_utf16_input.data(), valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF16 @@ -3973,10 +8870,18 @@ count_utf16be(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t count_utf8(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t count_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t count_utf8( const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { - return count_utf8(reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::count_code_points(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return count_utf8(reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } } #endif // SIMDUTF_SPAN @@ -3996,11 +8901,20 @@ simdutf_really_inline simdutf_warn_unused size_t count_utf8( */ simdutf_warn_unused size_t trim_partial_utf8(const char *input, size_t length); #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t trim_partial_utf8( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf8( const detail::input_span_of_byte_like auto &valid_utf8_input) noexcept { - return trim_partial_utf8( - reinterpret_cast(valid_utf8_input.data()), - valid_utf8_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf8::trim_partial_utf8(valid_utf8_input.data(), + valid_utf8_input.size()); + } else + #endif + { + return trim_partial_utf8( + reinterpret_cast(valid_utf8_input.data()), + valid_utf8_input.size()); + } } #endif // SIMDUTF_SPAN #endif // SIMDUTF_FEATURE_UTF8 @@ -4023,10 +8937,18 @@ simdutf_really_inline simdutf_warn_unused size_t trim_partial_utf8( simdutf_warn_unused size_t trim_partial_utf16be(const char16_t *input, size_t length); #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t trim_partial_utf16be(std::span valid_utf16_input) noexcept { - return trim_partial_utf16be(valid_utf16_input.data(), - valid_utf16_input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16be(valid_utf16_input.data(), + valid_utf16_input.size()); + } } #endif // SIMDUTF_SPAN @@ -4047,70 +8969,1960 @@ trim_partial_utf16be(std::span valid_utf16_input) noexcept { simdutf_warn_unused size_t trim_partial_utf16le(const char16_t *input, size_t length); #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t trim_partial_utf16le(std::span valid_utf16_input) noexcept { - return trim_partial_utf16le(valid_utf16_input.data(), + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16le(valid_utf16_input.data(), + valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Given a valid UTF-16 string having a possibly truncated last character, + * this function checks the end of string. If the last character is truncated + * (or partial), then it returns a shorter length (shorter by 1 unit) so that + * the short UTF-16 strings only contain complete characters. If there is no + * truncated character, the original length is returned. + * + * This function assumes that the input string is valid UTF-16, but possibly + * truncated. We use the native endianness. + * + * @param input the UTF-16 string to process + * @param length the length of the string in bytes + * @return the length of the string in unit, possibly shorter by 1 unit + */ +simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, + size_t length); + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +trim_partial_utf16(std::span valid_utf16_input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::utf16::trim_partial_utf16( + valid_utf16_input.data(), valid_utf16_input.size()); + } else + #endif + { + return trim_partial_utf16(valid_utf16_input.data(), valid_utf16_input.size()); + } +} + #endif // SIMDUTF_SPAN +#endif // SIMDUTF_FEATURE_UTF16 + +#if SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || \ + SIMDUTF_FEATURE_DETECT_ENCODING + #ifndef SIMDUTF_NEED_TRAILING_ZEROES + #define SIMDUTF_NEED_TRAILING_ZEROES 1 + #endif +#endif // SIMDUTF_FEATURE_BASE64 || SIMDUTF_FEATURE_UTF16 || + // SIMDUTF_FEATURE_DETECT_ENCODING + +#if SIMDUTF_FEATURE_BASE64 +// base64_options are used to specify the base64 encoding options. +// ASCII spaces are ' ', '\t', '\n', '\r', '\f' +// garbage characters are characters that are not part of the base64 alphabet +// nor ASCII spaces. +constexpr uint64_t base64_reverse_padding = + 2; /* modifier for base64_default and base64_url */ +enum base64_options : uint64_t { + base64_default = 0, /* standard base64 format (with padding) */ + base64_url = 1, /* base64url format (no padding) */ + base64_default_no_padding = + base64_default | + base64_reverse_padding, /* standard base64 format without padding */ + base64_url_with_padding = + base64_url | base64_reverse_padding, /* base64url with padding */ + base64_default_accept_garbage = + 4, /* standard base64 format accepting garbage characters, the input stops + with the first '=' if any */ + base64_url_accept_garbage = + 5, /* base64url format accepting garbage characters, the input stops with + the first '=' if any */ + base64_default_or_url = + 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ + base64_default_or_url_accept_garbage = + 12, /* standard/base64url hybrid format accepting garbage characters + (only meaningful for decoding!), the input stops with the first '=' + if any */ +}; + +// last_chunk_handling_options are used to specify the handling of the last +// chunk in base64 decoding. +// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 +enum last_chunk_handling_options : uint64_t { + loose = 0, /* standard base64 format, decode partial final chunk */ + strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and + unpadded, or non-zero bit padding */ + stop_before_partial = + 2, /* if the last chunk is partial, ignore it (no error) */ + only_full_chunks = + 3 /* only decode full blocks (4 base64 characters, no padding) */ +}; + +inline simdutf_constexpr23 bool +is_partial(last_chunk_handling_options options) { + return (options == stop_before_partial) || (options == only_full_chunks); +} + +namespace detail { +simdutf_warn_unused const char *find(const char *start, const char *end, + char character) noexcept; +simdutf_warn_unused const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept; +} // namespace detail + +/** + * Find the first occurrence of a character in a string. If the character is + * not found, return a pointer to the end of the string. + * @param start the start of the string + * @param end the end of the string + * @param character the character to find + * @return a pointer to the first occurrence of the character in the string, + * or a pointer to the end of the string if the character is not found. + * + */ +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * +find(const char *start, const char *end, char character) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * +find(const char16_t *start, const char16_t *end, char16_t character) noexcept { + // implementation note: this is repeated instead of a template, to ensure + // the api is still a function and compiles without concepts + #if SIMDUTF_CPLUSPLUS23 + if consteval { + for (; start != end; ++start) + if (*start == character) + return start; + return end; + } else + #endif + { + return detail::find(start, end, character); + } +} +} + // We include base64_tables once. +/* begin file include/simdutf/base64_tables.h */ +#ifndef SIMDUTF_BASE64_TABLES_H +#define SIMDUTF_BASE64_TABLES_H +#include + +namespace simdutf { +namespace { +namespace tables { +namespace base64 { +namespace base64_default { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', '+', '+', '/', '/', '/', + '/'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', + '/'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; + +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default + +namespace base64_url { + +constexpr char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', 'C', 'C', 'D', 'D', 'D', + 'D', 'E', 'E', 'E', 'E', 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', 'K', 'K', 'K', 'K', 'L', + 'L', 'L', 'L', 'M', 'M', 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', 'R', 'R', 'S', 'S', 'S', + 'S', 'T', 'T', 'T', 'T', 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', 'Z', 'Z', 'Z', 'Z', 'a', + 'a', 'a', 'a', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', 'g', 'g', 'h', 'h', 'h', + 'h', 'i', 'i', 'i', 'i', 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', 'o', 'o', 'o', 'o', 'p', + 'p', 'p', 'p', 'q', 'q', 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', 'v', 'v', 'w', 'w', 'w', + 'w', 'x', 'x', 'x', 'x', 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', '3', '3', '3', '3', '4', + '4', '4', '4', '5', '5', '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '-', '-', '-', '-', '_', '_', '_', + '_'}; + +constexpr char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', + 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', + '_'}; + +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_url + +namespace base64_default_or_url { +constexpr uint32_t d0[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000f8, 0x01ffffff, 0x000000fc, + 0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, + 0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, + 0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, + 0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, + 0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, + 0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, + 0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, + 0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, + 0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, + 0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, + 0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d1[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000e003, 0x01ffffff, 0x0000f003, + 0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, + 0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, + 0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, + 0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, + 0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, + 0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, + 0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, + 0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, + 0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, + 0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, + 0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d2[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00800f00, 0x01ffffff, 0x00c00f00, + 0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, + 0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, + 0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, + 0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, + 0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, + 0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, + 0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, + 0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, + 0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, + 0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, + 0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +constexpr uint32_t d3[256] = { + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003e0000, 0x01ffffff, 0x003f0000, + 0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, + 0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, + 0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, + 0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, + 0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, + 0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, + 0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, + 0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, + 0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, + 0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, + 0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, + 0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, + 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}; +} // namespace base64_default_or_url +constexpr uint64_t thintable_epi8[256] = { + 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, + 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, + 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, + 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, + 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, + 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, + 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, + 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, + 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, + 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, + 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, + 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, + 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, + 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, + 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, + 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, + 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, + 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, + 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, + 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, + 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, + 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, + 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, + 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, + 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, + 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, + 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, + 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, + 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, + 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, + 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, + 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, + 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, + 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, + 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, + 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, + 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, + 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, + 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, + 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, + 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, + 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, + 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, + 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, + 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, + 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, + 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, + 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, + 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, + 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, + 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, + 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, + 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, + 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, + 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, + 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, + 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, + 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, + 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, + 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, + 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, + 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, + 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, + 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, + 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, + 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, + 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, + 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, + 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, + 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, + 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, + 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, + 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, + 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, + 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, + 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, + 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, + 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, + 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, + 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, + 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, + 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, + 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, + 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, + 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, + 0x0000000000000000, +}; + +constexpr uint8_t pshufb_combine_table[272] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +constexpr unsigned char BitsSetTable256mul2[256] = { + 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, + 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, + 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, + 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, + 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, + 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, + 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, + 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, + 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, + 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, + 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, + 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, + 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, + 14, 10, 12, 12, 14, 12, 14, 14, 16}; + +constexpr uint8_t to_base64_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +constexpr uint8_t to_base64_default_or_url_value[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 64, 64, 255, 64, 64, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, + 62, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, + 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 255, 255, 255, 255, 63, 255, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + +static_assert(sizeof(to_base64_value) == 256, + "to_base64_value must have 256 elements"); +static_assert(sizeof(to_base64_url_value) == 256, + "to_base64_url_value must have 256 elements"); +static_assert(to_base64_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t(' ')] == 64, + "space must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\t')] == 64, + "tab must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\r')] == 64, + "cr must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\n')] == 64, + "lf must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('\f')] == 64, + "ff must be == 64 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('+')] == 62, + "+ must be == 62 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('-')] == 62, + "- must be == 62 in to_base64_url_value"); +static_assert(to_base64_value[uint8_t('/')] == 63, + "/ must be == 63 in to_base64_value"); +static_assert(to_base64_url_value[uint8_t('_')] == 63, + "_ must be == 63 in to_base64_url_value"); +} // namespace base64 +} // namespace tables +} // unnamed namespace +} // namespace simdutf + +#endif // SIMDUTF_BASE64_TABLES_H +/* end file include/simdutf/base64_tables.h */ +/* begin file include/simdutf/scalar/base64.h */ +#ifndef SIMDUTF_BASE64_H +#define SIMDUTF_BASE64_H + +#include +#include +#include + +namespace simdutf { +namespace scalar { +namespace { +namespace base64 { + +// This function is not expected to be fast. Do not use in long loops. +// In most instances you should be using is_ignorable. +template bool is_ascii_white_space(char_type c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; +} + +template simdutf_constexpr23 bool is_eight_byte(char_type c) { + if constexpr (sizeof(char_type) == 1) { + return true; + } + return uint8_t(c) == c; +} + +template +simdutf_constexpr23 bool is_ignorable(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return false; + } + if (is_eight_byte(c) && code == 64) { + return true; + } + return ignore_garbage; +} +template +simdutf_constexpr23 bool is_base64(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +simdutf_constexpr23 bool is_base64_or_padding(char_type c, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + if (c == '=') { + return true; + } + uint8_t code = to_base64[uint8_t(c)]; + if (is_eight_byte(c) && code <= 63) { + return true; + } + return false; +} + +template +bool is_ignorable_or_padding(char_type c, simdutf::base64_options options) { + return is_ignorable(c, options) || c == '='; +} + +struct reduced_input { + size_t equalsigns; // number of padding characters '=', typically 0, 1, 2. + size_t equallocation; // location of the first padding character if any + size_t srclen; // length of the input buffer before padding + size_t full_input_length; // length of the input buffer with padding but + // without ignorable characters +}; + +// find the end of the base64 input buffer +// It returns the number of padding characters, the location of the first +// padding character if any, the length of the input buffer before padding +// and the length of the input buffer with padding. The input buffer is not +// modified. The function assumes that there are at most two padding characters. +template +simdutf_constexpr23 reduced_input find_end(const char_type *src, size_t srclen, + simdutf::base64_options options) { + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + size_t equalsigns = 0; + // We intentionally include trailing spaces in the full input length. + // See https://github.com/simdutf/simdutf/issues/824 + size_t full_input_length = srclen; + // skip trailing spaces + while (!ignore_garbage && srclen > 0 && + scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + size_t equallocation = + srclen; // location of the first padding character if any + if (ignore_garbage) { + // Technically, we don't need to find the first padding character, we can + // just change our algorithms, but it adds substantial complexity. + auto it = simdutf::find(src, src + srclen, '='); + if (it != src + srclen) { + equallocation = it - src; + equalsigns = 1; + srclen = equallocation; + full_input_length = equallocation + 1; + } + return {equalsigns, equallocation, srclen, full_input_length}; + } + if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { + // This is the last '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 1; + // skip trailing spaces + while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && + to_base64[uint8_t(src[srclen - 1])] == 64) { + srclen--; + } + if (srclen > 0 && src[srclen - 1] == '=') { + // This is the second '=' sign. + equallocation = srclen - 1; + srclen--; + equalsigns = 2; + } + } + return {equalsigns, equallocation, srclen, full_input_length}; +} + +// Returns true upon success. The destination buffer must be large enough. +// This functions assumes that the padding (=) has been removed. +// if check_capacity is true, it will check that the destination buffer is +// large enough. If it is not, it will return OUTPUT_BUFFER_TOO_SMALL. +template +simdutf_constexpr23 full_result base64_tail_decode_impl( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + char *dstend = dst + outlen; + (void)dstend; + // This looks like 10 branches, but we expect the compiler to resolve this to + // two branches (easily predicted): + const uint8_t *to_base64 = + (options & base64_default_or_url) + ? tables::base64::to_base64_default_or_url_value + : ((options & base64_url) ? tables::base64::to_base64_url_value + : tables::base64::to_base64_value); + const uint32_t *d0 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d0 + : ((options & base64_url) ? tables::base64::base64_url::d0 + : tables::base64::base64_default::d0); + const uint32_t *d1 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d1 + : ((options & base64_url) ? tables::base64::base64_url::d1 + : tables::base64::base64_default::d1); + const uint32_t *d2 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d2 + : ((options & base64_url) ? tables::base64::base64_url::d2 + : tables::base64::base64_default::d2); + const uint32_t *d3 = + (options & base64_default_or_url) + ? tables::base64::base64_default_or_url::d3 + : ((options & base64_url) ? tables::base64::base64_url::d3 + : tables::base64::base64_default::d3); + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + + const char_type *srcend = src + length; + const char_type *srcinit = src; + const char *dstinit = dst; + + uint32_t x; + size_t idx; + uint8_t buffer[4]; + while (true) { + while (srcend - src >= 4 && is_eight_byte(src[0]) && + is_eight_byte(src[1]) && is_eight_byte(src[2]) && + is_eight_byte(src[3]) && + (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | + d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast(x & 0xFF); + *dst++ = static_cast((x >> 8) & 0xFF); + *dst++ = static_cast((x >> 16) & 0xFF); + src += 4; + } + const char_type *srccur = src; + idx = 0; + // we need at least four characters. +#ifdef __clang__ + // If possible, we read four characters at a time. (It is an optimization.) + if (ignore_garbage && src + 4 <= srcend) { + char_type c0 = src[0]; + char_type c1 = src[1]; + char_type c2 = src[2]; + char_type c3 = src[3]; + + uint8_t code0 = to_base64[uint8_t(c0)]; + uint8_t code1 = to_base64[uint8_t(c1)]; + uint8_t code2 = to_base64[uint8_t(c2)]; + uint8_t code3 = to_base64[uint8_t(c3)]; + + buffer[idx] = code0; + idx += (is_eight_byte(c0) && code0 <= 63); + buffer[idx] = code1; + idx += (is_eight_byte(c1) && code1 <= 63); + buffer[idx] = code2; + idx += (is_eight_byte(c2) && code2 <= 63); + buffer[idx] = code3; + idx += (is_eight_byte(c3) && code3 <= 63); + src += 4; + } +#endif + while ((idx < 4) && (src < srcend)) { + char_type c = *src; + + uint8_t code = to_base64[uint8_t(c)]; + buffer[idx] = uint8_t(code); + if (is_eight_byte(c) && code <= 63) { + idx++; + } else if (!ignore_garbage && + (code > 64 || !scalar::base64::is_eight_byte(c))) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else { + // We have a space or a newline or garbage. We ignore it. + } + src++; + } + if (idx != 4) { + simdutf_log_assert(idx < 4, "idx should be less than 4"); + // We never should have that the number of base64 characters + the + // number of padding characters is more than 4. + if (!ignore_garbage && (idx + padding_characters > 4)) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + + // The idea here is that in loose mode, + // if there is padding at all, it must be used + // to form 4-wise chunk. However, in loose mode, + // we do accept no padding at all. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::loose && + (idx >= 2) && padding_characters > 0 && + ((idx + padding_characters) & 3) != 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + + // The idea here is that in strict mode, we do not want to accept + // incomplete base64 chunks. So if the chunk was otherwise valid, we + // return BASE64_INPUT_REMAINDER. + if (!ignore_garbage && + last_chunk_options == last_chunk_handling_options::strict && + (idx >= 2) && ((idx + padding_characters) & 3) != 0) { + // The partial chunk was at src - idx + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } else + // If there is a partial chunk with insufficient padding, with + // stop_before_partial, we need to just ignore it. In "only full" + // mode, skip the minute there are padding characters. + if ((last_chunk_options == + last_chunk_handling_options::stop_before_partial && + (padding_characters + idx < 4) && (idx != 0) && + (idx >= 2 || padding_characters == 0)) || + (last_chunk_options == + last_chunk_handling_options::only_full_chunks && + (idx >= 2 || padding_characters == 0))) { + // partial means that we are *not* going to consume the read + // characters. We need to rewind the src pointer. + src = srccur; + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } else { + if (idx == 2) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xffff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 1) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + } else if (idx == 3) { + uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6); + if (!ignore_garbage && + (last_chunk_options == last_chunk_handling_options::strict) && + (triple & 0xff)) { + return {BASE64_EXTRA_BITS, size_t(src - srcinit), + size_t(dst - dstinit)}; + } + if (check_capacity && dstend - dst < 2) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + } else if (!ignore_garbage && idx == 1 && + (!is_partial(last_chunk_options) || + (is_partial(last_chunk_options) && + padding_characters > 0))) { + return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), + size_t(dst - dstinit)}; + } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { + return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), + size_t(dst - dstinit), true}; + } + return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; + } + } + if (check_capacity && dstend - dst < 3) { + return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), + size_t(dst - dstinit)}; + } + uint32_t triple = + (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + + (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); + *dst++ = static_cast((triple >> 16) & 0xFF); + *dst++ = static_cast((triple >> 8) & 0xFF); + *dst++ = static_cast(triple & 0xFF); + } +} + +template +simdutf_constexpr23 full_result base64_tail_decode( + char *dst, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, 0, src, length, padding_characters, + options, last_chunk_options); +} + +// like base64_tail_decode, but it will not write past the end of the output +// buffer. The outlen parameter is modified to reflect the number of bytes +// written. This functions assumes that the padding (=) has been removed. +// +template +simdutf_constexpr23 full_result base64_tail_decode_safe( + char *dst, size_t outlen, const char_type *src, size_t length, + size_t padding_characters, // number of padding characters + // '=', typically 0, 1, 2. + base64_options options, last_chunk_handling_options last_chunk_options) { + return base64_tail_decode_impl(dst, outlen, src, length, + padding_characters, options, + last_chunk_options); +} + +inline simdutf_constexpr23 full_result +patch_tail_result(full_result r, size_t previous_input, size_t previous_output, + size_t equallocation, size_t full_input_length, + last_chunk_handling_options last_chunk_options) { + r.input_count += previous_input; + r.output_count += previous_output; + if (r.padding_error) { + r.input_count = equallocation; + } + + if (r.error == error_code::SUCCESS) { + if (!is_partial(last_chunk_options)) { + // A success when we are not in stop_before_partial mode. + // means that we have consumed the whole input buffer. + r.input_count = full_input_length; + } else if (r.output_count % 3 != 0) { + r.input_count = full_input_length; + } + } + return r; +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +template +simdutf_constexpr23 size_t tail_encode_base64_impl( + char *dst, const char *src, size_t srclen, base64_options options, + size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { + if constexpr (use_lines) { + // sanitize line_length and starting_line_offset. + // line_length must be greater than 3. + if (line_length < 4) { + line_length = 4; + } + simdutf_log_assert(line_offset <= line_length, + "line_offset should be less than line_length"); + } + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + // This looks like 3 branches, but we expect the compiler to resolve this to + // a single branch: + const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 + : tables::base64::base64_default::e0; + const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 + : tables::base64::base64_default::e1; + const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 + : tables::base64::base64_default::e2; + char *out = dst; + size_t i = 0; + uint8_t t1, t2, t3; + for (; i + 2 < srclen; i += 3) { + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + t3 = uint8_t(src[i + 2]); + if constexpr (use_lines) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 4; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 3; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset = 2; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = '\n'; + *out++ = e2[t3]; + line_offset = 1; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + line_offset += 4; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *out++ = e2[t3]; + } + } + switch (srclen - i) { + case 0: + break; + case 1: + t1 = uint8_t(src[i]); + if constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '\n'; + *out++ = '='; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + } + } else { + if (line_offset + 2 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[uint8_t(src[i])]; + *out++ = '\n'; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[uint8_t(src[i])]; + *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[(t1 & 0x03) << 4]; + if (use_padding) { + *out++ = '='; + *out++ = '='; + } + } + break; + default: /* case 2 */ + t1 = uint8_t(src[i]); + t2 = uint8_t(src[i + 1]); + if constexpr (use_lines) { + if (use_padding) { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } else if (line_offset + 3 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '\n'; + *out++ = '='; + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + *out++ = '='; + } + } else { + if (line_offset + 3 >= line_length) { + if (line_offset == line_length) { + *out++ = '\n'; + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 1 == line_length) { + *out++ = e0[t1]; + *out++ = '\n'; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } else if (line_offset + 2 == line_length) { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = '\n'; + *out++ = e2[(t2 & 0x0F) << 2]; + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + // *out++ = '\n'; ==> no newline at the end of the output + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + } + } + } else { + *out++ = e0[t1]; + *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *out++ = e2[(t2 & 0x0F) << 2]; + if (use_padding) { + *out++ = '='; + } + } + } + return (size_t)(out - dst); +} + +// Returns the number of bytes written. The destination buffer must be large +// enough. It will add padding (=) if needed. +simdutf_unused inline simdutf_constexpr23 size_t tail_encode_base64( + char *dst, const char *src, size_t srclen, base64_options options) { + return tail_encode_base64_impl(dst, src, srclen, options); +} + +template +simdutf_warn_unused simdutf_constexpr23 size_t +maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { + // We process the padding characters ('=') at the end to make sure + // that we return an exact result when the input has no ignorable characters + // (e.g., spaces). + size_t padding = 0; + if (length > 0) { + if (input[length - 1] == '=') { + padding++; + if (length > 1 && input[length - 2] == '=') { + padding++; + } + } + } + // The input is not otherwise processed for ignorable characters or + // validation, so that the function runs in constant time (very fast). In + // practice, base64 inputs without ignorable characters are common and the + // common case are line separated inputs with relatively long lines (e.g., 76 + // characters) which leads this function to a slight (1%) overestimation of + // the output size. + // + // Of course, some inputs might contain an arbitrary number of spaces or + // newlines, which would make this function return a very pessimistic output + // size but systems that produce base64 outputs typically do not do that and + // if they do, they do not care much about minimizing memory usage. + // + // In specialized applications, users may know that their input is line + // separated, which can be checked very quickly by by iterating (e.g., over 76 + // character chunks, looking for the linefeed characters only). We could + // provide a specialized function for that, but it is not clear that the added + // complexity is worth it for us. + // + size_t actual_length = length - padding; + if (actual_length % 4 <= 1) { + return actual_length / 4 * 3; + } + // if we have a valid input, then the remainder must be 2 or 3 adding one or + // two extra bytes. + return actual_length / 4 * 3 + (actual_length % 4) - 1; +} + +// This function computes the binary length by iterating through the input +// and counting non-whitespace characters (excluding padding characters). +// We use a simple check (c > ' ') which is easy to parallelize and matches +// SIMD behavior. Only the last few characters are checked for padding '='. +template +simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64(const char_type *input, size_t length) noexcept { + // Count non-whitespace characters (c > ' ') with loop unrolling + size_t count = 0; + for (size_t i = 0; i < length; i++) { + count += (input[i] > ' '); + } + + // Check for padding '=' at the end (at most 2 padding characters) + // Scan backwards, skipping whitespace, to find padding + size_t padding = 0; + size_t pos = length; + // Skip trailing whitespace + while (pos > 0 && padding < 2) { + char_type c = input[--pos]; + if (c == '=') { + padding++; + } else if (c > ' ') { + break; + } + } + return ((count - padding) * 3) / 4; +} + +template +simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details_impl( + const char_type *input, size_t length, char *output, base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode( + output, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count, true}; + } + } + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +template +simdutf_constexpr23 simdutf_warn_unused full_result +base64_to_binary_details_safe_impl( + const char_type *input, size_t length, char *output, size_t outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = + (options == base64_options::base64_url_accept_garbage) || + (options == base64_options::base64_default_accept_garbage) || + (options == base64_options::base64_default_or_url_accept_garbage); + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + if (length == 0) { + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation, 0}; + } + return {SUCCESS, full_input_length, 0}; + } + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0 && !ignore_garbage) { + // additional checks + if ((r.output_count % 3 == 0) || + ((r.output_count % 3) + 1 + equalsigns != 4)) { + return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; + } + } + + // When is_partial(last_chunk_options) is true, we must either end with + // the end of the stream (beyond whitespace) or right after a non-ignorable + // character or at the very beginning of the stream. + // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + r.input_count < full_input_length) { + // First check if we can extend the input to the end of the stream + while (r.input_count < full_input_length && + base64_ignorable(*(input + r.input_count), options)) { + r.input_count++; + } + // If we are still not at the end of the stream, then we must backtrack + // to the last non-ignorable character. + if (r.input_count < full_input_length) { + while (r.input_count > 0 && + base64_ignorable(*(input + r.input_count - 1), options)) { + r.input_count--; + } + } + } + return r; +} + +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary(size_t length, base64_options options) noexcept { + // By default, we use padding if we are not using the URL variant. + // This is check with ((options & base64_url) == 0) which returns true if we + // are not using the URL variant. However, we also allow 'inversion' of the + // convention with the base64_reverse_padding option. If the + // base64_reverse_padding option is set, we use padding if we are using the + // URL variant, and we omit it if we are not using the URL variant. This is + // checked with + // ((options & base64_reverse_padding) == base64_reverse_padding). + bool use_padding = + ((options & base64_url) == 0) ^ + ((options & base64_reverse_padding) == base64_reverse_padding); + if (!use_padding) { + return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); + } + return (length + 2) / 3 * + 4; // We use padding to make the length a multiple of 4. } - #endif // SIMDUTF_SPAN -/** - * Given a valid UTF-16 string having a possibly truncated last character, - * this function checks the end of string. If the last character is truncated - * (or partial), then it returns a shorter length (shorter by 1 unit) so that - * the short UTF-16 strings only contain complete characters. If there is no - * truncated character, the original length is returned. - * - * This function assumes that the input string is valid UTF-16, but possibly - * truncated. We use the native endianness. - * - * @param input the UTF-16 string to process - * @param length the length of the string in bytes - * @return the length of the string in unit, possibly shorter by 1 unit - */ -simdutf_warn_unused size_t trim_partial_utf16(const char16_t *input, - size_t length); - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t -trim_partial_utf16(std::span valid_utf16_input) noexcept { - return trim_partial_utf16(valid_utf16_input.data(), valid_utf16_input.size()); +simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines(size_t length, base64_options options, + size_t line_length) noexcept { + if (length == 0) { + return 0; + } + size_t base64_length = + scalar::base64::base64_length_from_binary(length, options); + if (line_length < 4) { + line_length = 4; + } + size_t lines = + (base64_length + line_length - 1) / line_length; // number of lines + return base64_length + lines - 1; +} + +// Return the length of the prefix that contains count base64 characters. +// Thus, if count is 3, the function returns the length of the prefix +// that contains 3 base64 characters. +// The function returns (size_t)-1 if there is not enough base64 characters in +// the input. +template +simdutf_warn_unused size_t prefix_length(size_t count, + simdutf::base64_options options, + const char_type *input, + size_t length) noexcept { + size_t i = 0; + while (i < length && is_ignorable(input[i], options)) { + i++; + } + if (count == 0) { + return i; // duh! + } + for (; i < length; i++) { + if (is_ignorable(input[i], options)) { + continue; + } + // We have a base64 character or a padding character. + count--; + if (count == 0) { + return i + 1; + } + } + simdutf_log_assert(false, "You never get here"); + + return -1; // should never happen } - #endif // SIMDUTF_SPAN -#endif // SIMDUTF_FEATURE_UTF16 -#if SIMDUTF_FEATURE_BASE64 - #ifndef SIMDUTF_NEED_TRAILING_ZEROES - #define SIMDUTF_NEED_TRAILING_ZEROES 1 - #endif -// base64_options are used to specify the base64 encoding options. -// ASCII spaces are ' ', '\t', '\n', '\r', '\f' -// garbage characters are characters that are not part of the base64 alphabet -// nor ASCII spaces. -constexpr uint64_t base64_reverse_padding = - 2; /* modifier for base64_default and base64_url */ -enum base64_options : uint64_t { - base64_default = 0, /* standard base64 format (with padding) */ - base64_url = 1, /* base64url format (no padding) */ - base64_default_no_padding = - base64_default | - base64_reverse_padding, /* standard base64 format without padding */ - base64_url_with_padding = - base64_url | base64_reverse_padding, /* base64url with padding */ - base64_default_accept_garbage = - 4, /* standard base64 format accepting garbage characters, the input stops - with the first '=' if any */ - base64_url_accept_garbage = - 5, /* base64url format accepting garbage characters, the input stops with - the first '=' if any */ - base64_default_or_url = - 8, /* standard/base64url hybrid format (only meaningful for decoding!) */ - base64_default_or_url_accept_garbage = - 12, /* standard/base64url hybrid format accepting garbage characters - (only meaningful for decoding!), the input stops with the first '=' - if any */ -}; +} // namespace base64 +} // unnamed namespace +} // namespace scalar +} // namespace simdutf + +#endif +/* end file include/simdutf/scalar/base64.h */ + +namespace simdutf { - #if SIMDUTF_CPLUSPLUS17 inline std::string_view to_string(base64_options options) { switch (options) { case base64_default: @@ -4132,26 +10944,7 @@ inline std::string_view to_string(base64_options options) { } return ""; } - #endif // SIMDUTF_CPLUSPLUS17 - -// last_chunk_handling_options are used to specify the handling of the last -// chunk in base64 decoding. -// https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 -enum last_chunk_handling_options : uint64_t { - loose = 0, /* standard base64 format, decode partial final chunk */ - strict = 1, /* error when the last chunk is partial, 2 or 3 chars, and - unpadded, or non-zero bit padding */ - stop_before_partial = - 2, /* if the last chunk is partial, ignore it (no error) */ - only_full_chunks = - 3 /* only decode full blocks (4 base64 characters, no padding) */ -}; - -inline bool is_partial(last_chunk_handling_options options) { - return (options == stop_before_partial) || (options == only_full_chunks); -} - #if SIMDUTF_CPLUSPLUS17 inline std::string_view to_string(last_chunk_handling_options options) { switch (options) { case loose: @@ -4165,12 +10958,15 @@ inline std::string_view to_string(last_chunk_handling_options options) { } return ""; } - #endif /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less than - * the maximum length. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. * * @param input the base64 input to process * @param length the length of the base64 input in bytes @@ -4179,18 +10975,30 @@ inline std::string_view to_string(last_chunk_handling_options options) { simdutf_warn_unused size_t maximal_binary_length_from_base64(const char *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t maximal_binary_length_from_base64( const detail::input_span_of_byte_like auto &input) noexcept { - return maximal_binary_length_from_base64( - reinterpret_cast(input.data()), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64( + detail::constexpr_cast_ptr(input.data()), input.size()); + } else + #endif + { + return maximal_binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } } #endif // SIMDUTF_SPAN /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less than - * the maximum length. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. * * @param input the base64 input to process, in ASCII stored as 16-bit * units @@ -4200,9 +11008,82 @@ maximal_binary_length_from_base64( simdutf_warn_unused size_t maximal_binary_length_from_base64( const char16_t *input, size_t length) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t maximal_binary_length_from_base64(std::span input) noexcept { - return maximal_binary_length_from_base64(input.data(), input.size()); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::maximal_binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return maximal_binary_length_from_base64(input.data(), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the binary length from a base64 input. + * This function is useful for base64 inputs that may contain ASCII whitespaces + * (such as line breaks). For such inputs, the result is exact, and for any + * inputs the result can be used to size the output buffer passed to + * `base64_to_binary`. + * + * The function ignores whitespace and does not require padding characters + * ('='). + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return number of binary bytes + */ +simdutf_warn_unused size_t binary_length_from_base64(const char *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64( + const detail::input_span_of_byte_like auto &input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return binary_length_from_base64( + reinterpret_cast(input.data()), input.size()); + } +} + #endif // SIMDUTF_SPAN + +/** + * Compute the binary length from a base64 input. + * This function is useful for base64 inputs that may contain ASCII whitespaces + * (such as line breaks). For such inputs, the result is exact, and for any + * inputs the result can be used to size the output buffer passed to + * `base64_to_binary`. + * + * The function ignores whitespace and does not require padding characters + * ('='). + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return number of binary bytes + */ +simdutf_warn_unused size_t binary_length_from_base64(const char16_t *input, + size_t length) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_length_from_base64(std::span input) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::binary_length_from_base64(input.data(), + input.size()); + } else + #endif + { + return binary_length_from_base64(input.data(), input.size()); + } } #endif // SIMDUTF_SPAN @@ -4265,15 +11146,25 @@ simdutf_warn_unused result base64_to_binary( base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result base64_to_binary( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept { - return base64_to_binary(reinterpret_cast(input.data()), - input.size(), - reinterpret_cast(binary_output.data()), - options, last_chunk_options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(reinterpret_cast(input.data()), + input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } } #endif // SIMDUTF_SPAN @@ -4281,23 +11172,31 @@ simdutf_really_inline simdutf_warn_unused result base64_to_binary( * Provide the base64 length in bytes given the length of a binary input. * * @param length the length of the input in bytes + * @param options the base64 options to use (default: base64_default) * @return number of base64 bytes */ -simdutf_warn_unused size_t base64_length_from_binary( - size_t length, base64_options options = base64_default) noexcept; +inline simdutf_warn_unused simdutf_constexpr23 size_t base64_length_from_binary( + size_t length, base64_options options = base64_default) noexcept { + return scalar::base64::base64_length_from_binary(length, options); +} /** * Provide the base64 length in bytes given the length of a binary input, * taking into account line breaks. * * @param length the length of the input in bytes + * @param options the base64 options to use (default: base64_default) * @param line_length the length of lines, must be at least 4 (otherwise it is * interpreted as 4), * @return number of base64 bytes */ -simdutf_warn_unused size_t base64_length_from_binary_with_lines( +inline simdutf_warn_unused simdutf_constexpr23 size_t +base64_length_from_binary_with_lines( size_t length, base64_options options = base64_default, - size_t line_length = default_line_length) noexcept; + size_t line_length = default_line_length) noexcept { + return scalar::base64::base64_length_from_binary_with_lines(length, options, + line_length); +} /** * Convert a binary input to a base64 output. @@ -4323,13 +11222,21 @@ simdutf_warn_unused size_t base64_length_from_binary_with_lines( size_t binary_to_base64(const char *input, size_t length, char *output, base64_options options = base64_default) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t binary_to_base64(const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default) noexcept { - return binary_to_base64( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64( + binary_output.data(), input.data(), input.size(), options); + } else + #endif + { + return binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); + } } #endif // SIMDUTF_SPAN @@ -4362,14 +11269,23 @@ binary_to_base64_with_lines(const char *input, size_t length, char *output, size_t line_length = simdutf::default_line_length, base64_options options = base64_default) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t binary_to_base64_with_lines( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t +binary_to_base64_with_lines( const detail::input_span_of_byte_like auto &input, detail::output_span_of_byte_like auto &&binary_output, size_t line_length = simdutf::default_line_length, base64_options options = base64_default) noexcept { - return binary_to_base64_with_lines( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), line_length, options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::tail_encode_base64_impl( + binary_output.data(), input.data(), input.size(), options, line_length); + } else + #endif + { + return binary_to_base64_with_lines( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), line_length, options); + } } #endif // SIMDUTF_SPAN @@ -4409,29 +11325,188 @@ simdutf_really_inline simdutf_warn_unused size_t binary_to_base64_with_lines( * @param input the binary to process * @param length the length of the input in bytes * @param output the pointer to a buffer that can hold the conversion - * result (should be at least base64_length_from_binary(length) bytes long) + * result (should be at least base64_length_from_binary(length) bytes long) + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @return number of written bytes, will be equal to + * base64_length_from_binary(length, options) + */ +size_t +atomic_binary_to_base64(const char *input, size_t length, char *output, + base64_options options = base64_default) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused size_t +atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default) noexcept { + return atomic_binary_to_base64( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options); +} + #endif // SIMDUTF_SPAN + #endif // SIMDUTF_ATOMIC_REF + +/** + * Convert a base64 input to a binary output. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * This function will fail in case of invalid input. When last_chunk_options = + * loose, there are two possible reasons for failure: the input contains a + * number of base64 characters that when divided by 4, leaves a single remainder + * character (BASE64_INPUT_REMAINDER), or the input contains a character that is + * not a valid base64 character (INVALID_BASE64_CHARACTER). + * + * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the + * input where the invalid character was found. When the error is + * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * + * The default option (simdutf::base64_default) expects the characters `+` and + * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the + * characters `-` and `_` as part of its alphabet. + * + * The padding (`=`) is validated if present. There may be at most two padding + * characters at the end of the input. If there are any padding characters, the + * total number of characters (excluding spaces but including padding + * characters) must be divisible by four. + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail + * to provide that much space, the function may cause a buffer overflow. + * + * Advanced users may want to tailor how the last chunk is handled. By default, + * we use a loose (forgiving) approach but we also support a strict approach + * as well as a stop_before_partial approach, as per the following proposal: + * + * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * + * @param input the base64 string to process, in ASCII stored as 16-bit + * units + * @param length the length of the string in 16-bit units + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). + * @param options the base64 options to use, can be base64_default or + * base64_url, is base64_default by default. + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a result pair struct (of type simdutf::result containing the two + * fields error and count) with an error code and position of the + * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number + * of bytes written if successful. + */ +simdutf_warn_unused result +base64_to_binary(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 result +base64_to_binary( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary(input.data(), input.size(), + reinterpret_cast(binary_output.data()), + options, last_chunk_options); + } +} + #endif // SIMDUTF_SPAN + +/** + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. + * + * This function follows the WHATWG forgiving-base64 format, which means that it + * will ignore any ASCII spaces in the input. You may provide a padded input + * (with one or two equal signs at the end) or an unpadded input (without any + * equal signs at the end). + * + * See https://infra.spec.whatwg.org/#forgiving-base64-decode + * + * Unlike base64_to_binary, this function returns a full_result with both + * input_count and output_count, so you always know how much input was consumed + * and how much output was written. There are three cases where the input may + * not be fully consumed: + * + * 1. stop_before_partial: When last_chunk_options is set to + * stop_before_partial, any incomplete 4-character group at the end of the + * input is left unconsumed. This is useful for streaming/chunked decoding + * where you can carry over the unconsumed input to the next chunk. + * + * 2. INVALID_BASE64_CHARACTER: The input contains a character that is not a + * valid base64 character. In this case, input_count indicates where the + * invalid character was found. + * + * 3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input + * contains a number of base64 characters that, when divided by 4, leaves + * a single remainder character (which cannot encode any bytes). + * + * You should call this function with a buffer that is at least + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. + * + * @param input the base64 string to process + * @param length the length of the string in bytes + * @param output the pointer to a buffer that can hold the conversion + * result (should be at least maximal_binary_length_from_base64(input, length) + * bytes long). * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. - * @return number of written bytes, will be equal to - * base64_length_from_binary(length, options) - */ -size_t -atomic_binary_to_base64(const char *input, size_t length, char *output, - base64_options options = base64_default) noexcept; - #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused size_t -atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, - detail::output_span_of_byte_like auto &&binary_output, - base64_options options = base64_default) noexcept { - return atomic_binary_to_base64( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), options); + * @param last_chunk_options the last chunk handling options, + * last_chunk_handling_options::loose by default + * but can also be last_chunk_handling_options::strict or + * last_chunk_handling_options::stop_before_partial. + * @return a full_result struct (of type simdutf::full_result containing the + * three fields error, input_count and output_count). + */ +simdutf_warn_unused full_result +base64_to_binary_details(const char *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; + #if SIMDUTF_SPAN +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose) noexcept { + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary_details( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), options, + last_chunk_options); + } } - #endif // SIMDUTF_SPAN - #endif // SIMDUTF_ATOMIC_REF + #endif // SIMDUTF_SPAN /** - * Convert a base64 input to a binary output. + * Convert a base64 input to a binary output while returning more details + * than base64_to_binary. * * This function follows the WHATWG forgiving-base64 format, which means that it * will ignore any ASCII spaces in the input. You may provide a padded input @@ -4440,34 +11515,27 @@ atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, * * See https://infra.spec.whatwg.org/#forgiving-base64-decode * - * This function will fail in case of invalid input. When last_chunk_options = - * loose, there are two possible reasons for failure: the input contains a - * number of base64 characters that when divided by 4, leaves a single remainder - * character (BASE64_INPUT_REMAINDER), or the input contains a character that is - * not a valid base64 character (INVALID_BASE64_CHARACTER). + * Unlike base64_to_binary, this function returns a full_result with both + * input_count and output_count, so you always know how much input was consumed + * and how much output was written. There are three cases where the input may + * not be fully consumed: * - * When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the - * input where the invalid character was found. When the error is - * BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded. + * 1. stop_before_partial: When last_chunk_options is set to + * stop_before_partial, any incomplete 4-character group at the end of the + * input is left unconsumed. This is useful for streaming/chunked decoding + * where you can carry over the unconsumed input to the next chunk. * - * The default option (simdutf::base64_default) expects the characters `+` and - * `/` as part of its alphabet. The URL option (simdutf::base64_url) expects the - * characters `-` and `_` as part of its alphabet. + * 2. INVALID_BASE64_CHARACTER: The input contains a character that is not a + * valid base64 character. In this case, input_count indicates where the + * invalid character was found. * - * The padding (`=`) is validated if present. There may be at most two padding - * characters at the end of the input. If there are any padding characters, the - * total number of characters (excluding spaces but including padding - * characters) must be divisible by four. + * 3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input + * contains a number of base64 characters that, when divided by 4, leaves + * a single remainder character (which cannot encode any bytes). * * You should call this function with a buffer that is at least - * maximal_binary_length_from_base64(input, length) bytes long. If you fail - * to provide that much space, the function may cause a buffer overflow. - * - * Advanced users may want to tailor how the last chunk is handled. By default, - * we use a loose (forgiving) approach but we also support a strict approach - * as well as a stop_before_partial approach, as per the following proposal: - * - * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + * maximal_binary_length_from_base64(input, length) bytes long. If you fail to + * provide that much space, the function may cause a buffer overflow. * * @param input the base64 string to process, in ASCII stored as 16-bit * units @@ -4481,43 +11549,56 @@ atomic_binary_to_base64(const detail::input_span_of_byte_like auto &input, * last_chunk_handling_options::loose by default * but can also be last_chunk_handling_options::strict or * last_chunk_handling_options::stop_before_partial. - * @return a result pair struct (of type simdutf::result containing the two - * fields error and count) with an error code and position of the - * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number - * of bytes written if successful. - */ -simdutf_warn_unused result -base64_to_binary(const char16_t *input, size_t length, char *output, - base64_options options = base64_default, - last_chunk_handling_options last_chunk_options = - last_chunk_handling_options::loose) noexcept; + * @return a full_result struct (of type simdutf::full_result containing the + * three fields error, input_count and output_count). + */ +simdutf_warn_unused full_result +base64_to_binary_details(const char16_t *input, size_t length, char *output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = + last_chunk_handling_options::loose) noexcept; #if SIMDUTF_SPAN -simdutf_really_inline simdutf_warn_unused result base64_to_binary( +simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 full_result +base64_to_binary_details( std::span input, detail::output_span_of_byte_like auto &&binary_output, base64_options options = base64_default, last_chunk_handling_options last_chunk_options = loose) noexcept { - return base64_to_binary(input.data(), input.size(), - reinterpret_cast(binary_output.data()), - options, last_chunk_options); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + return scalar::base64::base64_to_binary_details_impl( + input.data(), input.size(), binary_output.data(), options, + last_chunk_options); + } else + #endif + { + return base64_to_binary_details( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), options, + last_chunk_options); + } } #endif // SIMDUTF_SPAN /** - * Check if a character is an ignorabl base64 character. + * Check if a character is an ignorable base64 character. * Checking a large input, character by character, is not computationally * efficient. * * @param input the character to check * @param options the base64 options to use, is base64_default by default. - * @return true if the character is an ignorablee base64 character, false + * @return true if the character is an ignorable base64 character, false * otherwise. */ -simdutf_warn_unused bool -base64_ignorable(char input, base64_options options = base64_default) noexcept; -simdutf_warn_unused bool +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_ignorable(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_ignorable(char16_t input, - base64_options options = base64_default) noexcept; + base64_options options = base64_default) noexcept { + return scalar::base64::is_ignorable(input, options); +} /** * Check if a character is a valid base64 character. @@ -4530,10 +11611,14 @@ base64_ignorable(char16_t input, * @param options the base64 options to use, is base64_default by default. * @return true if the character is a base64 character, false otherwise. */ -simdutf_warn_unused bool -base64_valid(char input, base64_options options = base64_default) noexcept; -simdutf_warn_unused bool -base64_valid(char16_t input, base64_options options = base64_default) noexcept; +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool +base64_valid(char16_t input, base64_options options = base64_default) noexcept { + return scalar::base64::is_base64(input, options); +} /** * Check if a character is a valid base64 character or the padding character @@ -4544,12 +11629,16 @@ base64_valid(char16_t input, base64_options options = base64_default) noexcept; * @param options the base64 options to use, is base64_default by default. * @return true if the character is a base64 character, false otherwise. */ -simdutf_warn_unused bool +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_valid_or_padding(char input, - base64_options options = base64_default) noexcept; -simdutf_warn_unused bool + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} +simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool base64_valid_or_padding(char16_t input, - base64_options options = base64_default) noexcept; + base64_options options = base64_default) noexcept { + return scalar::base64::is_base64_or_padding(input, options); +} /** * Convert a base64 input to a binary output. @@ -4596,6 +11685,10 @@ base64_valid_or_padding(char16_t input, * * https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 * + * The base64_to_binary_safe function has negligible overhead compared with + * base64_to_binary in the absence of ignorable characters; however, on short + * inputs containing ignorable characters, it can be up to three times slower. + * * @param input the base64 string to process, in ASCII stored as 8-bit * or 16-bit units * @param length the length of the string in 8-bit or 16-bit units. @@ -4624,25 +11717,7 @@ base64_to_binary_safe(const char *input, size_t length, char *output, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose, bool decode_up_to_bad_char = false) noexcept; - #if SIMDUTF_SPAN -/** - * @brief span overload - * @return a tuple of result and outlen - */ -simdutf_really_inline simdutf_warn_unused std::tuple -base64_to_binary_safe(const detail::input_span_of_byte_like auto &input, - detail::output_span_of_byte_like auto &&binary_output, - base64_options options = base64_default, - last_chunk_handling_options last_chunk_options = loose, - bool decode_up_to_bad_char = false) noexcept { - size_t outlen = binary_output.size(); - auto r = base64_to_binary_safe( - reinterpret_cast(input.data()), input.size(), - reinterpret_cast(binary_output.data()), outlen, options, - last_chunk_options, decode_up_to_bad_char); - return {r, outlen}; -} - #endif // SIMDUTF_SPAN +// the span overload has moved to the bottom of the file simdutf_warn_unused result base64_to_binary_safe(const char16_t *input, size_t length, char *output, @@ -4650,25 +11725,7 @@ base64_to_binary_safe(const char16_t *input, size_t length, char *output, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose, bool decode_up_to_bad_char = false) noexcept; - #if SIMDUTF_SPAN -/** - * @brief span overload - * @return a tuple of result and outlen - */ -simdutf_really_inline simdutf_warn_unused std::tuple -base64_to_binary_safe(std::span input, - detail::output_span_of_byte_like auto &&binary_output, - base64_options options = base64_default, - last_chunk_handling_options last_chunk_options = loose, - bool decode_up_to_bad_char = false) noexcept { - size_t outlen = binary_output.size(); - auto r = base64_to_binary_safe(input.data(), input.size(), - reinterpret_cast(binary_output.data()), - outlen, options, last_chunk_options, - decode_up_to_bad_char); - return {r, outlen}; -} - #endif // SIMDUTF_SPAN + // span overload moved to bottom of file #if SIMDUTF_ATOMIC_REF /** @@ -4762,20 +11819,6 @@ atomic_base64_to_binary_safe( #endif // SIMDUTF_SPAN #endif // SIMDUTF_ATOMIC_REF -/** - * Find the first occurrence of a character in a string. If the character is - * not found, return a pointer to the end of the string. - * @param start the start of the string - * @param end the end of the string - * @param character the character to find - * @return a pointer to the first occurrence of the character in the string, - * or a pointer to the end of the string if the character is not found. - * - */ -simdutf_warn_unused const char *find(const char *start, const char *end, - char character) noexcept; -simdutf_warn_unused const char16_t * -find(const char16_t *start, const char16_t *end, char16_t character) noexcept; #endif // SIMDUTF_FEATURE_BASE64 /** @@ -4796,7 +11839,7 @@ class implementation { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string name() const { return std::string(_name); } + virtual std::string_view name() const noexcept { return _name; } /** * The description of this implementation. @@ -4807,7 +11850,7 @@ class implementation { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string description() const { return std::string(_description); } + virtual std::string_view description() const noexcept { return _description; } /** * The instruction sets this implementation is compiled against @@ -5110,7 +12153,7 @@ class implementation { * * @param input the Latin1 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if conversion is not possible */ simdutf_warn_unused virtual size_t @@ -5124,7 +12167,7 @@ class implementation { * * @param input the Latin1 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if conversion is not possible */ simdutf_warn_unused virtual size_t @@ -5218,7 +12261,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ @@ -5234,7 +12277,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ @@ -5251,7 +12294,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of code units validated @@ -5270,7 +12313,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf16_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of code units validated @@ -5334,7 +12377,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf32_buffer the pointer to buffer that can hold conversion result + * @param utf32_output the pointer to buffer that can hold conversion result * @return the number of written char16_t; 0 if the input was not valid UTF-8 * string */ @@ -5350,7 +12393,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf32_buffer the pointer to buffer that can hold conversion result + * @param utf32_output the pointer to buffer that can hold conversion result * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in code units) if any, or the number of char32_t written if @@ -5399,7 +12442,7 @@ class implementation { * * @param input the UTF-8 string to convert * @param length the length of the string in bytes - * @param utf16_buffer the pointer to buffer that can hold conversion result + * @param utf32_buffer the pointer to buffer that can hold conversion result * @return the number of written char32_t */ simdutf_warn_unused virtual size_t @@ -5662,6 +12705,44 @@ class implementation { convert_utf16be_to_utf8_with_errors(const char16_t *input, size_t length, char *utf8_buffer) const noexcept = 0; + /** + * Convert possibly broken UTF-16LE string into UTF-8 string, replacing + * unpaired surrogates with the Unicode replacement character U+FFFD. + * + * This function always succeeds: unpaired surrogates are replaced with + * U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD). + * + * This function is not BOM-aware. + * + * @param input the UTF-16LE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units + */ + simdutf_warn_unused virtual size_t convert_utf16le_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + + /** + * Convert possibly broken UTF-16BE string into UTF-8 string, replacing + * unpaired surrogates with the Unicode replacement character U+FFFD. + * + * This function always succeeds: unpaired surrogates are replaced with + * U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD). + * + * This function is not BOM-aware. + * + * @param input the UTF-16BE string to convert + * @param length the length of the string in 2-byte code units + * (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written code units + */ + simdutf_warn_unused virtual size_t convert_utf16be_to_utf8_with_replacement( + const char16_t *input, size_t length, + char *utf8_buffer) const noexcept = 0; + /** * Convert valid UTF-16LE string into UTF-8 string. * @@ -5994,7 +13075,6 @@ class implementation { * format. * * - * @param input the UTF-16 string to convert * @param length the length of the string in 2-byte code units * (char16_t) * @return the number of bytes required to encode the UTF-16 string as Latin1 @@ -6219,7 +13299,6 @@ class implementation { * * This function is not BOM-aware. * - * @param input the UTF-16LE string to convert * @param length the length of the string in 2-byte code units * (char16_t) * @return the number of bytes required to encode the UTF-16LE string as @@ -6366,9 +13445,12 @@ class implementation { #if SIMDUTF_FEATURE_BASE64 /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less - * than the maximum length. It is acceptable to pass invalid base64 strings - * but in such cases the result is implementation defined. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read.. * * @param input the base64 input to process * @param length the length of the base64 input in bytes @@ -6379,9 +13461,12 @@ class implementation { /** * Provide the maximal binary length in bytes given the base64 input. - * In general, if the input contains ASCII spaces, the result will be less - * than the maximum length. It is acceptable to pass invalid base64 strings - * but in such cases the result is implementation defined. + * As long as the input does not contain ignorable characters (e.g., ASCII + * spaces or linefeed characters), the result is exact. In particular, the + * function checks for padding characters. + * + * The function is fast (constant time). It checks up to two characters at + * the end of the string. The input is not otherwise validated or read. * * @param input the base64 input to process, in ASCII stored as 16-bit * units @@ -6391,6 +13476,38 @@ class implementation { simdutf_warn_unused size_t maximal_binary_length_from_base64( const char16_t *input, size_t length) const noexcept; + /** + * Compute the binary length from a base64 input with ASCII spaces. + * This function is useful for well-formed base64 inputs that may contain + * ASCII spaces (such as line breaks). For such inputs, the result is exact. + * + * The function counts non-whitespace characters (ASCII value > 0x20) and + * subtracts padding characters ('=') found at the end. + * + * @param input the base64 input to process + * @param length the length of the base64 input in bytes + * @return number of binary bytes + */ + simdutf_warn_unused virtual size_t + binary_length_from_base64(const char *input, size_t length) const noexcept; + + /** + * Compute the binary length from a base64 input with ASCII spaces. + * This function is useful for well-formed base64 inputs that may contain + * ASCII spaces (such as line breaks). For such inputs, the result is exact. + * + * The function counts non-whitespace characters (ASCII value > 0x20) and + * subtracts padding characters ('=') found at the end. + * + * @param input the base64 input to process, in ASCII stored as 16-bit + * units + * @param length the length of the base64 input in 16-bit units + * @return number of binary bytes + */ + simdutf_warn_unused virtual size_t + binary_length_from_base64(const char16_t *input, + size_t length) const noexcept; + /** * Convert a base64 input to a binary output. * @@ -6418,6 +13535,7 @@ class implementation { * bytes long). * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. + * @param last_chunk_options the handling of the last chunk (default: loose) * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and either position of the error * (in the input in bytes) if any, or the number of bytes written if @@ -6457,6 +13575,7 @@ class implementation { * bytes long). * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. + * @param last_chunk_options the handling of the last chunk (default: loose) * @return a full_result pair struct (of type simdutf::result containing the * three fields error, input_count and output_count). */ @@ -6465,6 +13584,7 @@ class implementation { base64_options options = base64_default, last_chunk_handling_options last_chunk_options = last_chunk_handling_options::loose) const noexcept = 0; + /** * Convert a base64 input to a binary output. * @@ -6493,6 +13613,7 @@ class implementation { * bytes long). * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. + * @param last_chunk_options the handling of the last chunk (default: loose) * @return a result pair struct (of type simdutf::result containing the two * fields error and count) with an error code and position of the * INVALID_BASE64_CHARACTER error (in the input in units) if any, or the @@ -6532,6 +13653,7 @@ class implementation { * bytes long). * @param options the base64 options to use, can be base64_default or * base64_url, is base64_default by default. + * @param last_chunk_options the handling of the last chunk (default: loose) * @return a full_result pair struct (of type simdutf::result containing the * three fields error, input_count and output_count). */ @@ -6606,6 +13728,7 @@ class implementation { const char *input, size_t length, char *output, size_t line_length = simdutf::default_line_length, base64_options options = base64_default) const noexcept = 0; + /** * Find the first occurrence of a character in a string. If the character is * not found, return a pointer to the end of the string. @@ -6633,7 +13756,7 @@ class implementation { struct TestProcedure { // display name - std::string name; + std::string_view name; // procedure should return whether given test pass or not void (*procedure)(const implementation &); @@ -6644,7 +13767,12 @@ class implementation { protected: /** @private Construct an implementation with the given name and description. - * For subclasses. */ + * For subclasses. + * @param name the name of this implementation + * @param description a description of this implementation + * @param required_instruction_sets the instruction sets this implementation + * requires + */ simdutf_really_inline implementation(const char *name, const char *description, uint32_t required_instruction_sets) @@ -6701,7 +13829,7 @@ class available_implementation_list { * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" * @return the implementation, or nullptr if the parse failed. */ - const implementation *operator[](const std::string &name) const noexcept { + const implementation *operator[](std::string_view name) const noexcept { for (const implementation *impl : *this) { if (impl->name() == name) { return impl; @@ -6787,6 +13915,321 @@ get_active_implementation(); } // namespace simdutf +#if SIMDUTF_FEATURE_BASE64 + // this header is not part of the public api +/* begin file include/simdutf/base64_implementation.h */ +#ifndef SIMDUTF_BASE64_IMPLEMENTATION_H +#define SIMDUTF_BASE64_IMPLEMENTATION_H + +// this is not part of the public api + +#include // for is_same + +namespace simdutf { + +template +simdutf_warn_unused simdutf_constexpr23 result slow_base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_options) noexcept { + const bool ignore_garbage = (options & base64_default_accept_garbage) != 0; + auto ri = simdutf::scalar::base64::find_end(input, length, options); + size_t equallocation = ri.equallocation; + size_t equalsigns = ri.equalsigns; + length = ri.srclen; + size_t full_input_length = ri.full_input_length; + (void)full_input_length; + if (length == 0) { + outlen = 0; + if (!ignore_garbage && equalsigns > 0) { + return {INVALID_BASE64_CHARACTER, equallocation}; + } + return {SUCCESS, 0}; + } + + // The parameters of base64_tail_decode_safe are: + // - dst: the output buffer + // - outlen: the size of the output buffer + // - srcr: the input buffer + // - length: the size of the input buffer + // - padded_characters: the number of padding characters + // - options: the options for the base64 decoder + // - last_chunk_options: the options for the last chunk + // The function will return the number of bytes written to the output buffer + // and the number of bytes read from the input buffer. + // The function will also return an error code if the input buffer is not + // valid base64. + full_result r = scalar::base64::base64_tail_decode_safe( + output, outlen, input, length, equalsigns, options, last_chunk_options); + r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, + full_input_length, last_chunk_options); + outlen = r.output_count; + if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && + equalsigns > 0) { + // additional checks + if ((outlen % 3 == 0) || ((outlen % 3) + 1 + equalsigns != 4)) { + r.error = error_code::INVALID_BASE64_CHARACTER; + } + } + return {r.error, r.input_count}; // we cannot return r itself because it gets + // converted to error/output_count +} + +template +simdutf_warn_unused simdutf_constexpr23 result base64_to_binary_safe_impl( + const chartype *input, size_t length, char *output, size_t &outlen, + base64_options options, + last_chunk_handling_options last_chunk_handling_options, + bool decode_up_to_bad_char) noexcept { + static_assert(std::is_same::value || + std::is_same::value, + "Only char and char16_t are supported."); + size_t remaining_input_length = length; + size_t remaining_output_length = outlen; + size_t input_position = 0; + size_t output_position = 0; + + // We also do a first pass using the fast path to decode as much as possible + size_t safe_input = detail::min( + remaining_input_length, + base64_length_from_binary(remaining_output_length / 3 * 3, options)); + bool done_with_partial = (safe_input == remaining_input_length); + simdutf::full_result r; + +#if SIMDUTF_CPLUSPLUS23 + if consteval { + r = scalar::base64::base64_to_binary_details_impl( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } else +#endif + { + r = get_active_implementation()->base64_to_binary_details( + input + input_position, safe_input, output + output_position, options, + done_with_partial + ? last_chunk_handling_options + : simdutf::last_chunk_handling_options::only_full_chunks); + } + simdutf_log_assert(r.input_count <= safe_input, + "You should not read more than safe_input"); + simdutf_log_assert(r.output_count <= remaining_output_length, + "You should not write more than remaining_output_length"); + // Technically redundant, but we want to be explicit about it. + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + + if (done_with_partial) { + // We are done. We have decoded everything. + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; + } + // We have decoded some data, but we still have some data to decode. + // We need to decode the rest of the input buffer. + r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( + input + input_position, remaining_input_length, output + output_position, + remaining_output_length, options, last_chunk_handling_options); + input_position += r.input_count; + output_position += r.output_count; + remaining_input_length -= r.input_count; + remaining_output_length -= r.output_count; + + if (r.error != simdutf::error_code::SUCCESS) { + // There is an error. We return. + if (decode_up_to_bad_char && + r.error == error_code::INVALID_BASE64_CHARACTER) { + return slow_base64_to_binary_safe_impl( + input, length, output, outlen, options, last_chunk_handling_options); + } + outlen = output_position; + return {r.error, input_position}; + } + if (input_position < length) { + // We cannot process the entire input in one go, so we need to + // process it in two steps: first the fast path, then the slow path. + // In some cases, the processing might 'eat up' trailing ignorable + // characters in the fast path, but that can be a problem. + // suppose we have just white space followed by a single base64 character. + // If we first process the white space with the fast path, it will + // eat all of it. But, by the JavaScript standard, we should consume + // no character. See + // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 + while (input_position > 0 && + base64_ignorable(input[input_position - 1], options)) { + input_position--; + } + } + outlen = output_position; + return {simdutf::error_code::SUCCESS, input_position}; +} + +} // namespace simdutf +#endif // SIMDUTF_BASE64_IMPLEMENTATION_H +/* end file include/simdutf/base64_implementation.h */ + +namespace simdutf { + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_constexpr23 simdutf_warn_unused std::tuple + base64_to_binary_safe( + const detail::input_span_of_byte_like auto &input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + using CInput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "input of type char"); + using COutput = std::decay_t; + static_assert(std::is_same_v, + "sorry, the constexpr implementation is for now limited to " + "output of type char"); + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe_impl( + reinterpret_cast(input.data()), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + + #if SIMDUTF_SPAN +/** + * @brief span overload + * @return a tuple of result and outlen + */ +simdutf_really_inline + simdutf_warn_unused simdutf_constexpr23 std::tuple + base64_to_binary_safe( + std::span input, + detail::output_span_of_byte_like auto &&binary_output, + base64_options options = base64_default, + last_chunk_handling_options last_chunk_options = loose, + bool decode_up_to_bad_char = false) noexcept { + size_t outlen = binary_output.size(); + #if SIMDUTF_CPLUSPLUS23 + if consteval { + auto r = base64_to_binary_safe_impl( + input.data(), input.size(), binary_output.data(), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } else + #endif + { + auto r = base64_to_binary_safe( + input.data(), input.size(), + reinterpret_cast(binary_output.data()), outlen, options, + last_chunk_options, decode_up_to_bad_char); + return {r, outlen}; + } +} + #endif // SIMDUTF_SPAN + + #endif // SIMDUTF_SPAN +} // namespace simdutf + +#endif // SIMDUTF_FEATURE_BASE64 + +#if SIMDUTF_CPLUSPLUS23 && SIMDUTF_FEATURE_BASE64 + +namespace simdutf { +namespace literals { + +namespace detail { + +// the detail namespace is not part of the public api + +template struct base64_literal_helper { + std::array storage{}; + static constexpr std::size_t size() noexcept { return N - 1; } + consteval base64_literal_helper(const char (&str)[N]) { + for (std::size_t i = 0; i < size(); i++) { + storage[i] = str[i]; + } + } +}; + +template struct base64_decode_result { + static constexpr std::size_t max_out = (InputLen + 3) / 4 * 3; + std::array buffer{}; + std::size_t output_count{}; +}; + +template +consteval auto base64_decode_literal(const char *str) { + base64_decode_result result{}; + auto r = scalar::base64::base64_to_binary_details_impl( + str, InputLen, result.buffer.data(), base64_default, loose); + if (r.error != error_code::SUCCESS) { + #if __cpp_lib_unreachable >= 202202L + std::unreachable(); // invalid base64 input in _base64 literal + #else + // workaround for older stdlib + throw "invalid base64 input in _base64 literal"; + #endif + } + result.output_count = r.output_count; + return result; +} + +template consteval auto base64_make_array() { + constexpr auto decoded = base64_decode_literal(a.storage.data()); + std::array ret{}; + for (std::size_t i = 0; i < decoded.output_count; i++) { + ret[i] = decoded.buffer[i]; + } + return ret; +} + +} // namespace detail + +/** + * User-defined literal for compile-time base64 decoding. + * + * Usage: + * using namespace simdutf::literals; + * constexpr auto decoded = "SGVsbG8gV29ybGQh"_base64; + * // decoded is a std::array containing "Hello World!" + * + * The input must be valid base64. Whitepace is allowed and ignored. + * A compilation error occurs if the input is invalid. + */ +template consteval auto operator""_base64() { + return detail::base64_make_array(); +} + +} // namespace literals +} // namespace simdutf + +#endif // SIMDUTF_CPLUSPLUS23 && SIMDUTF_FEATURE_BASE64 + #endif // SIMDUTF_IMPLEMENTATION_H /* end file include/simdutf/implementation.h */ From 3da329944203cf17d81f14b41d0c37e53347f20c Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 18 Aug 2026 13:31:06 -0400 Subject: [PATCH 2/2] src: use convert_utf16_to_utf8_with_replacement StringBytes::Write() validated UTF-16 and, on unpaired surrogates, copied through to_well_formed_utf16() before converting. simdutf 9.1.0 can replace unpaired surrogates during the convert, so drop the extra pass. Signed-off-by: Daniel Lemire --- src/string_bytes.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/string_bytes.cc b/src/string_bytes.cc index 88b499fb4289..19ca9de21bf9 100644 --- a/src/string_bytes.cc +++ b/src/string_bytes.cc @@ -322,20 +322,16 @@ size_t StringBytes::Write(Isolate* isolate, const char16_t* data = reinterpret_cast(input_view.data16()); const size_t length = input_view.length(); - MaybeStackBuffer well_formed; - if (!simdutf::validate_utf16(data, length)) { - // Unpaired surrogates: encode a copy in which each of them has been - // replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces. - well_formed.AllocateSufficientStorage(length); - simdutf::to_well_formed_utf16(data, length, well_formed.out()); - data = well_formed.out(); - } // A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so // 3 * length is what StorageSize() hands most callers; only compute // the exact length when the buffer is smaller than that. if (buflen >= 3 * length || - buflen >= simdutf::utf8_length_from_utf16(data, length)) { - nbytes = simdutf::convert_utf16_to_utf8(data, length, buf); + buflen >= + simdutf::utf8_length_from_utf16_with_replacement(data, length) + .count) { + // Unpaired surrogates become U+FFFD, matching kReplaceInvalidUtf8. + nbytes = simdutf::convert_utf16_to_utf8_with_replacement( + data, length, buf); } else { // Does not fit: let V8 truncate at a character boundary. nbytes = str->WriteUtf8V2(