From f976362e83f460efa2c4a757a0a671881acba515 Mon Sep 17 00:00:00 2001 From: Benjamin Brock Date: Mon, 3 Aug 2026 00:02:23 +0000 Subject: [PATCH 1/3] Store SSMC text so its MATLAB class survives the round trip A text aux component was written as a variable-length HDF5 string dataset whatever it was, and binsparse_to_ssmc_problem guessed the class to rebuild from the matrix id: use_cellstr = isfield(Problem, 'id') && Problem.id > 2776; That rule is wrong in both directions. Across the collection there are 247 text aux components: 239 char matrices, ten of which have id > 2776, and eight cellstr, all of which have id <= 2776. So every real cellstr came back as a char matrix and every SNAP char matrix came back as a cellstr, and isequal failed for 13 matrices. The id was never what made this work for MM and RB either. sswrite only sends a cellstr to a single .txt when sscellstring(X) && id > 2776, which no matrix in the collection satisfies; every real cellstr takes the generic iscell branch and is written one element per file, and it is that structure, not the heuristic, that round trips. Let the HDF5 string datatype record the class instead, so a file says what it holds and no side channel is needed: fixed-length (H5T_STR_NULLPAD) <-> char matrix, m-by-n variable-length (H5T_VARIABLE) <-> cellstr, m-by-1 A char matrix is rectangular, so each element of a fixed-length dataset holds one complete row including the blanks MATLAB pads it with, and the datatype size carries the width. Nothing is deblanked on either side, which retires strip_text_rows and its trick of keeping padding on the widest row so that char() could recover the width. A cellstr is ragged, so it stays variable-length and needs no padding convention. Padding is NUL rather than blank because HDF5 sizes a fixed-length string in bytes while MATLAB sizes a char matrix in characters. The two agree only for ASCII; blank padding would widen any row holding a multi-byte character. Stripping trailing NULs recovers a row's exact characters whatever it contains. Trailing blanks are stripped as well when a foreign file declares H5T_STR_SPACEPAD, the Fortran convention. Text now converts through matlab_bsp_strings.h rather than mxArrayToString, which goes by way of the local code page and replaces what it cannot represent with SUB (0x1A). That silently mangled every non-ASCII aux row, 65,409 of them in Pajek/IMDB alone. The same fix applies to the JSON descriptor in binsparse_write.c, where it was corrupting Problem.notes for SNAP/wiki-RfA (en dash) and Pajek/Journals (replacement character). Text datasets are also chunked and deflated now. A variable-length payload lives on the global heap, which no dataset filter reaches, and costs a 16-byte descriptor per element; Pajek/IMDB colname alone took 27.6 MB of a 58.2 MB file. The whole file is now 18.7 MB. Every one of the 341 locally cached matrices round trips through sswrite and ssread with isequal true, including all 13 that used to fail. Co-Authored-By: Claude Opus 5 (1M context) --- bindings/matlab/Contents.m | 1 + .../matlab/binsparse_build_matlab_bindings.m | 3 +- .../matlab/binsparse_build_octave_bindings.m | 3 +- .../matlab/binsparse_read_string_dataset.c | 276 ++++++++++++++++ bindings/matlab/binsparse_to_ssmc_problem.m | 52 +-- bindings/matlab/binsparse_write.c | 10 +- .../matlab/binsparse_write_ssmc_problem.m | 26 +- .../matlab/binsparse_write_string_dataset.c | 217 ++++++++---- bindings/matlab/compile_octave.sh | 2 +- bindings/matlab/matlab_bsp_strings.h | 311 ++++++++++++++++++ .../matlab/test_binsparse_string_dataset.m | 123 +++++++ .../matlab/test_binsparse_to_ssmc_problem.m | 19 +- .../test_binsparse_write_ssmc_problem.m | 52 +-- 13 files changed, 962 insertions(+), 133 deletions(-) create mode 100644 bindings/matlab/binsparse_read_string_dataset.c create mode 100644 bindings/matlab/matlab_bsp_strings.h create mode 100644 bindings/matlab/test_binsparse_string_dataset.m diff --git a/bindings/matlab/Contents.m b/bindings/matlab/Contents.m index f44becc..c785ccc 100644 --- a/bindings/matlab/Contents.m +++ b/bindings/matlab/Contents.m @@ -6,6 +6,7 @@ % binsparse_from_ssmc - convert SSMC A+Zeros to a Binsparse matrix struct % binsparse_minimize_types - minimize value/index types in a Binsparse struct % binsparse_write_string_dataset - write an HDF5 UTF-8 string dataset +% binsparse_read_string_dataset - read an HDF5 UTF-8 string dataset % % MATLAB helpers: % binsparse_write_ssmc_problem - write an SSMC Problem to a Binsparse file diff --git a/bindings/matlab/binsparse_build_matlab_bindings.m b/bindings/matlab/binsparse_build_matlab_bindings.m index 1bed03f..701d837 100644 --- a/bindings/matlab/binsparse_build_matlab_bindings.m +++ b/bindings/matlab/binsparse_build_matlab_bindings.m @@ -107,7 +107,8 @@ function compile_mex_functions(paths, verbose) % List of MEX functions to compile mex_files = {'binsparse_read.c', 'binsparse_write.c', ... 'binsparse_from_ssmc.c', 'binsparse_minimize_types.c', ... - 'binsparse_write_string_dataset.c'}; + 'binsparse_write_string_dataset.c', ... + 'binsparse_read_string_dataset.c'}; fprintf('Compiling MEX functions...\n'); failed_files = {}; diff --git a/bindings/matlab/binsparse_build_octave_bindings.m b/bindings/matlab/binsparse_build_octave_bindings.m index d376b66..c9ba44d 100644 --- a/bindings/matlab/binsparse_build_octave_bindings.m +++ b/bindings/matlab/binsparse_build_octave_bindings.m @@ -120,7 +120,8 @@ function compile_octave_functions(paths, verbose) % List of MEX functions to compile mex_files = {'binsparse_read.c', 'binsparse_write.c', ... 'binsparse_from_ssmc.c', 'binsparse_minimize_types.c', ... - 'binsparse_write_string_dataset.c'}; + 'binsparse_write_string_dataset.c', ... + 'binsparse_read_string_dataset.c'}; fprintf('Compiling MEX functions with mkoctfile...\n'); failed_files = {}; diff --git a/bindings/matlab/binsparse_read_string_dataset.c b/bindings/matlab/binsparse_read_string_dataset.c new file mode 100644 index 0000000..53c04ad --- /dev/null +++ b/bindings/matlab/binsparse_read_string_dataset.c @@ -0,0 +1,276 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * binsparse_read_string_dataset.c - Read an HDF5 UTF-8 string dataset. + * + * Usage in MATLAB/Octave: + * value = binsparse_read_string_dataset(filename, dataset_name) + * + * The MATLAB class of the result comes from the HDF5 string datatype: a + * fixed-length dataset reads back as a char matrix and a variable-length one as + * an m-by-1 cellstr. See matlab_bsp_strings.h for the format. + */ + +#include "matlab_bsp_strings.h" +#include "mex.h" +#include +#include +#include + +static char* get_required_string(const mxArray* value, const char* name) { + if (!mxIsChar(value)) { + mexErrMsgIdAndTxt("BinSparse:InvalidString", + "%s must be a character vector", name); + } + + char* string = mxArrayToString(value); + if (!string) { + mexErrMsgIdAndTxt("BinSparse:MemoryError", "Failed to read %s", name); + } + return string; +} + +// Number of elements in a scalar or one-dimensional dataspace. Returns +// (hssize_t) -1 for any other shape. +static hssize_t string_dataset_count(hid_t space) { + int rank = H5Sget_simple_extent_ndims(space); + if (rank < 0 || rank > 1) { + return -1; + } + if (rank == 0) { + // Text written before the fixed/variable convention was introduced could + // be a scalar dataset holding a single string. + return 1; + } + hsize_t dims[1] = {0}; + if (H5Sget_simple_extent_dims(space, dims, NULL) < 0) { + return -1; + } + return (hssize_t) dims[0]; +} + +/*---------------------------------------------------------------------------- + * fixed-length dataset -> char matrix + *--------------------------------------------------------------------------*/ + +static mxArray* read_fixed_strings(hid_t dset, hid_t ftype, size_t count, + const char** error_id, + const char** error_message) { + size_t width = H5Tget_size(ftype); + if (width == 0) { + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to size the fixed-length string datatype"; + return NULL; + } + + // Padding is stripped from the right: NULs always, because that is what this + // writer pads with, and blanks as well when the file declares the Fortran + // convention. + bool space_padded = (H5Tget_strpad(ftype) == H5T_STR_SPACEPAD); + + char* buffer = (char*) mxCalloc(count > 0 ? count * width : 1, sizeof(char)); + if (count > 0 && + H5Dread(dset, ftype, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer) < 0) { + mxFree(buffer); + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to read the fixed-length string dataset"; + return NULL; + } + + size_t* lengths = (size_t*) mxCalloc(count > 0 ? count : 1, sizeof(size_t)); + size_t* units = (size_t*) mxCalloc(count > 0 ? count : 1, sizeof(size_t)); + size_t columns = 0; + + for (size_t i = 0; i < count; i++) { + const char* row = buffer + i * width; + size_t len = width; + while (len > 0 && (row[len - 1] == '\0' || + (space_padded && row[len - 1] == ' '))) { + len--; + } + size_t n = bsp_utf16_length(row, len); + if (n == BSP_UTF_INVALID) { + mxFree(units); + mxFree(lengths); + mxFree(buffer); + *error_id = "BinSparse:InvalidText"; + *error_message = "String dataset contains malformed UTF-8"; + return NULL; + } + lengths[i] = len; + units[i] = n; + if (n > columns) { + columns = n; + } + } + + // Rows written by this binding all decode to the same width. A foreign file + // need not be rectangular, so short rows are blank filled the way MATLAB's + // char() would pad them. + mwSize dims[2] = {(mwSize) count, (mwSize) columns}; + mxArray* result = mxCreateCharArray(2, dims); + mxChar* chars = (mxChar*) mxGetData(result); + for (size_t i = 0; i < count * columns; i++) { + chars[i] = (mxChar) ' '; + } + + mxChar* row_units = + (mxChar*) mxCalloc(columns > 0 ? columns : 1, sizeof(mxChar)); + for (size_t i = 0; i < count; i++) { + bsp_utf16_decode(buffer + i * width, lengths[i], row_units); + for (size_t j = 0; j < units[i]; j++) { + chars[i + j * count] = row_units[j]; + } + } + + mxFree(row_units); + mxFree(units); + mxFree(lengths); + mxFree(buffer); + return result; +} + +/*---------------------------------------------------------------------------- + * variable-length dataset -> cellstr + *--------------------------------------------------------------------------*/ + +static mxArray* read_variable_strings(hid_t dset, hid_t ftype, hid_t space, + size_t count, const char** error_id, + const char** error_message) { + char** buffer = (char**) mxCalloc(count > 0 ? count : 1, sizeof(char*)); + if (count > 0 && + H5Dread(dset, ftype, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer) < 0) { + mxFree(buffer); + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to read the variable-length string dataset"; + return NULL; + } + + mxArray* result = mxCreateCellMatrix((mwSize) count, 1); + bool malformed = false; + + for (size_t i = 0; i < count; i++) { + const char* string = buffer[i] ? buffer[i] : ""; + size_t len = strlen(string); + size_t n = bsp_utf16_length(string, len); + if (n == BSP_UTF_INVALID) { + malformed = true; + n = 0; + len = 0; + } + mwSize dims[2] = {1, (mwSize) n}; + mxArray* cell = mxCreateCharArray(2, dims); + bsp_utf16_decode(string, len, (mxChar*) mxGetData(cell)); + mxSetCell(result, (mwIndex) i, cell); + } + + H5Dvlen_reclaim(ftype, space, H5P_DEFAULT, buffer); + mxFree(buffer); + + if (malformed) { + mxDestroyArray(result); + *error_id = "BinSparse:InvalidText"; + *error_message = "String dataset contains malformed UTF-8"; + return NULL; + } + return result; +} + +void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { + bsp_lock_mex_module(); + + if (nrhs != 2) { + mexErrMsgIdAndTxt("BinSparse:InvalidArgs", + "Usage: value = binsparse_read_string_dataset(filename, " + "dataset_name)"); + } + if (nlhs > 1) { + mexErrMsgIdAndTxt("BinSparse:TooManyOutputs", + "Only one output argument is produced"); + } + + char* filename = get_required_string(prhs[0], "filename"); + char* dataset_name = get_required_string(prhs[1], "dataset_name"); + + hid_t file = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t ftype = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + mxArray* result = NULL; + const char* error_id = NULL; + const char* error_message = NULL; + + file = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file == H5I_INVALID_HID) { + error_id = "BinSparse:FileError"; + error_message = "Failed to open HDF5 file"; + goto cleanup; + } + + dset = H5Dopen2(file, dataset_name, H5P_DEFAULT); + if (dset == H5I_INVALID_HID) { + error_id = "BinSparse:MissingDataset"; + error_message = "Failed to open string dataset"; + goto cleanup; + } + + ftype = H5Dget_type(dset); + if (ftype == H5I_INVALID_HID || H5Tget_class(ftype) != H5T_STRING) { + error_id = "BinSparse:InvalidDataset"; + error_message = "Dataset does not hold strings"; + goto cleanup; + } + + space = H5Dget_space(dset); + hssize_t count = (space == H5I_INVALID_HID) ? -1 : string_dataset_count(space); + if (count < 0) { + error_id = "BinSparse:InvalidDataset"; + error_message = "String dataset must be scalar or one-dimensional"; + goto cleanup; + } + + htri_t variable = H5Tis_variable_str(ftype); + if (variable < 0) { + error_id = "BinSparse:HDF5Error"; + error_message = "Failed to classify the string datatype"; + goto cleanup; + } + + // The datatype is what says which MATLAB class the text came from. + if (variable > 0) { + result = read_variable_strings(dset, ftype, space, (size_t) count, + &error_id, &error_message); + } else { + result = read_fixed_strings(dset, ftype, (size_t) count, &error_id, + &error_message); + } + +cleanup: + if (space != H5I_INVALID_HID) { + H5Sclose(space); + } + if (ftype != H5I_INVALID_HID) { + H5Tclose(ftype); + } + if (dset != H5I_INVALID_HID) { + H5Dclose(dset); + } + if (file != H5I_INVALID_HID) { + H5Fclose(file); + } + mxFree(dataset_name); + mxFree(filename); + + if (error_id) { + if (result) { + mxDestroyArray(result); + } + mexErrMsgIdAndTxt(error_id, "%s", error_message); + } + + plhs[0] = result; +} diff --git a/bindings/matlab/binsparse_to_ssmc_problem.m b/bindings/matlab/binsparse_to_ssmc_problem.m index dbc4746..8c06695 100644 --- a/bindings/matlab/binsparse_to_ssmc_problem.m +++ b/bindings/matlab/binsparse_to_ssmc_problem.m @@ -6,8 +6,9 @@ % bsp_problem is an in-memory representation of one SuiteSparse Matrix % Collection problem. Its A, b, x, and aux numeric entries are raw structs % returned by binsparse_read. The metadata field contains the user metadata -% from the root Binsparse JSON descriptor. Text entries may be char, string, -% or cellstr values returned by h5read. +% from the root Binsparse JSON descriptor. Text entries are the char matrices +% and cellstr values returned by binsparse_read_string_dataset, which recovers +% the MATLAB class from the HDF5 string datatype. % SPDX-FileCopyrightText: 2026 Binsparse Developers % @@ -26,13 +27,13 @@ end if isfield(bsp_problem, 'b') - Problem.b = convert_component(bsp_problem.b, Problem); + Problem.b = convert_component(bsp_problem.b); end if isfield(bsp_problem, 'x') - Problem.x = convert_component(bsp_problem.x, Problem); + Problem.x = convert_component(bsp_problem.x); end if isfield(bsp_problem, 'aux') - Problem.aux = convert_aux(bsp_problem.aux, Problem); + Problem.aux = convert_aux(bsp_problem.aux); end end @@ -81,19 +82,18 @@ function validate_problem(bsp_problem) end end -function value = convert_component(value, Problem) +function value = convert_component(value) if is_bsp_matrix(value) value = convert_matrix(value, false); elseif is_text(value) - use_cellstr = isfield(Problem, 'id') && Problem.id > 2776; - value = normalize_component_text(value, use_cellstr); + value = normalize_component_text(value); else error('BinSparse:InvalidComponent', ... 'Unsupported Binsparse problem component'); end end -function aux = convert_aux(raw_aux, Problem) +function aux = convert_aux(raw_aux) if ~isstruct(raw_aux) || ~isscalar(raw_aux) error('BinSparse:InvalidAux', 'aux must be a scalar struct'); end @@ -102,7 +102,7 @@ function validate_problem(bsp_problem) names = fieldnames(raw_aux); for k = 1:numel(names) name = names{k}; - value = convert_component(raw_aux.(name), Problem); + value = convert_component(raw_aux.(name)); tokens = regexp(name, '^(.*)_([0-9]+)$', 'tokens', 'once'); if isempty(tokens) || isempty(tokens{1}) if isfield(aux, name) @@ -464,28 +464,28 @@ function require_strictly_increasing(values, label) value = char(rows); end -function value = normalize_component_text(value, use_cellstr) +function value = normalize_component_text(value) +% The MATLAB class of a text component is carried by the HDF5 string datatype +% and restored by binsparse_read_string_dataset, so it is preserved here rather +% than inferred. A char matrix stays a char matrix and a cellstr stays a +% cellstr; only a string array, which a foreign reader may produce, is mapped +% onto one of the two classes SSMC uses. if ischar(value) - if size(value, 1) > 1 - rows = cellstr(value); - else - rows = {value}; + return; +elseif iscellstr(value) + value = value(:); + for k = 1:numel(value) + value{k} = reshape(char(value{k}), 1, []); end elseif isstring(value) - rows = cellstr(value(:)); -elseif iscellstr(value) - rows = value(:); + if isscalar(value) + value = char(value); + else + value = cellstr(value(:)); + end else error('BinSparse:InvalidComponent', 'Text component is invalid'); end -for k = 1:numel(rows) - rows{k} = reshape(char(rows{k}), 1, []); -end -if use_cellstr - value = rows; -else - value = char(rows); -end end function ok = is_text(value) diff --git a/bindings/matlab/binsparse_write.c b/bindings/matlab/binsparse_write.c index 14da9a8..53e39b3 100644 --- a/bindings/matlab/binsparse_write.c +++ b/bindings/matlab/binsparse_write.c @@ -24,6 +24,7 @@ #include #include "matlab_bsp_helpers.h" +#include "matlab_bsp_strings.h" // Keep this MEX function loaded for the whole MATLAB session: the HDF5 // library used by libbinsparse installs process-wide state that is not safe @@ -114,14 +115,17 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { mexErrMsgIdAndTxt("BinSparse:InvalidJSON", "JSON must be a string"); } - json_string = mxArrayToString(prhs[3]); + // The descriptor carries SSMC metadata such as Problem.notes, which is + // not always ASCII, so it is encoded as UTF-8 rather than run through + // mxArrayToString and its local code page. + json_string = bsp_mx_to_utf8(prhs[3]); if (!json_string) { bsp_destroy_matrix_t(&matrix); if (group) mxFree(group); mxFree(filename); - mexErrMsgIdAndTxt("BinSparse:MemoryError", - "Failed to convert JSON string"); + mexErrMsgIdAndTxt("BinSparse:InvalidJSON", + "Failed to encode the JSON string as UTF-8"); } } diff --git a/bindings/matlab/binsparse_write_ssmc_problem.m b/bindings/matlab/binsparse_write_ssmc_problem.m index 2fba51c..41e8f4e 100644 --- a/bindings/matlab/binsparse_write_ssmc_problem.m +++ b/bindings/matlab/binsparse_write_ssmc_problem.m @@ -115,7 +115,7 @@ function handle_aux_entry(name, value, output_filename, format, compression_leve end if is_text_value(value) - write_string_dataset(output_filename, name, value); + write_string_dataset(output_filename, name, value, compression_level); return; end @@ -179,35 +179,35 @@ function handle_aux_entry(name, value, output_filename, format, compression_leve ok = ischar(value) || isstring(value) || iscellstr(value); end -function write_string_dataset(output_filename, name, value) +function write_string_dataset(output_filename, name, value, compression_level) if exist('binsparse_write_string_dataset', 'file') ~= 3 error('binsparse_write_ssmc_problem:MissingStringWriter', ... 'BSP text output requires binsparse_write_string_dataset on the path'); end - % Trailing blanks are stripped row by row, as sstextwrite does for the - % MM and RB formats. A single row is left alone: it is the widest row, - % so its blanks are what a char() rebuild needs to recover the width. + % The MATLAB class of the value selects the HDF5 string datatype: a char + % matrix becomes a fixed-length dataset and a cellstr a variable-length + % one, which is what lets the reader restore the class. Nothing is + % deblanked here. A char matrix is rectangular, so its trailing blanks + % are part of the value and the fixed width carries them; a cellstr is + % ragged, and any trailing blanks in an element are the element's own. if isstring(value) if isscalar(value) value = char(value); else - value = strip_text_rows(value); + value = cellstr(value(:)); end elseif ischar(value) - if size(value, 1) > 1 - value = strip_text_rows(value); - else - value = char(value); - end + value = char(value); elseif iscellstr(value) - value = strip_text_rows(value); + value = value(:); else error('binsparse_write_ssmc_problem:InvalidStringValue', ... 'Text aux value must be char, string, or cellstr'); end - binsparse_write_string_dataset(output_filename, name, value); + binsparse_write_string_dataset(output_filename, name, value, ... + compression_level); end function json = metadata_json(P, role) diff --git a/bindings/matlab/binsparse_write_string_dataset.c b/bindings/matlab/binsparse_write_string_dataset.c index 098c2db..b158bb0 100644 --- a/bindings/matlab/binsparse_write_string_dataset.c +++ b/bindings/matlab/binsparse_write_string_dataset.c @@ -9,28 +9,20 @@ * * Usage in MATLAB/Octave: * binsparse_write_string_dataset(filename, dataset_name, value) + * binsparse_write_string_dataset(filename, dataset_name, value, level) * - * The value may be a character vector or a cell array of character vectors. - * Strings are stored as variable-length UTF-8 HDF5 strings, matching the usual - * h5py representation for Python str values. + * A char matrix is written as a fixed-length dataset and a cell array of + * character vectors as a variable-length one, so the datatype alone says which + * MATLAB class the text came from. See matlab_bsp_strings.h for the format. */ +#include "matlab_bsp_strings.h" #include "mex.h" #include #include #include #include -// Keep this MEX function loaded for the whole MATLAB session, and stop HDF5 -// from registering atexit handlers: both guard against crashes when MATLAB -// tears down MEX files that share HDF5 process-wide state. -static void lock_mex_module(void) { - if (!mexIsLocked()) { - H5dont_atexit(); - mexLock(); - } -} - static char* get_required_string(const mxArray* value, const char* name) { if (!mxIsChar(value)) { mexErrMsgIdAndTxt("BinSparse:InvalidString", @@ -44,39 +36,121 @@ static char* get_required_string(const mxArray* value, const char* name) { return string; } -static char** collect_strings(const mxArray* value, size_t* count, - bool* scalar_dataset) { - char** strings = NULL; +static int get_compression_level(int nrhs, const mxArray* prhs[]) { + if (nrhs < 4 || mxIsEmpty(prhs[3])) { + return 0; + } + if (!mxIsNumeric(prhs[3]) || mxIsComplex(prhs[3]) || + mxGetNumberOfElements(prhs[3]) != 1) { + mexErrMsgIdAndTxt("BinSparse:InvalidCompression", + "Compression level must be a real numeric scalar"); + } + double level = mxGetScalar(prhs[3]); + if (!(level >= 0.0) || level > 9.0 || level != (double) (int) level) { + mexErrMsgIdAndTxt("BinSparse:InvalidCompression", + "Compression level must be an integer from 0 to 9"); + } + return (int) level; +} + +/*---------------------------------------------------------------------------- + * char matrix -> fixed-length dataset + * + * Every row is encoded in full, trailing blanks included: the datatype size is + * what carries the column count back to the reader, so nothing is deblanked. + *--------------------------------------------------------------------------*/ + +static char* encode_char_matrix(const mxArray* value, size_t* rows, + size_t* width) { + size_t m = mxGetM(value); + size_t n = mxGetN(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + + // Lay each row out contiguously so it can be encoded as one string. + mxChar* row = (mxChar*) mxCalloc(n > 0 ? n : 1, sizeof(mxChar)); + size_t max_bytes = 0; - if (mxIsCell(value)) { - *count = mxGetNumberOfElements(value); - *scalar_dataset = false; - strings = (char**) mxCalloc(*count, sizeof(char*)); - for (size_t i = 0; i < *count; i++) { - const mxArray* cell = mxGetCell(value, i); - if (!cell || !mxIsChar(cell)) { - mexErrMsgIdAndTxt("BinSparse:InvalidValue", - "Cell values must be character vectors"); - } - strings[i] = mxArrayToString(cell); - if (!strings[i]) { - mexErrMsgIdAndTxt("BinSparse:MemoryError", - "Failed to read string cell"); - } + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + row[j] = chars[i + j * m]; } - } else if (mxIsChar(value)) { - *count = 1; - *scalar_dataset = true; - strings = (char**) mxCalloc(1, sizeof(char*)); - strings[0] = mxArrayToString(value); - if (!strings[0]) { - mexErrMsgIdAndTxt("BinSparse:MemoryError", "Failed to read string"); + size_t bytes = bsp_utf8_length(row, n); + if (bytes == BSP_UTF_INVALID) { + mxFree(row); + mexErrMsgIdAndTxt("BinSparse:InvalidText", + "Row %zu contains an unpaired UTF-16 surrogate", i + 1); } - } else { + if (bytes > max_bytes) { + max_bytes = bytes; + } + } + + // H5Tset_size rejects a zero-byte string, so an all-empty char matrix is + // stored as one NUL per row, which strips back to zero characters. + size_t w = max_bytes > 0 ? max_bytes : 1; + char* buffer = (char*) mxCalloc(m > 0 ? m * w : 1, sizeof(char)); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + row[j] = chars[i + j * m]; + } + // Anything past the encoded row is already NUL from mxCalloc. + bsp_utf8_encode(row, n, buffer + i * w); + } + + mxFree(row); + + *rows = m; + *width = w; + return buffer; +} + +/*---------------------------------------------------------------------------- + * cellstr -> variable-length dataset + *--------------------------------------------------------------------------*/ + +static char* encode_char_vector(const mxArray* value, size_t index) { + if (!value || !mxIsChar(value)) { mexErrMsgIdAndTxt("BinSparse:InvalidValue", - "Value must be a character vector or cellstr"); + "Cell element %zu is not a character vector", index + 1); } + if (mxGetNumberOfDimensions(value) > 2 || mxGetM(value) > 1) { + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Cell element %zu must be a character row vector", + index + 1); + } + + size_t len = mxGetNumberOfElements(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + for (size_t k = 0; k < len; k++) { + if (chars[k] == 0) { + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Cell element %zu contains a NUL character, which a " + "variable-length HDF5 string cannot represent", + index + 1); + } + } + + size_t bytes = bsp_utf8_length(chars, len); + if (bytes == BSP_UTF_INVALID) { + mexErrMsgIdAndTxt("BinSparse:InvalidText", + "Cell element %zu contains an unpaired UTF-16 surrogate", + index + 1); + } + + char* encoded = (char*) mxCalloc(bytes + 1, sizeof(char)); + bsp_utf8_encode(chars, len, encoded); + encoded[bytes] = '\0'; + return encoded; +} +static char** encode_cellstr(const mxArray* value, size_t* count) { + size_t m = mxGetNumberOfElements(value); + char** strings = (char**) mxCalloc(m > 0 ? m : 1, sizeof(char*)); + for (size_t i = 0; i < m; i++) { + strings[i] = encode_char_vector(mxGetCell(value, i), i); + } + *count = m; return strings; } @@ -94,12 +168,12 @@ static void free_strings(char** strings, size_t count) { void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { (void) plhs; - lock_mex_module(); + bsp_lock_mex_module(); - if (nrhs != 3) { + if (nrhs < 3 || nrhs > 4) { mexErrMsgIdAndTxt("BinSparse:InvalidArgs", "Usage: binsparse_write_string_dataset(filename, " - "dataset_name, value)"); + "dataset_name, value [, compression_level])"); } if (nlhs > 0) { @@ -109,14 +183,33 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { char* filename = get_required_string(prhs[0], "filename"); char* dataset_name = get_required_string(prhs[1], "dataset_name"); + int compression_level = get_compression_level(nrhs, prhs); + + const mxArray* value = prhs[2]; + bool is_cellstr = mxIsCell(value); + if ((!is_cellstr && !mxIsChar(value)) || + mxGetNumberOfDimensions(value) > 2) { + mxFree(dataset_name); + mxFree(filename); + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Value must be a two-dimensional char matrix or a cell " + "array of character vectors"); + } size_t count = 0; - bool scalar_dataset = false; - char** strings = collect_strings(prhs[2], &count, &scalar_dataset); + size_t width = 0; + char* fixed_buffer = NULL; + char** strings = NULL; + if (is_cellstr) { + strings = encode_cellstr(value, &count); + } else { + fixed_buffer = encode_char_matrix(value, &count, &width); + } hid_t file = H5I_INVALID_HID; hid_t type = H5I_INVALID_HID; hid_t space = H5I_INVALID_HID; + hid_t properties = H5P_DEFAULT; hid_t dset = H5I_INVALID_HID; const char* error_id = NULL; const char* error_message = NULL; @@ -143,27 +236,25 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { goto cleanup; } - type = H5Tcopy(H5T_C_S1); - if (type == H5I_INVALID_HID || H5Tset_size(type, H5T_VARIABLE) < 0 || - H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + type = is_cellstr ? bsp_variable_string_type() : bsp_fixed_string_type(width); + if (type == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; error_message = "Failed to create UTF-8 string datatype"; goto cleanup; } - if (scalar_dataset) { - space = H5Screate(H5S_SCALAR); - } else { - hsize_t dims[1] = {(hsize_t) count}; - space = H5Screate_simple(1, dims, NULL); - } + hsize_t dims[1] = {(hsize_t) count}; + space = H5Screate_simple(1, dims, NULL); if (space == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; error_message = "Failed to create string dataspace"; goto cleanup; } - dset = H5Dcreate2(file, dataset_name, type, space, H5P_DEFAULT, H5P_DEFAULT, + properties = bsp_text_dataset_properties( + count, is_cellstr ? sizeof(char*) : width, compression_level); + + dset = H5Dcreate2(file, dataset_name, type, space, H5P_DEFAULT, properties, H5P_DEFAULT); if (dset == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; @@ -171,16 +262,21 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { goto cleanup; } - if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, strings) < 0) { - error_id = "BinSparse:HDF5Error"; - error_message = "Failed to write string dataset"; - goto cleanup; + if (count > 0) { + const void* data = + is_cellstr ? (const void*) strings : (const void*) fixed_buffer; + if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) { + error_id = "BinSparse:HDF5Error"; + error_message = "Failed to write string dataset"; + goto cleanup; + } } cleanup: if (dset != H5I_INVALID_HID) { H5Dclose(dset); } + bsp_close_text_dataset_properties(properties); if (space != H5I_INVALID_HID) { H5Sclose(space); } @@ -190,6 +286,9 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { if (file != H5I_INVALID_HID) { H5Fclose(file); } + if (fixed_buffer) { + mxFree(fixed_buffer); + } free_strings(strings, count); mxFree(dataset_name); mxFree(filename); diff --git a/bindings/matlab/compile_octave.sh b/bindings/matlab/compile_octave.sh index 7a32e7b..53e7f80 100755 --- a/bindings/matlab/compile_octave.sh +++ b/bindings/matlab/compile_octave.sh @@ -148,7 +148,7 @@ if [ "$CLEAN" = true ]; then fi # List of MEX files to compile -MEX_FILES=("binsparse_read.c" "binsparse_write.c" "binsparse_from_ssmc.c" "binsparse_minimize_types.c" "binsparse_write_string_dataset.c") +MEX_FILES=("binsparse_read.c" "binsparse_write.c" "binsparse_from_ssmc.c" "binsparse_minimize_types.c" "binsparse_write_string_dataset.c" "binsparse_read_string_dataset.c") print_info "Compiling MEX functions..." diff --git a/bindings/matlab/matlab_bsp_strings.h b/bindings/matlab/matlab_bsp_strings.h new file mode 100644 index 0000000..5cf3407 --- /dev/null +++ b/bindings/matlab/matlab_bsp_strings.h @@ -0,0 +1,311 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * matlab_bsp_strings.h - shared helpers for the HDF5 text datasets that carry + * SuiteSparse Matrix Collection `aux` text components. + * + * Wire format + * ----------- + * A text component is a one-dimensional HDF5 string dataset with character set + * H5T_CSET_UTF8. The string datatype records which MATLAB class the component + * came from, so the file is self-describing and no side-channel metadata is + * needed to read it back: + * + * fixed-length (H5T_STR_NULLPAD) <-> MATLAB char matrix, m-by-n + * variable-length (H5T_VARIABLE) <-> MATLAB cellstr, m-by-1 + * + * A char matrix is rectangular, so every element of a fixed-length dataset + * holds one complete row: all n characters, including the trailing blanks + * MATLAB uses to pad short rows. The datatype size W is the largest UTF-8 + * encoding of a row, in bytes; rows that encode to fewer bytes are padded with + * NULs. For the ASCII text that makes up nearly all of the collection W is + * exactly n and no padding is stored at all, so the width is carried by the + * datatype and the round trip is exact without deblanking either side. + * + * NULs pad rather than blanks because HDF5 sizes a fixed-length string in + * bytes while MATLAB sizes a char matrix in characters. The two agree only + * for ASCII; blank padding would silently widen any row holding a multi-byte + * character. Stripping trailing NULs recovers the row's exact n characters + * whatever it contains. Trailing blanks are stripped as well when a foreign + * file declares H5T_STR_SPACEPAD, which is the Fortran convention. + * + * A cellstr is a ragged list rather than a rectangle, so it is stored + * variable-length and needs no padding convention. + */ + +#ifndef MATLAB_BSP_STRINGS_H +#define MATLAB_BSP_STRINGS_H + +#include "mex.h" +#include +#include +#include +#include +#include + +#define BSP_UTF_INVALID ((size_t) -1) + +/* Chunks are sized to about this many bytes when a text dataset is compressed. */ +#define BSP_TEXT_CHUNK_BYTES ((size_t) (1024 * 1024)) + +/*---------------------------------------------------------------------------- + * UTF-16 (the encoding of a MATLAB mxChar) <-> UTF-8 + *--------------------------------------------------------------------------*/ + +// Number of UTF-8 bytes needed for len UTF-16 code units, or BSP_UTF_INVALID +// if the sequence contains an unpaired surrogate. +static inline size_t bsp_utf8_length(const mxChar* src, size_t len) { + size_t bytes = 0; + for (size_t i = 0; i < len; i++) { + uint32_t unit = src[i]; + if (unit < 0x80u) { + bytes += 1; + } else if (unit < 0x800u) { + bytes += 2; + } else if (unit >= 0xD800u && unit <= 0xDBFFu) { + if (i + 1 >= len || src[i + 1] < 0xDC00u || src[i + 1] > 0xDFFFu) { + return BSP_UTF_INVALID; + } + bytes += 4; + i++; + } else if (unit >= 0xDC00u && unit <= 0xDFFFu) { + return BSP_UTF_INVALID; + } else { + bytes += 3; + } + } + return bytes; +} + +// Encode len UTF-16 code units into out, which must hold bsp_utf8_length +// bytes. Returns the number of bytes written. The caller is responsible for +// having rejected unpaired surrogates via bsp_utf8_length. +static inline size_t bsp_utf8_encode(const mxChar* src, size_t len, char* out) { + unsigned char* p = (unsigned char*) out; + for (size_t i = 0; i < len; i++) { + uint32_t code = src[i]; + if (code >= 0xD800u && code <= 0xDBFFu && i + 1 < len && + src[i + 1] >= 0xDC00u && src[i + 1] <= 0xDFFFu) { + code = 0x10000u + ((code - 0xD800u) << 10) + (src[i + 1] - 0xDC00u); + i++; + } + if (code < 0x80u) { + *p++ = (unsigned char) code; + } else if (code < 0x800u) { + *p++ = (unsigned char) (0xC0u | (code >> 6)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } else if (code < 0x10000u) { + *p++ = (unsigned char) (0xE0u | (code >> 12)); + *p++ = (unsigned char) (0x80u | ((code >> 6) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } else { + *p++ = (unsigned char) (0xF0u | (code >> 18)); + *p++ = (unsigned char) (0x80u | ((code >> 12) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | ((code >> 6) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } + } + return (size_t) ((char*) p - out); +} + +// Number of UTF-16 code units needed for len UTF-8 bytes, or BSP_UTF_INVALID +// if the bytes are not well-formed UTF-8. +static inline size_t bsp_utf16_length(const char* src, size_t len) { + const unsigned char* p = (const unsigned char*) src; + size_t units = 0; + size_t i = 0; + while (i < len) { + unsigned char lead = p[i]; + size_t trail; + uint32_t code; + if (lead < 0x80u) { + trail = 0; + code = lead; + } else if ((lead & 0xE0u) == 0xC0u) { + trail = 1; + code = lead & 0x1Fu; + } else if ((lead & 0xF0u) == 0xE0u) { + trail = 2; + code = lead & 0x0Fu; + } else if ((lead & 0xF8u) == 0xF0u) { + trail = 3; + code = lead & 0x07u; + } else { + return BSP_UTF_INVALID; + } + if (i + trail >= len) { + return BSP_UTF_INVALID; + } + for (size_t k = 1; k <= trail; k++) { + if ((p[i + k] & 0xC0u) != 0x80u) { + return BSP_UTF_INVALID; + } + code = (code << 6) | (uint32_t) (p[i + k] & 0x3Fu); + } + if (code > 0x10FFFFu) { + return BSP_UTF_INVALID; + } + units += (code >= 0x10000u) ? 2 : 1; + i += trail + 1; + } + return units; +} + +// Decode len UTF-8 bytes into out, which must hold bsp_utf16_length code +// units. Returns the number of code units written. The caller is responsible +// for having validated the bytes via bsp_utf16_length. +static inline size_t bsp_utf16_decode(const char* src, size_t len, + mxChar* out) { + const unsigned char* p = (const unsigned char*) src; + size_t units = 0; + size_t i = 0; + while (i < len) { + unsigned char lead = p[i]; + size_t trail; + uint32_t code; + if (lead < 0x80u) { + trail = 0; + code = lead; + } else if ((lead & 0xE0u) == 0xC0u) { + trail = 1; + code = lead & 0x1Fu; + } else if ((lead & 0xF0u) == 0xE0u) { + trail = 2; + code = lead & 0x0Fu; + } else { + trail = 3; + code = lead & 0x07u; + } + for (size_t k = 1; k <= trail; k++) { + code = (code << 6) | (uint32_t) (p[i + k] & 0x3Fu); + } + if (code >= 0x10000u) { + code -= 0x10000u; + out[units++] = (mxChar) (0xD800u + (code >> 10)); + out[units++] = (mxChar) (0xDC00u + (code & 0x3FFu)); + } else { + out[units++] = (mxChar) code; + } + i += trail + 1; + } + return units; +} + +// Convert a MATLAB char array to a NUL-terminated UTF-8 string allocated with +// mxMalloc. Returns NULL if the array is not char, holds an unpaired +// surrogate, or contains a NUL, which a C string cannot carry. Unlike +// mxArrayToString this preserves every character: mxArrayToString goes through +// the local code page and replaces anything it cannot represent with SUB +// (0x1A), which silently corrupts non-ASCII text. +static inline char* bsp_mx_to_utf8(const mxArray* value) { + if (!value || !mxIsChar(value)) { + return NULL; + } + + size_t len = mxGetNumberOfElements(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + for (size_t i = 0; i < len; i++) { + if (chars[i] == 0) { + return NULL; + } + } + + size_t bytes = bsp_utf8_length(chars, len); + if (bytes == BSP_UTF_INVALID) { + return NULL; + } + + char* encoded = (char*) mxMalloc(bytes + 1); + bsp_utf8_encode(chars, len, encoded); + encoded[bytes] = '\0'; + return encoded; +} + +/*---------------------------------------------------------------------------- + * HDF5 datatype and property list helpers + *--------------------------------------------------------------------------*/ + +// Create the fixed-length, NUL-padded UTF-8 datatype used for char matrices. +static inline hid_t bsp_fixed_string_type(size_t width) { + hid_t type = H5Tcopy(H5T_C_S1); + if (type == H5I_INVALID_HID) { + return H5I_INVALID_HID; + } + if (H5Tset_size(type, width) < 0 || + H5Tset_strpad(type, H5T_STR_NULLPAD) < 0 || + H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + H5Tclose(type); + return H5I_INVALID_HID; + } + return type; +} + +// Create the variable-length UTF-8 datatype used for cellstr. +static inline hid_t bsp_variable_string_type(void) { + hid_t type = H5Tcopy(H5T_C_S1); + if (type == H5I_INVALID_HID) { + return H5I_INVALID_HID; + } + if (H5Tset_size(type, H5T_VARIABLE) < 0 || + H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + H5Tclose(type); + return H5I_INVALID_HID; + } + return type; +} + +// Build a dataset creation property list that chunks and deflates a text +// dataset of count elements of element_size bytes. Returns H5P_DEFAULT when +// compression is not requested or would not pay for itself. Note that HDF5 +// keeps variable-length payloads on the global heap, where dataset filters do +// not reach, so only a fixed-length dataset is really compressed. +static inline hid_t bsp_text_dataset_properties(size_t count, + size_t element_size, + int compression_level) { + if (compression_level <= 0 || count < 2 || element_size == 0) { + return H5P_DEFAULT; + } + + size_t chunk = BSP_TEXT_CHUNK_BYTES / element_size; + if (chunk == 0) { + chunk = 1; + } + if (chunk > count) { + chunk = count; + } + + hid_t properties = H5Pcreate(H5P_DATASET_CREATE); + if (properties == H5I_INVALID_HID) { + return H5P_DEFAULT; + } + + hsize_t chunk_dims[1] = {(hsize_t) chunk}; + if (H5Pset_chunk(properties, 1, chunk_dims) < 0 || + H5Pset_deflate(properties, (unsigned) compression_level) < 0) { + H5Pclose(properties); + return H5P_DEFAULT; + } + return properties; +} + +static inline void bsp_close_text_dataset_properties(hid_t properties) { + if (properties != H5P_DEFAULT && properties != H5I_INVALID_HID) { + H5Pclose(properties); + } +} + +// Keep a MEX function loaded for the whole MATLAB session, and stop HDF5 from +// registering atexit handlers: both guard against crashes when MATLAB tears +// down MEX files that share HDF5 process-wide state. +static inline void bsp_lock_mex_module(void) { + if (!mexIsLocked()) { + H5dont_atexit(); + mexLock(); + } +} + +#endif // MATLAB_BSP_STRINGS_H diff --git a/bindings/matlab/test_binsparse_string_dataset.m b/bindings/matlab/test_binsparse_string_dataset.m new file mode 100644 index 0000000..10391aa --- /dev/null +++ b/bindings/matlab/test_binsparse_string_dataset.m @@ -0,0 +1,123 @@ +function test_binsparse_string_dataset() +%TEST_BINSPARSE_STRING_DATASET round-trip tests for the HDF5 text datasets +% +% Checks that binsparse_write_string_dataset and binsparse_read_string_dataset +% preserve both the contents and the MATLAB class of a text component: a char +% matrix is stored fixed-length and a cellstr variable-length, so the two are +% told apart by the HDF5 string datatype alone. + +% SPDX-FileCopyrightText: 2026 Binsparse Developers +% +% SPDX-License-Identifier: BSD-3-Clause + +fprintf('=== Testing Binsparse string datasets ===\n\n'); + +if exist('binsparse_write_string_dataset', 'file') ~= 3 || ... + exist('binsparse_read_string_dataset', 'file') ~= 3 + error('test_binsparse_string_dataset:MissingMex', ... + 'The string dataset MEX functions are not on the path'); +end + +% Each case is {description, value}. The value must survive isequal. +% An empty cellstr element comes back as the 1-by-0 char that sstextread +% produces, not as the 0-by-0 char that '' denotes: an HDF5 string has a +% length but no MATLAB shape, and 1-by-0 is the row that sscellstring accepts. +cases = { + 'char matrix, ragged rows blank padded', char({'alpha'; 'be'; 'gamma!'}) + 'char matrix, every row full width', ['abcd'; 'efgh'; 'ijkl'] + 'char matrix, one row', 'a single row' + 'char matrix, blanks on every row', char({'ab '; 'cd '}) + 'char matrix, leading blanks kept', char({' indented'; 'flush'}) + 'char matrix, one row of blanks', char({'text'; ' '}) + 'char matrix, non-ASCII Latin-1', char({['Jo' 227 'o Pessoa']; ['Bras' 237 'lia']}) + 'char matrix, replacement characters', char({[65533 65533 'x']; 'plain'}) + 'char matrix, single column', char({'a'; 'b'; 'c'}) + 'cellstr, ragged', {'alpha'; 'be'; 'gamma!'} + 'cellstr, one element', {'only'} + 'cellstr, trailing blanks are content', {'ab '; 'cd'} + 'cellstr, empty element', {'first'; char(zeros(1, 0)); 'third'} + 'cellstr, non-ASCII', {['Jo' 227 'o']; ['Bras' 237 'lia']} + 'cellstr, long strings', {repmat('x', 1, 5000); 'short'} +}; + +tmpdir = tempname; +mkdir(tmpdir); +cleanup = onCleanup(@() rmdir(tmpdir, 's')); + +passed = 0; +failed = 0; +for level = [0 9] + fprintf('-- compression level %d\n', level); + for k = 1:size(cases, 1) + description = cases{k, 1}; + value = cases{k, 2}; + filename = fullfile(tmpdir, sprintf('case_%d_%d.h5', level, k)); + try + binsparse_write_string_dataset(filename, 'text', value, level); + actual = binsparse_read_string_dataset(filename, '/text'); + check_equal(actual, value, description); + fprintf(' PASS %s\n', description); + passed = passed + 1; + catch me + fprintf(' FAIL %s: %s\n', description, me.message); + failed = failed + 1; + end + end +end + +% The class must come from the datatype, not from the shape: a one-element +% cellstr and a one-row char matrix hold the same text but must not be +% confused with one another. +filename = fullfile(tmpdir, 'classes.h5'); +binsparse_write_string_dataset(filename, 'as_char', 'only'); +binsparse_write_string_dataset(filename, 'as_cell', {'only'}); +try + if ~ischar(binsparse_read_string_dataset(filename, '/as_char')) || ... + ~iscellstr(binsparse_read_string_dataset(filename, '/as_cell')) + error('test:ClassNotPreserved', 'class was not preserved'); + end + fprintf(' PASS one-row char and one-element cellstr stay distinct\n'); + passed = passed + 1; +catch me + fprintf(' FAIL one-row char vs one-element cellstr: %s\n', me.message); + failed = failed + 1; +end + +% A char matrix should not pay for the variable-length global heap, which no +% dataset filter reaches. A wide, mostly blank matrix must compress. +filename = fullfile(tmpdir, 'compressed.h5'); +wide = repmat(['padded row' blanks(500)], 400, 1); +binsparse_write_string_dataset(filename, 'text', wide, 9); +info = dir(filename); +if isequal(binsparse_read_string_dataset(filename, '/text'), wide) && ... + info.bytes < numel(wide) / 10 + fprintf(' PASS blank padding compresses (%d bytes for %d characters)\n', ... + info.bytes, numel(wide)); + passed = passed + 1; +else + fprintf(' FAIL blank padding did not compress (%d bytes)\n', info.bytes); + failed = failed + 1; +end + +fprintf('\n%d passed, %d failed\n', passed, failed); +if failed > 0 + error('test_binsparse_string_dataset:Failed', ... + '%d string dataset test(s) failed', failed); +end +fprintf('=== All string dataset tests passed ===\n'); + +end + +function check_equal(actual, expected, description) +if ~strcmp(class(actual), class(expected)) + error('test:ClassMismatch', 'expected %s, got %s (%s)', ... + class(expected), class(actual), description); +end +if ~isequal(size(actual), size(expected)) + error('test:SizeMismatch', 'expected %s, got %s (%s)', ... + mat2str(size(expected)), mat2str(size(actual)), description); +end +if ~isequal(actual, expected) + error('test:ValueMismatch', 'contents differ (%s)', description); +end +end diff --git a/bindings/matlab/test_binsparse_to_ssmc_problem.m b/bindings/matlab/test_binsparse_to_ssmc_problem.m index 3f0b252..3f998bf 100644 --- a/bindings/matlab/test_binsparse_to_ssmc_problem.m +++ b/bindings/matlab/test_binsparse_to_ssmc_problem.m @@ -72,14 +72,23 @@ assert(isequal(Problem.x, [1 2; 3 4])); assert(iscell(Problem.aux.seq) && numel(Problem.aux.seq) == 2); assert(isequal(Problem.aux.seq{2}, [7; 8])); -assert(isequal(Problem.aux.label, char({'abc'; 'def'}))); +assert(isequal(Problem.aux.label, {'abc'; 'def'}), ... + 'cellstr component did not stay a cellstr'); -% A stripped string dataset rebuilds the char matrix it was written from, -% including the width carried by the widest row. +% The reader hands over the MATLAB class recorded by the HDF5 string datatype, +% and it survives the conversion untouched: a cellstr keeps its own trailing +% blanks and its ragged element lengths, and a char matrix keeps its width. raw.aux.label = {'hello '; 'there'}; Problem = binsparse_to_ssmc_problem(raw); -assert(isequal(Problem.aux.label, ['hello '; 'there ']), ... - 'stripped string dataset mismatch'); +assert(isequal(Problem.aux.label, {'hello '; 'there'}), ... + 'cellstr component was reshaped or deblanked'); + +raw.aux.label = ['hello '; 'there ']; +Problem = binsparse_to_ssmc_problem(raw); +assert(ischar(Problem.aux.label) && ... + isequal(Problem.aux.label, ['hello '; 'there ']), ... + 'char matrix component did not stay a char matrix'); +raw.aux.label = {'abc'; 'def'}; dmat = dense_matrix([1; 2; 3; 4; 5; 6], 2, 3, 'DMAT'); raw = struct('metadata', metadata, 'A', formats{1}, 'b', dmat); diff --git a/bindings/matlab/test_binsparse_write_ssmc_problem.m b/bindings/matlab/test_binsparse_write_ssmc_problem.m index f6ec3d2..b3d3f39 100644 --- a/bindings/matlab/test_binsparse_write_ssmc_problem.m +++ b/bindings/matlab/test_binsparse_write_ssmc_problem.m @@ -85,16 +85,13 @@ function test_binsparse_write_ssmc_problem() expected_sparse = full(Problem.aux.S); assert(matrices_equal(aux_sparse_mat, expected_sparse), 'Aux sparse matrix mismatch'); -% String datasets are stripped row by row. No row of note reaches the char -% matrix width, so the widest row keeps one blank; note2 needs none. Either -% way char() rebuilds the original char matrix exactly. -check_string_dataset(out_file, 'note', {'hello '; 'there'}); -check_string_dataset(out_file, 'note2', {'hello!'; 'there'}); -check_string_dataset(out_file, 'tags', {'alpha'; 'beta'}); -assert(isequal(char(as_cellstr(h5read(out_file, '/note'))), Problem.aux.note), ... - 'Aux note round trip is not exact'); -assert(isequal(char(as_cellstr(h5read(out_file, '/note2'))), Problem.aux.note2), ... - 'Aux note2 round trip is not exact'); +% Text is stored so that the HDF5 string datatype records the MATLAB class: a +% char matrix goes to a fixed-length dataset, keeping its width and its +% trailing blanks, and a cellstr to a variable-length one. Nothing is +% deblanked, so both come back exactly as written. +check_string_dataset(out_file, 'note', Problem.aux.note); +check_string_dataset(out_file, 'note2', Problem.aux.note2); +check_string_dataset(out_file, 'tags', Problem.aux.tags); json = h5readatt(out_file, '/', 'binsparse'); assert(contains(json, '"metadata"'), 'Primary metadata not nested'); @@ -128,6 +125,23 @@ function test_binsparse_write_ssmc_problem() assert(isequal(char(regexp(clean_notes, '\n', 'split')), clean.notes), ... 'Notes round trip is not exact'); +% The descriptor is UTF-8, so metadata keeps characters that the local code +% page cannot represent. Both of these turn up in the collection: an en dash +% in SNAP/wiki-RfA and a replacement character in Pajek/Journals. +wide = Problem; +wide.notes = char({['en dash ' char(8211) ' here']; ['replacement ' char(65533)]}); +wide.title = ['t' char(233) 'tle']; +wide_file = [tempname() '.bsp.h5']; +cleanup_wide = onCleanup(@() delete_if_exists(wide_file)); %#ok +binsparse_write_ssmc_problem(struct('Problem', wide), wide_file, ... + format, compression_level); +wide_meta = jsondecode(h5readatt(wide_file, '/', 'binsparse')); +wide_meta = wide_meta.metadata; +assert(isequal(char(regexp(wide_meta.notes, '\n', 'split')), wide.notes), ... + 'Non-ASCII notes were not preserved'); +assert(isequal(wide_meta.title, wide.title), ... + 'Non-ASCII title was not preserved'); + fprintf('Test passed.\n'); end @@ -147,8 +161,10 @@ function check_vector_shape(filename, group, expected) end function check_string_dataset(filename, name, expected) - actual = h5read(filename, ['/' name]); - actual = as_cellstr(actual); + actual = binsparse_read_string_dataset(filename, ['/' name]); + assert(strcmp(class(actual), class(expected)), ... + 'String dataset "%s" changed class from %s to %s', name, ... + class(expected), class(actual)); assert(isequal(actual, expected), 'String dataset "%s" mismatch', name); end @@ -163,18 +179,6 @@ function expect_error(action, identifier) error('Expected error %s', identifier); end -function value = as_cellstr(value) - if iscell(value) - value = value(:); - elseif isstring(value) - value = cellstr(value(:)); - elseif ischar(value) - value = cellstr(value); - else - error('Unexpected string dataset value type'); - end -end - function ok = matrices_equal(a, b) if ~isequal(size(a), size(b)) ok = false; From 6e8335432ce63b75abb1c694b0a580b2368cecf4 Mon Sep 17 00:00:00 2001 From: Benjamin Brock Date: Mon, 3 Aug 2026 00:42:21 +0000 Subject: [PATCH 2/3] Choose text compression by total size, not element count The chunk-and-deflate decision keyed off the element count, so a text dataset of one very wide row was left uncompressed however large it was. Threshold on count * element_size instead, which covers both that case and the many-small- elements one, and record what compression actually buys: SNAP/wiki-topcats pagenames 372.6 MB -> 19.2 MB fixed-length Moqri/MISKnowledgeMap Abstract 3.25 MB -> 3.23 MB variable-length A fixed-length dataset compresses well, since blank padding is redundant. A variable-length one barely moves: HDF5 keeps the strings on the global heap, which the filter pipeline does not reach, so only the 16-byte-per-element descriptor array is filtered. That is an HDF5 limitation rather than a choice here, and it costs about 3.2 MB across the whole collection, which has only eight cellstr aux components. Co-Authored-By: Claude Opus 5 (1M context) --- bindings/matlab/matlab_bsp_strings.h | 20 ++++++++++--- .../matlab/test_binsparse_string_dataset.m | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/bindings/matlab/matlab_bsp_strings.h b/bindings/matlab/matlab_bsp_strings.h index 5cf3407..6708bf4 100644 --- a/bindings/matlab/matlab_bsp_strings.h +++ b/bindings/matlab/matlab_bsp_strings.h @@ -52,6 +52,9 @@ /* Chunks are sized to about this many bytes when a text dataset is compressed. */ #define BSP_TEXT_CHUNK_BYTES ((size_t) (1024 * 1024)) +/* Below this many bytes the chunk index costs more than the filter saves. */ +#define BSP_TEXT_MIN_COMPRESSED_BYTES ((size_t) 4096) + /*---------------------------------------------------------------------------- * UTF-16 (the encoding of a MATLAB mxChar) <-> UTF-8 *--------------------------------------------------------------------------*/ @@ -260,13 +263,22 @@ static inline hid_t bsp_variable_string_type(void) { // Build a dataset creation property list that chunks and deflates a text // dataset of count elements of element_size bytes. Returns H5P_DEFAULT when -// compression is not requested or would not pay for itself. Note that HDF5 -// keeps variable-length payloads on the global heap, where dataset filters do -// not reach, so only a fixed-length dataset is really compressed. +// compression is not requested or when the dataset is too small for a chunk +// index to pay for itself. The threshold is on the total size rather than on +// the element count, so that one very wide row is still compressed. +// +// A fixed-length dataset compresses well: blank padding is highly redundant, +// and SNAP/wiki-topcats pagenames goes from 372.6 MB to 19.2 MB. A +// variable-length one barely moves, because HDF5 keeps the strings themselves +// on the global heap, which the filter pipeline does not reach; only the +// 16-byte-per-element descriptor array is filtered. Requesting compression is +// still worthwhile there, since that descriptor array grows with the element +// count, but the payload cannot be reached from here. static inline hid_t bsp_text_dataset_properties(size_t count, size_t element_size, int compression_level) { - if (compression_level <= 0 || count < 2 || element_size == 0) { + if (compression_level <= 0 || element_size == 0 || + count * element_size < BSP_TEXT_MIN_COMPRESSED_BYTES) { return H5P_DEFAULT; } diff --git a/bindings/matlab/test_binsparse_string_dataset.m b/bindings/matlab/test_binsparse_string_dataset.m index 10391aa..57cca43 100644 --- a/bindings/matlab/test_binsparse_string_dataset.m +++ b/bindings/matlab/test_binsparse_string_dataset.m @@ -99,6 +99,35 @@ function test_binsparse_string_dataset() failed = failed + 1; end +% Compression is chosen on the total size, not the element count, so a single +% very wide row is compressed too. +filename = fullfile(tmpdir, 'one_row.h5'); +one_row = [repmat('abcdefgh', 1, 20000) blanks(20000)]; +binsparse_write_string_dataset(filename, 'text', one_row, 9); +info = dir(filename); +if isequal(binsparse_read_string_dataset(filename, '/text'), one_row) && ... + info.bytes < numel(one_row) / 10 + fprintf(' PASS a single wide row compresses (%d bytes for %d characters)\n', ... + info.bytes, numel(one_row)); + passed = passed + 1; +else + fprintf(' FAIL a single wide row did not compress (%d bytes for %d)\n', ... + info.bytes, numel(one_row)); + failed = failed + 1; +end + +% Below the threshold no chunk index is created, and the data still reads back. +filename = fullfile(tmpdir, 'tiny.h5'); +tiny = ['ab'; 'cd']; +binsparse_write_string_dataset(filename, 'text', tiny, 9); +if isequal(binsparse_read_string_dataset(filename, '/text'), tiny) + fprintf(' PASS a dataset below the compression threshold round-trips\n'); + passed = passed + 1; +else + fprintf(' FAIL a dataset below the compression threshold changed\n'); + failed = failed + 1; +end + fprintf('\n%d passed, %d failed\n', passed, failed); if failed > 0 error('test_binsparse_string_dataset:Failed', ... From d0d97138e07e6d19a36a331c782a6e13d728cdf8 Mon Sep 17 00:00:00 2001 From: Benjamin Brock Date: Sun, 2 Aug 2026 18:05:33 -0700 Subject: [PATCH 3/3] Fixes for linter --- bindings/matlab/binsparse_read_string_dataset.c | 7 ++++--- bindings/matlab/binsparse_write_string_dataset.c | 3 +-- bindings/matlab/matlab_bsp_strings.h | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bindings/matlab/binsparse_read_string_dataset.c b/bindings/matlab/binsparse_read_string_dataset.c index 53c04ad..b3ecb68 100644 --- a/bindings/matlab/binsparse_read_string_dataset.c +++ b/bindings/matlab/binsparse_read_string_dataset.c @@ -88,8 +88,8 @@ static mxArray* read_fixed_strings(hid_t dset, hid_t ftype, size_t count, for (size_t i = 0; i < count; i++) { const char* row = buffer + i * width; size_t len = width; - while (len > 0 && (row[len - 1] == '\0' || - (space_padded && row[len - 1] == ' '))) { + while (len > 0 && + (row[len - 1] == '\0' || (space_padded && row[len - 1] == ' '))) { len--; } size_t n = bsp_utf16_length(row, len); @@ -226,7 +226,8 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { } space = H5Dget_space(dset); - hssize_t count = (space == H5I_INVALID_HID) ? -1 : string_dataset_count(space); + hssize_t count = + (space == H5I_INVALID_HID) ? -1 : string_dataset_count(space); if (count < 0) { error_id = "BinSparse:InvalidDataset"; error_message = "String dataset must be scalar or one-dimensional"; diff --git a/bindings/matlab/binsparse_write_string_dataset.c b/bindings/matlab/binsparse_write_string_dataset.c index b158bb0..4854424 100644 --- a/bindings/matlab/binsparse_write_string_dataset.c +++ b/bindings/matlab/binsparse_write_string_dataset.c @@ -187,8 +187,7 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { const mxArray* value = prhs[2]; bool is_cellstr = mxIsCell(value); - if ((!is_cellstr && !mxIsChar(value)) || - mxGetNumberOfDimensions(value) > 2) { + if ((!is_cellstr && !mxIsChar(value)) || mxGetNumberOfDimensions(value) > 2) { mxFree(dataset_name); mxFree(filename); mexErrMsgIdAndTxt("BinSparse:InvalidValue", diff --git a/bindings/matlab/matlab_bsp_strings.h b/bindings/matlab/matlab_bsp_strings.h index 6708bf4..79ec7f8 100644 --- a/bindings/matlab/matlab_bsp_strings.h +++ b/bindings/matlab/matlab_bsp_strings.h @@ -49,7 +49,8 @@ #define BSP_UTF_INVALID ((size_t) -1) -/* Chunks are sized to about this many bytes when a text dataset is compressed. */ +/* Chunks are sized to about this many bytes when a text dataset is compressed. + */ #define BSP_TEXT_CHUNK_BYTES ((size_t) (1024 * 1024)) /* Below this many bytes the chunk index costs more than the filter saves. */