diff --git a/bindings/matlab/Contents.m b/bindings/matlab/Contents.m index f44becc..c785ccc 100644 --- a/bindings/matlab/Contents.m +++ b/bindings/matlab/Contents.m @@ -6,6 +6,7 @@ % binsparse_from_ssmc - convert SSMC A+Zeros to a Binsparse matrix struct % binsparse_minimize_types - minimize value/index types in a Binsparse struct % binsparse_write_string_dataset - write an HDF5 UTF-8 string dataset +% binsparse_read_string_dataset - read an HDF5 UTF-8 string dataset % % MATLAB helpers: % binsparse_write_ssmc_problem - write an SSMC Problem to a Binsparse file diff --git a/bindings/matlab/binsparse_build_matlab_bindings.m b/bindings/matlab/binsparse_build_matlab_bindings.m index 1bed03f..701d837 100644 --- a/bindings/matlab/binsparse_build_matlab_bindings.m +++ b/bindings/matlab/binsparse_build_matlab_bindings.m @@ -107,7 +107,8 @@ function compile_mex_functions(paths, verbose) % List of MEX functions to compile mex_files = {'binsparse_read.c', 'binsparse_write.c', ... 'binsparse_from_ssmc.c', 'binsparse_minimize_types.c', ... - 'binsparse_write_string_dataset.c'}; + 'binsparse_write_string_dataset.c', ... + 'binsparse_read_string_dataset.c'}; fprintf('Compiling MEX functions...\n'); failed_files = {}; diff --git a/bindings/matlab/binsparse_build_octave_bindings.m b/bindings/matlab/binsparse_build_octave_bindings.m index d376b66..c9ba44d 100644 --- a/bindings/matlab/binsparse_build_octave_bindings.m +++ b/bindings/matlab/binsparse_build_octave_bindings.m @@ -120,7 +120,8 @@ function compile_octave_functions(paths, verbose) % List of MEX functions to compile mex_files = {'binsparse_read.c', 'binsparse_write.c', ... 'binsparse_from_ssmc.c', 'binsparse_minimize_types.c', ... - 'binsparse_write_string_dataset.c'}; + 'binsparse_write_string_dataset.c', ... + 'binsparse_read_string_dataset.c'}; fprintf('Compiling MEX functions with mkoctfile...\n'); failed_files = {}; diff --git a/bindings/matlab/binsparse_read_string_dataset.c b/bindings/matlab/binsparse_read_string_dataset.c new file mode 100644 index 0000000..b3ecb68 --- /dev/null +++ b/bindings/matlab/binsparse_read_string_dataset.c @@ -0,0 +1,277 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * binsparse_read_string_dataset.c - Read an HDF5 UTF-8 string dataset. + * + * Usage in MATLAB/Octave: + * value = binsparse_read_string_dataset(filename, dataset_name) + * + * The MATLAB class of the result comes from the HDF5 string datatype: a + * fixed-length dataset reads back as a char matrix and a variable-length one as + * an m-by-1 cellstr. See matlab_bsp_strings.h for the format. + */ + +#include "matlab_bsp_strings.h" +#include "mex.h" +#include +#include +#include + +static char* get_required_string(const mxArray* value, const char* name) { + if (!mxIsChar(value)) { + mexErrMsgIdAndTxt("BinSparse:InvalidString", + "%s must be a character vector", name); + } + + char* string = mxArrayToString(value); + if (!string) { + mexErrMsgIdAndTxt("BinSparse:MemoryError", "Failed to read %s", name); + } + return string; +} + +// Number of elements in a scalar or one-dimensional dataspace. Returns +// (hssize_t) -1 for any other shape. +static hssize_t string_dataset_count(hid_t space) { + int rank = H5Sget_simple_extent_ndims(space); + if (rank < 0 || rank > 1) { + return -1; + } + if (rank == 0) { + // Text written before the fixed/variable convention was introduced could + // be a scalar dataset holding a single string. + return 1; + } + hsize_t dims[1] = {0}; + if (H5Sget_simple_extent_dims(space, dims, NULL) < 0) { + return -1; + } + return (hssize_t) dims[0]; +} + +/*---------------------------------------------------------------------------- + * fixed-length dataset -> char matrix + *--------------------------------------------------------------------------*/ + +static mxArray* read_fixed_strings(hid_t dset, hid_t ftype, size_t count, + const char** error_id, + const char** error_message) { + size_t width = H5Tget_size(ftype); + if (width == 0) { + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to size the fixed-length string datatype"; + return NULL; + } + + // Padding is stripped from the right: NULs always, because that is what this + // writer pads with, and blanks as well when the file declares the Fortran + // convention. + bool space_padded = (H5Tget_strpad(ftype) == H5T_STR_SPACEPAD); + + char* buffer = (char*) mxCalloc(count > 0 ? count * width : 1, sizeof(char)); + if (count > 0 && + H5Dread(dset, ftype, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer) < 0) { + mxFree(buffer); + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to read the fixed-length string dataset"; + return NULL; + } + + size_t* lengths = (size_t*) mxCalloc(count > 0 ? count : 1, sizeof(size_t)); + size_t* units = (size_t*) mxCalloc(count > 0 ? count : 1, sizeof(size_t)); + size_t columns = 0; + + for (size_t i = 0; i < count; i++) { + const char* row = buffer + i * width; + size_t len = width; + while (len > 0 && + (row[len - 1] == '\0' || (space_padded && row[len - 1] == ' '))) { + len--; + } + size_t n = bsp_utf16_length(row, len); + if (n == BSP_UTF_INVALID) { + mxFree(units); + mxFree(lengths); + mxFree(buffer); + *error_id = "BinSparse:InvalidText"; + *error_message = "String dataset contains malformed UTF-8"; + return NULL; + } + lengths[i] = len; + units[i] = n; + if (n > columns) { + columns = n; + } + } + + // Rows written by this binding all decode to the same width. A foreign file + // need not be rectangular, so short rows are blank filled the way MATLAB's + // char() would pad them. + mwSize dims[2] = {(mwSize) count, (mwSize) columns}; + mxArray* result = mxCreateCharArray(2, dims); + mxChar* chars = (mxChar*) mxGetData(result); + for (size_t i = 0; i < count * columns; i++) { + chars[i] = (mxChar) ' '; + } + + mxChar* row_units = + (mxChar*) mxCalloc(columns > 0 ? columns : 1, sizeof(mxChar)); + for (size_t i = 0; i < count; i++) { + bsp_utf16_decode(buffer + i * width, lengths[i], row_units); + for (size_t j = 0; j < units[i]; j++) { + chars[i + j * count] = row_units[j]; + } + } + + mxFree(row_units); + mxFree(units); + mxFree(lengths); + mxFree(buffer); + return result; +} + +/*---------------------------------------------------------------------------- + * variable-length dataset -> cellstr + *--------------------------------------------------------------------------*/ + +static mxArray* read_variable_strings(hid_t dset, hid_t ftype, hid_t space, + size_t count, const char** error_id, + const char** error_message) { + char** buffer = (char**) mxCalloc(count > 0 ? count : 1, sizeof(char*)); + if (count > 0 && + H5Dread(dset, ftype, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer) < 0) { + mxFree(buffer); + *error_id = "BinSparse:HDF5Error"; + *error_message = "Failed to read the variable-length string dataset"; + return NULL; + } + + mxArray* result = mxCreateCellMatrix((mwSize) count, 1); + bool malformed = false; + + for (size_t i = 0; i < count; i++) { + const char* string = buffer[i] ? buffer[i] : ""; + size_t len = strlen(string); + size_t n = bsp_utf16_length(string, len); + if (n == BSP_UTF_INVALID) { + malformed = true; + n = 0; + len = 0; + } + mwSize dims[2] = {1, (mwSize) n}; + mxArray* cell = mxCreateCharArray(2, dims); + bsp_utf16_decode(string, len, (mxChar*) mxGetData(cell)); + mxSetCell(result, (mwIndex) i, cell); + } + + H5Dvlen_reclaim(ftype, space, H5P_DEFAULT, buffer); + mxFree(buffer); + + if (malformed) { + mxDestroyArray(result); + *error_id = "BinSparse:InvalidText"; + *error_message = "String dataset contains malformed UTF-8"; + return NULL; + } + return result; +} + +void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { + bsp_lock_mex_module(); + + if (nrhs != 2) { + mexErrMsgIdAndTxt("BinSparse:InvalidArgs", + "Usage: value = binsparse_read_string_dataset(filename, " + "dataset_name)"); + } + if (nlhs > 1) { + mexErrMsgIdAndTxt("BinSparse:TooManyOutputs", + "Only one output argument is produced"); + } + + char* filename = get_required_string(prhs[0], "filename"); + char* dataset_name = get_required_string(prhs[1], "dataset_name"); + + hid_t file = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t ftype = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + mxArray* result = NULL; + const char* error_id = NULL; + const char* error_message = NULL; + + file = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file == H5I_INVALID_HID) { + error_id = "BinSparse:FileError"; + error_message = "Failed to open HDF5 file"; + goto cleanup; + } + + dset = H5Dopen2(file, dataset_name, H5P_DEFAULT); + if (dset == H5I_INVALID_HID) { + error_id = "BinSparse:MissingDataset"; + error_message = "Failed to open string dataset"; + goto cleanup; + } + + ftype = H5Dget_type(dset); + if (ftype == H5I_INVALID_HID || H5Tget_class(ftype) != H5T_STRING) { + error_id = "BinSparse:InvalidDataset"; + error_message = "Dataset does not hold strings"; + goto cleanup; + } + + space = H5Dget_space(dset); + hssize_t count = + (space == H5I_INVALID_HID) ? -1 : string_dataset_count(space); + if (count < 0) { + error_id = "BinSparse:InvalidDataset"; + error_message = "String dataset must be scalar or one-dimensional"; + goto cleanup; + } + + htri_t variable = H5Tis_variable_str(ftype); + if (variable < 0) { + error_id = "BinSparse:HDF5Error"; + error_message = "Failed to classify the string datatype"; + goto cleanup; + } + + // The datatype is what says which MATLAB class the text came from. + if (variable > 0) { + result = read_variable_strings(dset, ftype, space, (size_t) count, + &error_id, &error_message); + } else { + result = read_fixed_strings(dset, ftype, (size_t) count, &error_id, + &error_message); + } + +cleanup: + if (space != H5I_INVALID_HID) { + H5Sclose(space); + } + if (ftype != H5I_INVALID_HID) { + H5Tclose(ftype); + } + if (dset != H5I_INVALID_HID) { + H5Dclose(dset); + } + if (file != H5I_INVALID_HID) { + H5Fclose(file); + } + mxFree(dataset_name); + mxFree(filename); + + if (error_id) { + if (result) { + mxDestroyArray(result); + } + mexErrMsgIdAndTxt(error_id, "%s", error_message); + } + + plhs[0] = result; +} diff --git a/bindings/matlab/binsparse_to_ssmc_problem.m b/bindings/matlab/binsparse_to_ssmc_problem.m index dbc4746..8c06695 100644 --- a/bindings/matlab/binsparse_to_ssmc_problem.m +++ b/bindings/matlab/binsparse_to_ssmc_problem.m @@ -6,8 +6,9 @@ % bsp_problem is an in-memory representation of one SuiteSparse Matrix % Collection problem. Its A, b, x, and aux numeric entries are raw structs % returned by binsparse_read. The metadata field contains the user metadata -% from the root Binsparse JSON descriptor. Text entries may be char, string, -% or cellstr values returned by h5read. +% from the root Binsparse JSON descriptor. Text entries are the char matrices +% and cellstr values returned by binsparse_read_string_dataset, which recovers +% the MATLAB class from the HDF5 string datatype. % SPDX-FileCopyrightText: 2026 Binsparse Developers % @@ -26,13 +27,13 @@ end if isfield(bsp_problem, 'b') - Problem.b = convert_component(bsp_problem.b, Problem); + Problem.b = convert_component(bsp_problem.b); end if isfield(bsp_problem, 'x') - Problem.x = convert_component(bsp_problem.x, Problem); + Problem.x = convert_component(bsp_problem.x); end if isfield(bsp_problem, 'aux') - Problem.aux = convert_aux(bsp_problem.aux, Problem); + Problem.aux = convert_aux(bsp_problem.aux); end end @@ -81,19 +82,18 @@ function validate_problem(bsp_problem) end end -function value = convert_component(value, Problem) +function value = convert_component(value) if is_bsp_matrix(value) value = convert_matrix(value, false); elseif is_text(value) - use_cellstr = isfield(Problem, 'id') && Problem.id > 2776; - value = normalize_component_text(value, use_cellstr); + value = normalize_component_text(value); else error('BinSparse:InvalidComponent', ... 'Unsupported Binsparse problem component'); end end -function aux = convert_aux(raw_aux, Problem) +function aux = convert_aux(raw_aux) if ~isstruct(raw_aux) || ~isscalar(raw_aux) error('BinSparse:InvalidAux', 'aux must be a scalar struct'); end @@ -102,7 +102,7 @@ function validate_problem(bsp_problem) names = fieldnames(raw_aux); for k = 1:numel(names) name = names{k}; - value = convert_component(raw_aux.(name), Problem); + value = convert_component(raw_aux.(name)); tokens = regexp(name, '^(.*)_([0-9]+)$', 'tokens', 'once'); if isempty(tokens) || isempty(tokens{1}) if isfield(aux, name) @@ -464,28 +464,28 @@ function require_strictly_increasing(values, label) value = char(rows); end -function value = normalize_component_text(value, use_cellstr) +function value = normalize_component_text(value) +% The MATLAB class of a text component is carried by the HDF5 string datatype +% and restored by binsparse_read_string_dataset, so it is preserved here rather +% than inferred. A char matrix stays a char matrix and a cellstr stays a +% cellstr; only a string array, which a foreign reader may produce, is mapped +% onto one of the two classes SSMC uses. if ischar(value) - if size(value, 1) > 1 - rows = cellstr(value); - else - rows = {value}; + return; +elseif iscellstr(value) + value = value(:); + for k = 1:numel(value) + value{k} = reshape(char(value{k}), 1, []); end elseif isstring(value) - rows = cellstr(value(:)); -elseif iscellstr(value) - rows = value(:); + if isscalar(value) + value = char(value); + else + value = cellstr(value(:)); + end else error('BinSparse:InvalidComponent', 'Text component is invalid'); end -for k = 1:numel(rows) - rows{k} = reshape(char(rows{k}), 1, []); -end -if use_cellstr - value = rows; -else - value = char(rows); -end end function ok = is_text(value) diff --git a/bindings/matlab/binsparse_write.c b/bindings/matlab/binsparse_write.c index 14da9a8..53e39b3 100644 --- a/bindings/matlab/binsparse_write.c +++ b/bindings/matlab/binsparse_write.c @@ -24,6 +24,7 @@ #include #include "matlab_bsp_helpers.h" +#include "matlab_bsp_strings.h" // Keep this MEX function loaded for the whole MATLAB session: the HDF5 // library used by libbinsparse installs process-wide state that is not safe @@ -114,14 +115,17 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { mexErrMsgIdAndTxt("BinSparse:InvalidJSON", "JSON must be a string"); } - json_string = mxArrayToString(prhs[3]); + // The descriptor carries SSMC metadata such as Problem.notes, which is + // not always ASCII, so it is encoded as UTF-8 rather than run through + // mxArrayToString and its local code page. + json_string = bsp_mx_to_utf8(prhs[3]); if (!json_string) { bsp_destroy_matrix_t(&matrix); if (group) mxFree(group); mxFree(filename); - mexErrMsgIdAndTxt("BinSparse:MemoryError", - "Failed to convert JSON string"); + mexErrMsgIdAndTxt("BinSparse:InvalidJSON", + "Failed to encode the JSON string as UTF-8"); } } diff --git a/bindings/matlab/binsparse_write_ssmc_problem.m b/bindings/matlab/binsparse_write_ssmc_problem.m index 2fba51c..41e8f4e 100644 --- a/bindings/matlab/binsparse_write_ssmc_problem.m +++ b/bindings/matlab/binsparse_write_ssmc_problem.m @@ -115,7 +115,7 @@ function handle_aux_entry(name, value, output_filename, format, compression_leve end if is_text_value(value) - write_string_dataset(output_filename, name, value); + write_string_dataset(output_filename, name, value, compression_level); return; end @@ -179,35 +179,35 @@ function handle_aux_entry(name, value, output_filename, format, compression_leve ok = ischar(value) || isstring(value) || iscellstr(value); end -function write_string_dataset(output_filename, name, value) +function write_string_dataset(output_filename, name, value, compression_level) if exist('binsparse_write_string_dataset', 'file') ~= 3 error('binsparse_write_ssmc_problem:MissingStringWriter', ... 'BSP text output requires binsparse_write_string_dataset on the path'); end - % Trailing blanks are stripped row by row, as sstextwrite does for the - % MM and RB formats. A single row is left alone: it is the widest row, - % so its blanks are what a char() rebuild needs to recover the width. + % The MATLAB class of the value selects the HDF5 string datatype: a char + % matrix becomes a fixed-length dataset and a cellstr a variable-length + % one, which is what lets the reader restore the class. Nothing is + % deblanked here. A char matrix is rectangular, so its trailing blanks + % are part of the value and the fixed width carries them; a cellstr is + % ragged, and any trailing blanks in an element are the element's own. if isstring(value) if isscalar(value) value = char(value); else - value = strip_text_rows(value); + value = cellstr(value(:)); end elseif ischar(value) - if size(value, 1) > 1 - value = strip_text_rows(value); - else - value = char(value); - end + value = char(value); elseif iscellstr(value) - value = strip_text_rows(value); + value = value(:); else error('binsparse_write_ssmc_problem:InvalidStringValue', ... 'Text aux value must be char, string, or cellstr'); end - binsparse_write_string_dataset(output_filename, name, value); + binsparse_write_string_dataset(output_filename, name, value, ... + compression_level); end function json = metadata_json(P, role) diff --git a/bindings/matlab/binsparse_write_string_dataset.c b/bindings/matlab/binsparse_write_string_dataset.c index 098c2db..4854424 100644 --- a/bindings/matlab/binsparse_write_string_dataset.c +++ b/bindings/matlab/binsparse_write_string_dataset.c @@ -9,28 +9,20 @@ * * Usage in MATLAB/Octave: * binsparse_write_string_dataset(filename, dataset_name, value) + * binsparse_write_string_dataset(filename, dataset_name, value, level) * - * The value may be a character vector or a cell array of character vectors. - * Strings are stored as variable-length UTF-8 HDF5 strings, matching the usual - * h5py representation for Python str values. + * A char matrix is written as a fixed-length dataset and a cell array of + * character vectors as a variable-length one, so the datatype alone says which + * MATLAB class the text came from. See matlab_bsp_strings.h for the format. */ +#include "matlab_bsp_strings.h" #include "mex.h" #include #include #include #include -// Keep this MEX function loaded for the whole MATLAB session, and stop HDF5 -// from registering atexit handlers: both guard against crashes when MATLAB -// tears down MEX files that share HDF5 process-wide state. -static void lock_mex_module(void) { - if (!mexIsLocked()) { - H5dont_atexit(); - mexLock(); - } -} - static char* get_required_string(const mxArray* value, const char* name) { if (!mxIsChar(value)) { mexErrMsgIdAndTxt("BinSparse:InvalidString", @@ -44,39 +36,121 @@ static char* get_required_string(const mxArray* value, const char* name) { return string; } -static char** collect_strings(const mxArray* value, size_t* count, - bool* scalar_dataset) { - char** strings = NULL; +static int get_compression_level(int nrhs, const mxArray* prhs[]) { + if (nrhs < 4 || mxIsEmpty(prhs[3])) { + return 0; + } + if (!mxIsNumeric(prhs[3]) || mxIsComplex(prhs[3]) || + mxGetNumberOfElements(prhs[3]) != 1) { + mexErrMsgIdAndTxt("BinSparse:InvalidCompression", + "Compression level must be a real numeric scalar"); + } + double level = mxGetScalar(prhs[3]); + if (!(level >= 0.0) || level > 9.0 || level != (double) (int) level) { + mexErrMsgIdAndTxt("BinSparse:InvalidCompression", + "Compression level must be an integer from 0 to 9"); + } + return (int) level; +} + +/*---------------------------------------------------------------------------- + * char matrix -> fixed-length dataset + * + * Every row is encoded in full, trailing blanks included: the datatype size is + * what carries the column count back to the reader, so nothing is deblanked. + *--------------------------------------------------------------------------*/ + +static char* encode_char_matrix(const mxArray* value, size_t* rows, + size_t* width) { + size_t m = mxGetM(value); + size_t n = mxGetN(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + + // Lay each row out contiguously so it can be encoded as one string. + mxChar* row = (mxChar*) mxCalloc(n > 0 ? n : 1, sizeof(mxChar)); + size_t max_bytes = 0; - if (mxIsCell(value)) { - *count = mxGetNumberOfElements(value); - *scalar_dataset = false; - strings = (char**) mxCalloc(*count, sizeof(char*)); - for (size_t i = 0; i < *count; i++) { - const mxArray* cell = mxGetCell(value, i); - if (!cell || !mxIsChar(cell)) { - mexErrMsgIdAndTxt("BinSparse:InvalidValue", - "Cell values must be character vectors"); - } - strings[i] = mxArrayToString(cell); - if (!strings[i]) { - mexErrMsgIdAndTxt("BinSparse:MemoryError", - "Failed to read string cell"); - } + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + row[j] = chars[i + j * m]; } - } else if (mxIsChar(value)) { - *count = 1; - *scalar_dataset = true; - strings = (char**) mxCalloc(1, sizeof(char*)); - strings[0] = mxArrayToString(value); - if (!strings[0]) { - mexErrMsgIdAndTxt("BinSparse:MemoryError", "Failed to read string"); + size_t bytes = bsp_utf8_length(row, n); + if (bytes == BSP_UTF_INVALID) { + mxFree(row); + mexErrMsgIdAndTxt("BinSparse:InvalidText", + "Row %zu contains an unpaired UTF-16 surrogate", i + 1); } - } else { + if (bytes > max_bytes) { + max_bytes = bytes; + } + } + + // H5Tset_size rejects a zero-byte string, so an all-empty char matrix is + // stored as one NUL per row, which strips back to zero characters. + size_t w = max_bytes > 0 ? max_bytes : 1; + char* buffer = (char*) mxCalloc(m > 0 ? m * w : 1, sizeof(char)); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + row[j] = chars[i + j * m]; + } + // Anything past the encoded row is already NUL from mxCalloc. + bsp_utf8_encode(row, n, buffer + i * w); + } + + mxFree(row); + + *rows = m; + *width = w; + return buffer; +} + +/*---------------------------------------------------------------------------- + * cellstr -> variable-length dataset + *--------------------------------------------------------------------------*/ + +static char* encode_char_vector(const mxArray* value, size_t index) { + if (!value || !mxIsChar(value)) { mexErrMsgIdAndTxt("BinSparse:InvalidValue", - "Value must be a character vector or cellstr"); + "Cell element %zu is not a character vector", index + 1); } + if (mxGetNumberOfDimensions(value) > 2 || mxGetM(value) > 1) { + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Cell element %zu must be a character row vector", + index + 1); + } + + size_t len = mxGetNumberOfElements(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + for (size_t k = 0; k < len; k++) { + if (chars[k] == 0) { + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Cell element %zu contains a NUL character, which a " + "variable-length HDF5 string cannot represent", + index + 1); + } + } + + size_t bytes = bsp_utf8_length(chars, len); + if (bytes == BSP_UTF_INVALID) { + mexErrMsgIdAndTxt("BinSparse:InvalidText", + "Cell element %zu contains an unpaired UTF-16 surrogate", + index + 1); + } + + char* encoded = (char*) mxCalloc(bytes + 1, sizeof(char)); + bsp_utf8_encode(chars, len, encoded); + encoded[bytes] = '\0'; + return encoded; +} +static char** encode_cellstr(const mxArray* value, size_t* count) { + size_t m = mxGetNumberOfElements(value); + char** strings = (char**) mxCalloc(m > 0 ? m : 1, sizeof(char*)); + for (size_t i = 0; i < m; i++) { + strings[i] = encode_char_vector(mxGetCell(value, i), i); + } + *count = m; return strings; } @@ -94,12 +168,12 @@ static void free_strings(char** strings, size_t count) { void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { (void) plhs; - lock_mex_module(); + bsp_lock_mex_module(); - if (nrhs != 3) { + if (nrhs < 3 || nrhs > 4) { mexErrMsgIdAndTxt("BinSparse:InvalidArgs", "Usage: binsparse_write_string_dataset(filename, " - "dataset_name, value)"); + "dataset_name, value [, compression_level])"); } if (nlhs > 0) { @@ -109,14 +183,32 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { char* filename = get_required_string(prhs[0], "filename"); char* dataset_name = get_required_string(prhs[1], "dataset_name"); + int compression_level = get_compression_level(nrhs, prhs); + + const mxArray* value = prhs[2]; + bool is_cellstr = mxIsCell(value); + if ((!is_cellstr && !mxIsChar(value)) || mxGetNumberOfDimensions(value) > 2) { + mxFree(dataset_name); + mxFree(filename); + mexErrMsgIdAndTxt("BinSparse:InvalidValue", + "Value must be a two-dimensional char matrix or a cell " + "array of character vectors"); + } size_t count = 0; - bool scalar_dataset = false; - char** strings = collect_strings(prhs[2], &count, &scalar_dataset); + size_t width = 0; + char* fixed_buffer = NULL; + char** strings = NULL; + if (is_cellstr) { + strings = encode_cellstr(value, &count); + } else { + fixed_buffer = encode_char_matrix(value, &count, &width); + } hid_t file = H5I_INVALID_HID; hid_t type = H5I_INVALID_HID; hid_t space = H5I_INVALID_HID; + hid_t properties = H5P_DEFAULT; hid_t dset = H5I_INVALID_HID; const char* error_id = NULL; const char* error_message = NULL; @@ -143,27 +235,25 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { goto cleanup; } - type = H5Tcopy(H5T_C_S1); - if (type == H5I_INVALID_HID || H5Tset_size(type, H5T_VARIABLE) < 0 || - H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + type = is_cellstr ? bsp_variable_string_type() : bsp_fixed_string_type(width); + if (type == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; error_message = "Failed to create UTF-8 string datatype"; goto cleanup; } - if (scalar_dataset) { - space = H5Screate(H5S_SCALAR); - } else { - hsize_t dims[1] = {(hsize_t) count}; - space = H5Screate_simple(1, dims, NULL); - } + hsize_t dims[1] = {(hsize_t) count}; + space = H5Screate_simple(1, dims, NULL); if (space == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; error_message = "Failed to create string dataspace"; goto cleanup; } - dset = H5Dcreate2(file, dataset_name, type, space, H5P_DEFAULT, H5P_DEFAULT, + properties = bsp_text_dataset_properties( + count, is_cellstr ? sizeof(char*) : width, compression_level); + + dset = H5Dcreate2(file, dataset_name, type, space, H5P_DEFAULT, properties, H5P_DEFAULT); if (dset == H5I_INVALID_HID) { error_id = "BinSparse:HDF5Error"; @@ -171,16 +261,21 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { goto cleanup; } - if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, strings) < 0) { - error_id = "BinSparse:HDF5Error"; - error_message = "Failed to write string dataset"; - goto cleanup; + if (count > 0) { + const void* data = + is_cellstr ? (const void*) strings : (const void*) fixed_buffer; + if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) { + error_id = "BinSparse:HDF5Error"; + error_message = "Failed to write string dataset"; + goto cleanup; + } } cleanup: if (dset != H5I_INVALID_HID) { H5Dclose(dset); } + bsp_close_text_dataset_properties(properties); if (space != H5I_INVALID_HID) { H5Sclose(space); } @@ -190,6 +285,9 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) { if (file != H5I_INVALID_HID) { H5Fclose(file); } + if (fixed_buffer) { + mxFree(fixed_buffer); + } free_strings(strings, count); mxFree(dataset_name); mxFree(filename); diff --git a/bindings/matlab/compile_octave.sh b/bindings/matlab/compile_octave.sh index 7a32e7b..53e7f80 100755 --- a/bindings/matlab/compile_octave.sh +++ b/bindings/matlab/compile_octave.sh @@ -148,7 +148,7 @@ if [ "$CLEAN" = true ]; then fi # List of MEX files to compile -MEX_FILES=("binsparse_read.c" "binsparse_write.c" "binsparse_from_ssmc.c" "binsparse_minimize_types.c" "binsparse_write_string_dataset.c") +MEX_FILES=("binsparse_read.c" "binsparse_write.c" "binsparse_from_ssmc.c" "binsparse_minimize_types.c" "binsparse_write_string_dataset.c" "binsparse_read_string_dataset.c") print_info "Compiling MEX functions..." diff --git a/bindings/matlab/matlab_bsp_strings.h b/bindings/matlab/matlab_bsp_strings.h new file mode 100644 index 0000000..79ec7f8 --- /dev/null +++ b/bindings/matlab/matlab_bsp_strings.h @@ -0,0 +1,324 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * matlab_bsp_strings.h - shared helpers for the HDF5 text datasets that carry + * SuiteSparse Matrix Collection `aux` text components. + * + * Wire format + * ----------- + * A text component is a one-dimensional HDF5 string dataset with character set + * H5T_CSET_UTF8. The string datatype records which MATLAB class the component + * came from, so the file is self-describing and no side-channel metadata is + * needed to read it back: + * + * fixed-length (H5T_STR_NULLPAD) <-> MATLAB char matrix, m-by-n + * variable-length (H5T_VARIABLE) <-> MATLAB cellstr, m-by-1 + * + * A char matrix is rectangular, so every element of a fixed-length dataset + * holds one complete row: all n characters, including the trailing blanks + * MATLAB uses to pad short rows. The datatype size W is the largest UTF-8 + * encoding of a row, in bytes; rows that encode to fewer bytes are padded with + * NULs. For the ASCII text that makes up nearly all of the collection W is + * exactly n and no padding is stored at all, so the width is carried by the + * datatype and the round trip is exact without deblanking either side. + * + * NULs pad rather than blanks because HDF5 sizes a fixed-length string in + * bytes while MATLAB sizes a char matrix in characters. The two agree only + * for ASCII; blank padding would silently widen any row holding a multi-byte + * character. Stripping trailing NULs recovers the row's exact n characters + * whatever it contains. Trailing blanks are stripped as well when a foreign + * file declares H5T_STR_SPACEPAD, which is the Fortran convention. + * + * A cellstr is a ragged list rather than a rectangle, so it is stored + * variable-length and needs no padding convention. + */ + +#ifndef MATLAB_BSP_STRINGS_H +#define MATLAB_BSP_STRINGS_H + +#include "mex.h" +#include +#include +#include +#include +#include + +#define BSP_UTF_INVALID ((size_t) -1) + +/* Chunks are sized to about this many bytes when a text dataset is compressed. + */ +#define BSP_TEXT_CHUNK_BYTES ((size_t) (1024 * 1024)) + +/* Below this many bytes the chunk index costs more than the filter saves. */ +#define BSP_TEXT_MIN_COMPRESSED_BYTES ((size_t) 4096) + +/*---------------------------------------------------------------------------- + * UTF-16 (the encoding of a MATLAB mxChar) <-> UTF-8 + *--------------------------------------------------------------------------*/ + +// Number of UTF-8 bytes needed for len UTF-16 code units, or BSP_UTF_INVALID +// if the sequence contains an unpaired surrogate. +static inline size_t bsp_utf8_length(const mxChar* src, size_t len) { + size_t bytes = 0; + for (size_t i = 0; i < len; i++) { + uint32_t unit = src[i]; + if (unit < 0x80u) { + bytes += 1; + } else if (unit < 0x800u) { + bytes += 2; + } else if (unit >= 0xD800u && unit <= 0xDBFFu) { + if (i + 1 >= len || src[i + 1] < 0xDC00u || src[i + 1] > 0xDFFFu) { + return BSP_UTF_INVALID; + } + bytes += 4; + i++; + } else if (unit >= 0xDC00u && unit <= 0xDFFFu) { + return BSP_UTF_INVALID; + } else { + bytes += 3; + } + } + return bytes; +} + +// Encode len UTF-16 code units into out, which must hold bsp_utf8_length +// bytes. Returns the number of bytes written. The caller is responsible for +// having rejected unpaired surrogates via bsp_utf8_length. +static inline size_t bsp_utf8_encode(const mxChar* src, size_t len, char* out) { + unsigned char* p = (unsigned char*) out; + for (size_t i = 0; i < len; i++) { + uint32_t code = src[i]; + if (code >= 0xD800u && code <= 0xDBFFu && i + 1 < len && + src[i + 1] >= 0xDC00u && src[i + 1] <= 0xDFFFu) { + code = 0x10000u + ((code - 0xD800u) << 10) + (src[i + 1] - 0xDC00u); + i++; + } + if (code < 0x80u) { + *p++ = (unsigned char) code; + } else if (code < 0x800u) { + *p++ = (unsigned char) (0xC0u | (code >> 6)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } else if (code < 0x10000u) { + *p++ = (unsigned char) (0xE0u | (code >> 12)); + *p++ = (unsigned char) (0x80u | ((code >> 6) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } else { + *p++ = (unsigned char) (0xF0u | (code >> 18)); + *p++ = (unsigned char) (0x80u | ((code >> 12) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | ((code >> 6) & 0x3Fu)); + *p++ = (unsigned char) (0x80u | (code & 0x3Fu)); + } + } + return (size_t) ((char*) p - out); +} + +// Number of UTF-16 code units needed for len UTF-8 bytes, or BSP_UTF_INVALID +// if the bytes are not well-formed UTF-8. +static inline size_t bsp_utf16_length(const char* src, size_t len) { + const unsigned char* p = (const unsigned char*) src; + size_t units = 0; + size_t i = 0; + while (i < len) { + unsigned char lead = p[i]; + size_t trail; + uint32_t code; + if (lead < 0x80u) { + trail = 0; + code = lead; + } else if ((lead & 0xE0u) == 0xC0u) { + trail = 1; + code = lead & 0x1Fu; + } else if ((lead & 0xF0u) == 0xE0u) { + trail = 2; + code = lead & 0x0Fu; + } else if ((lead & 0xF8u) == 0xF0u) { + trail = 3; + code = lead & 0x07u; + } else { + return BSP_UTF_INVALID; + } + if (i + trail >= len) { + return BSP_UTF_INVALID; + } + for (size_t k = 1; k <= trail; k++) { + if ((p[i + k] & 0xC0u) != 0x80u) { + return BSP_UTF_INVALID; + } + code = (code << 6) | (uint32_t) (p[i + k] & 0x3Fu); + } + if (code > 0x10FFFFu) { + return BSP_UTF_INVALID; + } + units += (code >= 0x10000u) ? 2 : 1; + i += trail + 1; + } + return units; +} + +// Decode len UTF-8 bytes into out, which must hold bsp_utf16_length code +// units. Returns the number of code units written. The caller is responsible +// for having validated the bytes via bsp_utf16_length. +static inline size_t bsp_utf16_decode(const char* src, size_t len, + mxChar* out) { + const unsigned char* p = (const unsigned char*) src; + size_t units = 0; + size_t i = 0; + while (i < len) { + unsigned char lead = p[i]; + size_t trail; + uint32_t code; + if (lead < 0x80u) { + trail = 0; + code = lead; + } else if ((lead & 0xE0u) == 0xC0u) { + trail = 1; + code = lead & 0x1Fu; + } else if ((lead & 0xF0u) == 0xE0u) { + trail = 2; + code = lead & 0x0Fu; + } else { + trail = 3; + code = lead & 0x07u; + } + for (size_t k = 1; k <= trail; k++) { + code = (code << 6) | (uint32_t) (p[i + k] & 0x3Fu); + } + if (code >= 0x10000u) { + code -= 0x10000u; + out[units++] = (mxChar) (0xD800u + (code >> 10)); + out[units++] = (mxChar) (0xDC00u + (code & 0x3FFu)); + } else { + out[units++] = (mxChar) code; + } + i += trail + 1; + } + return units; +} + +// Convert a MATLAB char array to a NUL-terminated UTF-8 string allocated with +// mxMalloc. Returns NULL if the array is not char, holds an unpaired +// surrogate, or contains a NUL, which a C string cannot carry. Unlike +// mxArrayToString this preserves every character: mxArrayToString goes through +// the local code page and replaces anything it cannot represent with SUB +// (0x1A), which silently corrupts non-ASCII text. +static inline char* bsp_mx_to_utf8(const mxArray* value) { + if (!value || !mxIsChar(value)) { + return NULL; + } + + size_t len = mxGetNumberOfElements(value); + const mxChar* chars = (const mxChar*) mxGetData(value); + for (size_t i = 0; i < len; i++) { + if (chars[i] == 0) { + return NULL; + } + } + + size_t bytes = bsp_utf8_length(chars, len); + if (bytes == BSP_UTF_INVALID) { + return NULL; + } + + char* encoded = (char*) mxMalloc(bytes + 1); + bsp_utf8_encode(chars, len, encoded); + encoded[bytes] = '\0'; + return encoded; +} + +/*---------------------------------------------------------------------------- + * HDF5 datatype and property list helpers + *--------------------------------------------------------------------------*/ + +// Create the fixed-length, NUL-padded UTF-8 datatype used for char matrices. +static inline hid_t bsp_fixed_string_type(size_t width) { + hid_t type = H5Tcopy(H5T_C_S1); + if (type == H5I_INVALID_HID) { + return H5I_INVALID_HID; + } + if (H5Tset_size(type, width) < 0 || + H5Tset_strpad(type, H5T_STR_NULLPAD) < 0 || + H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + H5Tclose(type); + return H5I_INVALID_HID; + } + return type; +} + +// Create the variable-length UTF-8 datatype used for cellstr. +static inline hid_t bsp_variable_string_type(void) { + hid_t type = H5Tcopy(H5T_C_S1); + if (type == H5I_INVALID_HID) { + return H5I_INVALID_HID; + } + if (H5Tset_size(type, H5T_VARIABLE) < 0 || + H5Tset_cset(type, H5T_CSET_UTF8) < 0) { + H5Tclose(type); + return H5I_INVALID_HID; + } + return type; +} + +// Build a dataset creation property list that chunks and deflates a text +// dataset of count elements of element_size bytes. Returns H5P_DEFAULT when +// compression is not requested or when the dataset is too small for a chunk +// index to pay for itself. The threshold is on the total size rather than on +// the element count, so that one very wide row is still compressed. +// +// A fixed-length dataset compresses well: blank padding is highly redundant, +// and SNAP/wiki-topcats pagenames goes from 372.6 MB to 19.2 MB. A +// variable-length one barely moves, because HDF5 keeps the strings themselves +// on the global heap, which the filter pipeline does not reach; only the +// 16-byte-per-element descriptor array is filtered. Requesting compression is +// still worthwhile there, since that descriptor array grows with the element +// count, but the payload cannot be reached from here. +static inline hid_t bsp_text_dataset_properties(size_t count, + size_t element_size, + int compression_level) { + if (compression_level <= 0 || element_size == 0 || + count * element_size < BSP_TEXT_MIN_COMPRESSED_BYTES) { + return H5P_DEFAULT; + } + + size_t chunk = BSP_TEXT_CHUNK_BYTES / element_size; + if (chunk == 0) { + chunk = 1; + } + if (chunk > count) { + chunk = count; + } + + hid_t properties = H5Pcreate(H5P_DATASET_CREATE); + if (properties == H5I_INVALID_HID) { + return H5P_DEFAULT; + } + + hsize_t chunk_dims[1] = {(hsize_t) chunk}; + if (H5Pset_chunk(properties, 1, chunk_dims) < 0 || + H5Pset_deflate(properties, (unsigned) compression_level) < 0) { + H5Pclose(properties); + return H5P_DEFAULT; + } + return properties; +} + +static inline void bsp_close_text_dataset_properties(hid_t properties) { + if (properties != H5P_DEFAULT && properties != H5I_INVALID_HID) { + H5Pclose(properties); + } +} + +// Keep a MEX function loaded for the whole MATLAB session, and stop HDF5 from +// registering atexit handlers: both guard against crashes when MATLAB tears +// down MEX files that share HDF5 process-wide state. +static inline void bsp_lock_mex_module(void) { + if (!mexIsLocked()) { + H5dont_atexit(); + mexLock(); + } +} + +#endif // MATLAB_BSP_STRINGS_H diff --git a/bindings/matlab/test_binsparse_string_dataset.m b/bindings/matlab/test_binsparse_string_dataset.m new file mode 100644 index 0000000..57cca43 --- /dev/null +++ b/bindings/matlab/test_binsparse_string_dataset.m @@ -0,0 +1,152 @@ +function test_binsparse_string_dataset() +%TEST_BINSPARSE_STRING_DATASET round-trip tests for the HDF5 text datasets +% +% Checks that binsparse_write_string_dataset and binsparse_read_string_dataset +% preserve both the contents and the MATLAB class of a text component: a char +% matrix is stored fixed-length and a cellstr variable-length, so the two are +% told apart by the HDF5 string datatype alone. + +% SPDX-FileCopyrightText: 2026 Binsparse Developers +% +% SPDX-License-Identifier: BSD-3-Clause + +fprintf('=== Testing Binsparse string datasets ===\n\n'); + +if exist('binsparse_write_string_dataset', 'file') ~= 3 || ... + exist('binsparse_read_string_dataset', 'file') ~= 3 + error('test_binsparse_string_dataset:MissingMex', ... + 'The string dataset MEX functions are not on the path'); +end + +% Each case is {description, value}. The value must survive isequal. +% An empty cellstr element comes back as the 1-by-0 char that sstextread +% produces, not as the 0-by-0 char that '' denotes: an HDF5 string has a +% length but no MATLAB shape, and 1-by-0 is the row that sscellstring accepts. +cases = { + 'char matrix, ragged rows blank padded', char({'alpha'; 'be'; 'gamma!'}) + 'char matrix, every row full width', ['abcd'; 'efgh'; 'ijkl'] + 'char matrix, one row', 'a single row' + 'char matrix, blanks on every row', char({'ab '; 'cd '}) + 'char matrix, leading blanks kept', char({' indented'; 'flush'}) + 'char matrix, one row of blanks', char({'text'; ' '}) + 'char matrix, non-ASCII Latin-1', char({['Jo' 227 'o Pessoa']; ['Bras' 237 'lia']}) + 'char matrix, replacement characters', char({[65533 65533 'x']; 'plain'}) + 'char matrix, single column', char({'a'; 'b'; 'c'}) + 'cellstr, ragged', {'alpha'; 'be'; 'gamma!'} + 'cellstr, one element', {'only'} + 'cellstr, trailing blanks are content', {'ab '; 'cd'} + 'cellstr, empty element', {'first'; char(zeros(1, 0)); 'third'} + 'cellstr, non-ASCII', {['Jo' 227 'o']; ['Bras' 237 'lia']} + 'cellstr, long strings', {repmat('x', 1, 5000); 'short'} +}; + +tmpdir = tempname; +mkdir(tmpdir); +cleanup = onCleanup(@() rmdir(tmpdir, 's')); + +passed = 0; +failed = 0; +for level = [0 9] + fprintf('-- compression level %d\n', level); + for k = 1:size(cases, 1) + description = cases{k, 1}; + value = cases{k, 2}; + filename = fullfile(tmpdir, sprintf('case_%d_%d.h5', level, k)); + try + binsparse_write_string_dataset(filename, 'text', value, level); + actual = binsparse_read_string_dataset(filename, '/text'); + check_equal(actual, value, description); + fprintf(' PASS %s\n', description); + passed = passed + 1; + catch me + fprintf(' FAIL %s: %s\n', description, me.message); + failed = failed + 1; + end + end +end + +% The class must come from the datatype, not from the shape: a one-element +% cellstr and a one-row char matrix hold the same text but must not be +% confused with one another. +filename = fullfile(tmpdir, 'classes.h5'); +binsparse_write_string_dataset(filename, 'as_char', 'only'); +binsparse_write_string_dataset(filename, 'as_cell', {'only'}); +try + if ~ischar(binsparse_read_string_dataset(filename, '/as_char')) || ... + ~iscellstr(binsparse_read_string_dataset(filename, '/as_cell')) + error('test:ClassNotPreserved', 'class was not preserved'); + end + fprintf(' PASS one-row char and one-element cellstr stay distinct\n'); + passed = passed + 1; +catch me + fprintf(' FAIL one-row char vs one-element cellstr: %s\n', me.message); + failed = failed + 1; +end + +% A char matrix should not pay for the variable-length global heap, which no +% dataset filter reaches. A wide, mostly blank matrix must compress. +filename = fullfile(tmpdir, 'compressed.h5'); +wide = repmat(['padded row' blanks(500)], 400, 1); +binsparse_write_string_dataset(filename, 'text', wide, 9); +info = dir(filename); +if isequal(binsparse_read_string_dataset(filename, '/text'), wide) && ... + info.bytes < numel(wide) / 10 + fprintf(' PASS blank padding compresses (%d bytes for %d characters)\n', ... + info.bytes, numel(wide)); + passed = passed + 1; +else + fprintf(' FAIL blank padding did not compress (%d bytes)\n', info.bytes); + failed = failed + 1; +end + +% Compression is chosen on the total size, not the element count, so a single +% very wide row is compressed too. +filename = fullfile(tmpdir, 'one_row.h5'); +one_row = [repmat('abcdefgh', 1, 20000) blanks(20000)]; +binsparse_write_string_dataset(filename, 'text', one_row, 9); +info = dir(filename); +if isequal(binsparse_read_string_dataset(filename, '/text'), one_row) && ... + info.bytes < numel(one_row) / 10 + fprintf(' PASS a single wide row compresses (%d bytes for %d characters)\n', ... + info.bytes, numel(one_row)); + passed = passed + 1; +else + fprintf(' FAIL a single wide row did not compress (%d bytes for %d)\n', ... + info.bytes, numel(one_row)); + failed = failed + 1; +end + +% Below the threshold no chunk index is created, and the data still reads back. +filename = fullfile(tmpdir, 'tiny.h5'); +tiny = ['ab'; 'cd']; +binsparse_write_string_dataset(filename, 'text', tiny, 9); +if isequal(binsparse_read_string_dataset(filename, '/text'), tiny) + fprintf(' PASS a dataset below the compression threshold round-trips\n'); + passed = passed + 1; +else + fprintf(' FAIL a dataset below the compression threshold changed\n'); + failed = failed + 1; +end + +fprintf('\n%d passed, %d failed\n', passed, failed); +if failed > 0 + error('test_binsparse_string_dataset:Failed', ... + '%d string dataset test(s) failed', failed); +end +fprintf('=== All string dataset tests passed ===\n'); + +end + +function check_equal(actual, expected, description) +if ~strcmp(class(actual), class(expected)) + error('test:ClassMismatch', 'expected %s, got %s (%s)', ... + class(expected), class(actual), description); +end +if ~isequal(size(actual), size(expected)) + error('test:SizeMismatch', 'expected %s, got %s (%s)', ... + mat2str(size(expected)), mat2str(size(actual)), description); +end +if ~isequal(actual, expected) + error('test:ValueMismatch', 'contents differ (%s)', description); +end +end diff --git a/bindings/matlab/test_binsparse_to_ssmc_problem.m b/bindings/matlab/test_binsparse_to_ssmc_problem.m index 3f0b252..3f998bf 100644 --- a/bindings/matlab/test_binsparse_to_ssmc_problem.m +++ b/bindings/matlab/test_binsparse_to_ssmc_problem.m @@ -72,14 +72,23 @@ assert(isequal(Problem.x, [1 2; 3 4])); assert(iscell(Problem.aux.seq) && numel(Problem.aux.seq) == 2); assert(isequal(Problem.aux.seq{2}, [7; 8])); -assert(isequal(Problem.aux.label, char({'abc'; 'def'}))); +assert(isequal(Problem.aux.label, {'abc'; 'def'}), ... + 'cellstr component did not stay a cellstr'); -% A stripped string dataset rebuilds the char matrix it was written from, -% including the width carried by the widest row. +% The reader hands over the MATLAB class recorded by the HDF5 string datatype, +% and it survives the conversion untouched: a cellstr keeps its own trailing +% blanks and its ragged element lengths, and a char matrix keeps its width. raw.aux.label = {'hello '; 'there'}; Problem = binsparse_to_ssmc_problem(raw); -assert(isequal(Problem.aux.label, ['hello '; 'there ']), ... - 'stripped string dataset mismatch'); +assert(isequal(Problem.aux.label, {'hello '; 'there'}), ... + 'cellstr component was reshaped or deblanked'); + +raw.aux.label = ['hello '; 'there ']; +Problem = binsparse_to_ssmc_problem(raw); +assert(ischar(Problem.aux.label) && ... + isequal(Problem.aux.label, ['hello '; 'there ']), ... + 'char matrix component did not stay a char matrix'); +raw.aux.label = {'abc'; 'def'}; dmat = dense_matrix([1; 2; 3; 4; 5; 6], 2, 3, 'DMAT'); raw = struct('metadata', metadata, 'A', formats{1}, 'b', dmat); diff --git a/bindings/matlab/test_binsparse_write_ssmc_problem.m b/bindings/matlab/test_binsparse_write_ssmc_problem.m index f6ec3d2..b3d3f39 100644 --- a/bindings/matlab/test_binsparse_write_ssmc_problem.m +++ b/bindings/matlab/test_binsparse_write_ssmc_problem.m @@ -85,16 +85,13 @@ function test_binsparse_write_ssmc_problem() expected_sparse = full(Problem.aux.S); assert(matrices_equal(aux_sparse_mat, expected_sparse), 'Aux sparse matrix mismatch'); -% String datasets are stripped row by row. No row of note reaches the char -% matrix width, so the widest row keeps one blank; note2 needs none. Either -% way char() rebuilds the original char matrix exactly. -check_string_dataset(out_file, 'note', {'hello '; 'there'}); -check_string_dataset(out_file, 'note2', {'hello!'; 'there'}); -check_string_dataset(out_file, 'tags', {'alpha'; 'beta'}); -assert(isequal(char(as_cellstr(h5read(out_file, '/note'))), Problem.aux.note), ... - 'Aux note round trip is not exact'); -assert(isequal(char(as_cellstr(h5read(out_file, '/note2'))), Problem.aux.note2), ... - 'Aux note2 round trip is not exact'); +% Text is stored so that the HDF5 string datatype records the MATLAB class: a +% char matrix goes to a fixed-length dataset, keeping its width and its +% trailing blanks, and a cellstr to a variable-length one. Nothing is +% deblanked, so both come back exactly as written. +check_string_dataset(out_file, 'note', Problem.aux.note); +check_string_dataset(out_file, 'note2', Problem.aux.note2); +check_string_dataset(out_file, 'tags', Problem.aux.tags); json = h5readatt(out_file, '/', 'binsparse'); assert(contains(json, '"metadata"'), 'Primary metadata not nested'); @@ -128,6 +125,23 @@ function test_binsparse_write_ssmc_problem() assert(isequal(char(regexp(clean_notes, '\n', 'split')), clean.notes), ... 'Notes round trip is not exact'); +% The descriptor is UTF-8, so metadata keeps characters that the local code +% page cannot represent. Both of these turn up in the collection: an en dash +% in SNAP/wiki-RfA and a replacement character in Pajek/Journals. +wide = Problem; +wide.notes = char({['en dash ' char(8211) ' here']; ['replacement ' char(65533)]}); +wide.title = ['t' char(233) 'tle']; +wide_file = [tempname() '.bsp.h5']; +cleanup_wide = onCleanup(@() delete_if_exists(wide_file)); %#ok +binsparse_write_ssmc_problem(struct('Problem', wide), wide_file, ... + format, compression_level); +wide_meta = jsondecode(h5readatt(wide_file, '/', 'binsparse')); +wide_meta = wide_meta.metadata; +assert(isequal(char(regexp(wide_meta.notes, '\n', 'split')), wide.notes), ... + 'Non-ASCII notes were not preserved'); +assert(isequal(wide_meta.title, wide.title), ... + 'Non-ASCII title was not preserved'); + fprintf('Test passed.\n'); end @@ -147,8 +161,10 @@ function check_vector_shape(filename, group, expected) end function check_string_dataset(filename, name, expected) - actual = h5read(filename, ['/' name]); - actual = as_cellstr(actual); + actual = binsparse_read_string_dataset(filename, ['/' name]); + assert(strcmp(class(actual), class(expected)), ... + 'String dataset "%s" changed class from %s to %s', name, ... + class(expected), class(actual)); assert(isequal(actual, expected), 'String dataset "%s" mismatch', name); end @@ -163,18 +179,6 @@ function expect_error(action, identifier) error('Expected error %s', identifier); end -function value = as_cellstr(value) - if iscell(value) - value = value(:); - elseif isstring(value) - value = cellstr(value(:)); - elseif ischar(value) - value = cellstr(value); - else - error('Unexpected string dataset value type'); - end -end - function ok = matrices_equal(a, b) if ~isequal(size(a), size(b)) ok = false;