Skip to content

PERF: Keep fetchmany column metadata native and call-local #796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Jahnvi Thakkar (jahnvi480) wants to merge 1 commit into
base: main
Choose a base branch
from
+480 −28
Draft

PERF: Keep fetchmany column metadata native and call-local #796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
does not change the default provider or ship any Rust driver binaries.

### Changed
- `fetchmany()` keeps freshly described column types and sizes in call-local native
metadata instead of round-tripping them through Python dictionaries. Column names
retain eager Unicode conversion; public column descriptions and fetch behavior
are unchanged.
- `mssql-python` now depends on `mssql-python-rs==0.1.0` for `mssql_py_core`
instead of embedding files owned by that separately published distribution.
- **GH-769 deprecation policy:** The misplaced `GetInfoConstants` members
Expand Down
117 changes: 89 additions & 28 deletions mssql_python/pybind/ddbc_bindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3022,9 +3022,51 @@ SQLSMALLINT SQLNumResultCols_wrap(SqlHandlePtr statementHandle) {
return columnCount;
}

// Wrap SQLDescribeCol
SQLRETURN SQLDescribeCol_wrap(SqlHandlePtr StatementHandle, py::list& ColumnMetadata) {
PERF_TIMER("SQLDescribeCol_wrap");
namespace {

struct FetchColumnMetadata {
py::object name;
SQLSMALLINT dataType;
SQLULEN columnSize;
SQLSMALLINT decimalDigits;
SQLSMALLINT nullable;
};

py::dict GetFetchColumnMetadata(const py::list& columns, size_t index) {
return columns[index].cast<py::dict>();
}

const FetchColumnMetadata& GetFetchColumnMetadata(
const std::vector<FetchColumnMetadata>& columns, size_t index) {
return columns.at(index);
}

SQLSMALLINT GetFetchColumnType(const py::dict& column) {
return column["DataType"].cast<SQLSMALLINT>();
}

SQLSMALLINT GetFetchColumnType(const FetchColumnMetadata& column) {
return column.dataType;
}

SQLULEN GetFetchColumnSize(const py::dict& column) {
return column["ColumnSize"].cast<SQLULEN>();
}

SQLULEN GetFetchColumnSize(const FetchColumnMetadata& column) {
return column.columnSize;
}

std::string GetFetchColumnName(const py::dict& column) {
return column["ColumnName"].cast<std::string>();
}

std::string GetFetchColumnName(const FetchColumnMetadata& column) {
return column.name.cast<std::string>();
}

template <typename AppendColumn>
SQLRETURN DescribeColumns(SqlHandlePtr StatementHandle, AppendColumn&& appendColumn) {
LOG("SQLDescribeCol: Getting column descriptions for statement_handle=%p",
(void*)StatementHandle->get());
if (!SQLDescribeCol_ptr) {
Expand Down Expand Up @@ -3052,21 +3094,32 @@ SQLRETURN SQLDescribeCol_wrap(SqlHandlePtr StatementHandle, py::list& ColumnMeta
&ColumnSize, &DecimalDigits, &Nullable);

if (SQL_SUCCEEDED(retcode)) {
// Append a named py::dict to ColumnMetadata
// TODO: Should we define a struct for this task instead of dict?
ColumnMetadata.append(
py::dict("ColumnName"_a = dupeSqlWCharAsUtf16Le(
ColumnName, std::min(static_cast<size_t>(NameLength),
(sizeof(ColumnName) / sizeof(SQLWCHAR)) - 1)),
"DataType"_a = DataType, "ColumnSize"_a = ColumnSize,
"DecimalDigits"_a = DecimalDigits, "Nullable"_a = Nullable));
// Own the name and preserve eager UTF-16 conversion, including codec errors.
auto name = py::cast(dupeSqlWCharAsUtf16Le(
ColumnName, std::min(static_cast<size_t>(NameLength),
(sizeof(ColumnName) / sizeof(SQLWCHAR)) - 1)));
appendColumn(FetchColumnMetadata{
std::move(name), DataType, ColumnSize, DecimalDigits, Nullable});
} else {
return retcode;
}
}
return SQL_SUCCESS;
}

} // namespace

// Wrap SQLDescribeCol
SQLRETURN SQLDescribeCol_wrap(SqlHandlePtr StatementHandle, py::list& ColumnMetadata) {
PERF_TIMER("SQLDescribeCol_wrap");
return DescribeColumns(StatementHandle, [&](FetchColumnMetadata column) {
ColumnMetadata.append(
py::dict("ColumnName"_a = column.name, "DataType"_a = column.dataType,
"ColumnSize"_a = column.columnSize, "DecimalDigits"_a = column.decimalDigits,
"Nullable"_a = column.nullable));
});
}

SQLRETURN SQLSpecialColumns_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT identifierType,
const py::object& catalogObj, const py::object& schemaObj,
const std::u16string& table, SQLSMALLINT scope,
Expand Down Expand Up @@ -3999,16 +4052,17 @@ SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOri

// For column in the result set, binds a buffer to retrieve column data
// TODO: Move to anonymous namespace, since it is not used outside this file
SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& columnNames,
template <typename Metadata>
SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& columnNames,
SQLUSMALLINT numCols, int fetchSize, int charCtype = SQL_C_WCHAR) {
PERF_TIMER("SQLBindColums");
SQLRETURN ret = SQL_SUCCESS;
const bool useWideChar = (charCtype == SQL_C_WCHAR);
// Bind columns based on their data types
for (SQLUSMALLINT col = 1; col <= numCols; col++) {
auto columnMeta = columnNames[col - 1].cast<py::dict>();
SQLSMALLINT dataType = columnMeta["DataType"].cast<SQLSMALLINT>();
SQLULEN columnSize = columnMeta["ColumnSize"].cast<SQLULEN>();
const auto& columnMeta = GetFetchColumnMetadata(columnNames, col - 1);
SQLSMALLINT dataType = GetFetchColumnType(columnMeta);
SQLULEN columnSize = GetFetchColumnSize(columnMeta);

switch (dataType) {
case SQL_CHAR:
Expand Down Expand Up @@ -4140,7 +4194,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column
buffers.indicators[col - 1].data());
break;
default:
std::string columnName = columnMeta["ColumnName"].cast<std::string>();
std::string columnName = GetFetchColumnName(columnMeta);
std::ostringstream errorString;
errorString << "Unsupported data type for column - " << columnName.c_str()
<< ", Type - " << dataType << ", column ID - " << col;
Expand All @@ -4149,7 +4203,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column
break;
}
if (!SQL_SUCCEEDED(ret)) {
std::string columnName = columnMeta["ColumnName"].cast<std::string>();
std::string columnName = GetFetchColumnName(columnMeta);
std::ostringstream errorString;
errorString << "Failed to bind column - " << columnName.c_str() << ", Type - "
<< dataType << ", column ID - " << col;
Expand All @@ -4163,7 +4217,8 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column

// Fetch rows in batches
// TODO: Move to anonymous namespace, since it is not used outside this file
SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& columnNames,
template <typename Metadata>
SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, const Metadata& columnNames,
py::list& rows, SQLUSMALLINT numCols, SQLULEN& numRowsFetched,
const std::vector<SQLUSMALLINT>& lobColumns,
const std::string& charEncoding = "utf-16le",
Expand Down Expand Up @@ -4213,9 +4268,9 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
{
PERF_TIMER("FetchBatchData::cache_column_metadata");
for (SQLUSMALLINT col = 0; col < numCols; col++) {
const auto& columnMeta = columnNames[col].cast<py::dict>();
columnInfos[col].dataType = columnMeta["DataType"].cast<SQLSMALLINT>();
columnInfos[col].columnSize = columnMeta["ColumnSize"].cast<SQLULEN>();
const auto& columnMeta = GetFetchColumnMetadata(columnNames, col);
columnInfos[col].dataType = GetFetchColumnType(columnMeta);
columnInfos[col].columnSize = GetFetchColumnSize(columnMeta);
columnInfos[col].isLob =
std::find(lobColumns.begin(), lobColumns.end(), col + 1) != lobColumns.end();
columnInfos[col].processedColumnSize = columnInfos[col].columnSize;
Expand Down Expand Up @@ -4504,8 +4559,8 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
break;
}
default: {
const auto& columnMeta = columnNames[col - 1].cast<py::dict>();
std::string columnName = columnMeta["ColumnName"].cast<std::string>();
const auto& columnMeta = GetFetchColumnMetadata(columnNames, col - 1);
std::string columnName = GetFetchColumnName(columnMeta);
std::ostringstream errorString;
errorString << "Unsupported data type for column - " << columnName.c_str()
<< ", Type - " << dataType << ", column ID - " << col;
Expand Down Expand Up @@ -4650,18 +4705,24 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch
SQLSMALLINT numCols = SQLNumResultCols_wrap(StatementHandle);

// Retrieve column metadata
py::list columnNames;
ret = SQLDescribeCol_wrap(StatementHandle, columnNames);
std::vector<FetchColumnMetadata> columnNames;
ret = DescribeColumns(StatementHandle, [&](FetchColumnMetadata column) {
columnNames.push_back(std::move(column));
});
if (!SQL_SUCCEEDED(ret)) {
LOG("FetchMany_wrap: Failed to get column descriptions - SQLRETURN=%d", ret);
return ret;
}
if (numCols < 0 || columnNames.size() != static_cast<size_t>(numCols)) {
LOG("FetchMany_wrap: Column metadata count does not match result column count");
ThrowStdException("Column metadata count does not match result column count");
}

std::vector<SQLUSMALLINT> lobColumns;
for (SQLSMALLINT i = 0; i < numCols; i++) {
auto colMeta = columnNames[i].cast<py::dict>();
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
const auto& colMeta = GetFetchColumnMetadata(columnNames, i);
SQLSMALLINT dataType = GetFetchColumnType(colMeta);
SQLULEN columnSize = GetFetchColumnSize(colMeta);

if (IsLobOrVariantColumn(dataType, columnSize)) {
lobColumns.push_back(i + 1); // 1-based
Expand Down
Loading
Loading