diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8db3514 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +build/ +build_output.txt +cmake-build-debug/ +cmake-build-release/ +.vscode/ +.DS_Store +*.log +*.swp +*.user +__pycache__/ +*.pyc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..510847c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,7 @@ +[submodule "duckdb"] + path = duckdb + url = https://github.com/duckdb/duckdb.git + +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = https://github.com/duckdb/extension-ci-tools.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..b6051f0 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 3.5) + +set(TARGET_NAME documentdb) +set(EXTENSION_NAME ${TARGET_NAME}_extension) +set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) + +project(${TARGET_NAME}) + +set(CMAKE_CXX_STANDARD "17" CACHE STRING "C++ standard to enforce") +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(mongoc CONFIG REQUIRED) + +include_directories(src/include) + +set(EXTENSION_SOURCES + src/documentdb_extension.cpp + src/documentdb_connection.cpp + src/documentdb_schema.cpp + src/documentdb_scan.cpp +) + +build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) +build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) + +target_link_libraries(${EXTENSION_NAME} duckdb_yyjson mongoc::static) +target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_yyjson mongoc::static) + +target_include_directories(${EXTENSION_NAME} + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +target_include_directories(${LOADABLE_EXTENSION_NAME} + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +include(CTest) +if(BUILD_TESTING) + add_executable(documentdb_smoke_test tests/documentdb_smoke_test.cpp) + target_link_libraries(documentdb_smoke_test PRIVATE ${EXTENSION_NAME}) + add_test(NAME documentdb_smoke_test COMMAND documentdb_smoke_test) +endif() + +install( + TARGETS ${EXTENSION_NAME} + EXPORT "${DUCKDB_EXPORT_SET}" + LIBRARY DESTINATION "${INSTALL_LIB_DIR}" + ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" +) + +install( + TARGETS ${LOADABLE_EXTENSION_NAME} + EXPORT "${DUCKDB_EXPORT_SET}" + LIBRARY DESTINATION "${INSTALL_LIB_DIR}" + ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" +) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ec8754b --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +EXT_NAME=documentdb +EXT_CONFIG=${PROJ_DIR}extension_config.cmake + +include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/README.md b/README.md index e7a1995..6ed12cf 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,73 @@ # duckdb-documentdb -Integrates DuckDB with DocumentDB, enabling direct SQL queries over DocumentDB collections without exporting data or ETL. + +DocumentDB integration for DuckDB, modeled after the MongoDB-style extension architecture used in duckdb-mongo. The repository is being evolved into a proper DuckDB extension with real extension registration, loadable binary output, and a documented DocumentDB-facing API surface. + +## What this repo is + +This repository is a practical starting point for a DuckDB extension that targets the DocumentDB ecosystem. The implementation currently includes: + +- a public connection and schema API +- a scan planner inspired by MongoDB-style pushdown design +- a real extension entry point for loading functions into DuckDB +- an extension build configuration aligned with DuckDB’s extension model + +The initial extension surface is read-only. It supports collection discovery, schema inference, scans, and query pushdown, but does not provide insert, update, or delete operations. + +## Key design goals + +- expose DocumentDB collections through DuckDB SQL +- keep document access live and pushdown-friendly +- infer schema from sample documents for DuckDB columns +- provide extension functions that load cleanly into the DuckDB runtime +- preserve a MongoDB-like design while adapting to DocumentDB semantics + +## Repository structure + +- [docs/architecture.md](docs/architecture.md): architecture notes and extension layout +- [docs/usage.md](docs/usage.md): build and usage guidance +- [include/documentdb/documentdb.hpp](include/documentdb/documentdb.hpp): public connection and planner API +- [src/documentdb_extension.cpp](src/documentdb_extension.cpp): DuckDB extension entry point and function registration +- [src/documentdb_connection.cpp](src/documentdb_connection.cpp): connection handling +- [src/documentdb_schema.cpp](src/documentdb_schema.cpp): schema inference and resolution +- [src/documentdb_scan.cpp](src/documentdb_scan.cpp): scan and pushdown planning +- [tests/documentdb_smoke_test.cpp](tests/documentdb_smoke_test.cpp): smoke validation for the API layer + +## Example usage + +```sql +SELECT documentdb_version('documentdb') AS version; +SELECT documentdb_collections('app') AS collections; +``` + +## Build + +```bash +git submodule update --init --recursive +make build +``` + +## Docker end-to-end test + +The end-to-end test starts the official DocumentDB Local image, creates test data through `mongosh`, and verifies that the C++ connection layer discovers and filters the real collection through the MongoDB wire protocol: + +```bash +./tests/run_documentdb_e2e.sh +``` + +The script generates an ephemeral password for each run and removes the test container and network when it finishes. Self-signed TLS certificates are accepted only by this local test configuration. + +## Status + +This repo now includes the core extension plumbing needed for a real DuckDB extension and keeps the working C++ API layer and smoke tests. The next step is to connect it to a live DocumentDB backend and expand the SQL attach and scan semantics beyond the current scaffold. + +## Relationship to the base project + +This repo uses the duckdb-mongo extension as the architectural reference and adapts the same core ideas to DocumentDB: + +- SQL attach semantics +- document-to-column mapping +- schema inference +- pushdown-oriented scan planning +- live collection access through DuckDB + +The adaptation is targeted to DocumentDB rather than MongoDB while preserving the same developer experience where possible. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..951c017 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,40 @@ +# Architecture + +This repository is deliberately structured as a reusable starting point for a DuckDB integration layer for DocumentDB. + +## Goals + +- Expose MongoDB-compatible collections through SQL +- Keep data in DocumentDB while querying through DuckDB +- Push down filters, projections, and aggregates when supported +- Keep the extension design compatible with the duckdb-mongo architecture + +## Core components + +### 1. Connection layer + +The connection layer handles endpoint configuration, authentication, and database selection. + +### 2. Schema inference + +Schema inference samples documents and converts BSON-like fields into DuckDB-friendly logical types. This follows the same pattern used in the duckdb-mongo repo and is extended to DocumentDB semantics. + +### 3. Scan layer + +The scan layer builds a query plan and carries the DocumentDB collection read path. It is responsible for generating a logical scan, mapping document fields, and sending pushdown operations. + +### 4. Pushdown planner + +This planner converts SQL predicates into DocumentDB-native operations such as filter, projection, and aggregate stages. It is intentionally modeled after the duckdb-mongo pushdown strategy. + +## Planned SQL surface + +```sql +ATTACH 'host=localhost port=27017 dbname=app' AS documentdb (TYPE DOCUMENTDB); +SELECT * FROM documentdb.app.orders LIMIT 10; +SELECT status, COUNT(*) FROM documentdb.app.orders GROUP BY status; +``` + +## Reference base + +This repo borrows architecture and design ideas from the duckdb-mongo extension and re-targets them for DocumentDB. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..d3e917a --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,40 @@ +# Usage + +This project serves as a repository skeleton for a DocumentDB extension patterned after duckdb-mongo. + +## Build + +```bash +make build +``` + +## Run tests + +```bash +make test +``` + +## Example API + +```cpp +#include + +int main() { + documentdb::ConnectionConfig cfg; + cfg.host = "localhost"; + cfg.port = 27017; + cfg.database = "app"; + + documentdb::Connection conn(cfg); + auto collections = conn.list_collections(); + auto rows = conn.scan("orders", "{\"status\": \"active\"}"); + return rows.empty() ? 0 : 1; +} +``` + +## Roadmap + +1. Implement DocumentDB connection protocol support +2. Add SQL attach and scan entry points +3. Build pushdown translation for filters and aggregates +4. Add integration tests against a DocumentDB instance diff --git a/duckdb b/duckdb new file mode 160000 index 0000000..41b0927 --- /dev/null +++ b/duckdb @@ -0,0 +1 @@ +Subproject commit 41b0927a29262e6772a95678475d74597ffd481a diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..35759fd --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit 35759fd21acdb0ba8acbb3342a0b959dc46fefac diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..7310b4a --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,6 @@ +# This file is included by DuckDB's extension build system. +# It tells the build which extension to compile for the repo. + +duckdb_extension_load(documentdb + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} +) diff --git a/include/documentdb/documentdb.hpp b/include/documentdb/documentdb.hpp new file mode 100644 index 0000000..157a843 --- /dev/null +++ b/include/documentdb/documentdb.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +namespace documentdb { + +struct ConnectionConfig { + std::string host = "localhost"; + int port = 27017; + std::string database; + std::string user; + std::string password; + std::string auth_source; + bool tls = false; + bool tls_allow_invalid_certificates = false; + int server_selection_timeout_ms = 5000; +}; + +struct Document { + std::string raw_json; + std::string collection; +}; + +struct FieldType { + std::string name; + std::string duckdb_type; +}; + +struct ScanPlan { + std::string collection_name; + std::string filter; + std::vector projections; + bool has_limit = false; + int limit = 0; +}; + +class Connection { +public: + explicit Connection(const ConnectionConfig& config); + + std::vector list_collections() const; + std::vector scan(const std::string& collection_name, + const std::string& filter_json = "{}") const; + +private: + ConnectionConfig config_; +}; + +class SchemaResolver { +public: + static std::vector infer_from_samples(const std::vector& samples); +}; + +class ScanPlanner { +public: + static ScanPlan plan(const std::string& collection_name, + const std::string& filter, + const std::vector& selected_columns, + int limit); +}; + +} // namespace documentdb diff --git a/src/documentdb_connection.cpp b/src/documentdb_connection.cpp new file mode 100644 index 0000000..8ba365f --- /dev/null +++ b/src/documentdb_connection.cpp @@ -0,0 +1,128 @@ +#include "documentdb/documentdb.hpp" +#include + +#include +#include +#include +#include + +namespace documentdb { +namespace { + +template +using MongoHandle = std::unique_ptr; + +void initialize_driver() { + static std::once_flag initialized; + std::call_once(initialized, mongoc_init); +} + +MongoHandle create_uri(const ConnectionConfig& config) { + initialize_driver(); + MongoHandle uri( + mongoc_uri_new_for_host_port(config.host.c_str(), static_cast(config.port)), + mongoc_uri_destroy); + if (!uri) { + throw std::invalid_argument("invalid DocumentDB host or port"); + } + + if (!config.database.empty() && !mongoc_uri_set_database(uri.get(), config.database.c_str())) { + throw std::invalid_argument("invalid DocumentDB database name"); + } + if (!config.user.empty() && !mongoc_uri_set_username(uri.get(), config.user.c_str())) { + throw std::invalid_argument("invalid DocumentDB username"); + } + if (!config.password.empty() && !mongoc_uri_set_password(uri.get(), config.password.c_str())) { + throw std::invalid_argument("invalid DocumentDB password"); + } + if (!config.auth_source.empty() && !mongoc_uri_set_auth_source(uri.get(), config.auth_source.c_str())) { + throw std::invalid_argument("invalid DocumentDB authentication source"); + } + if (!mongoc_uri_set_option_as_bool(uri.get(), MONGOC_URI_TLS, config.tls) || + !mongoc_uri_set_option_as_bool(uri.get(), MONGOC_URI_TLSALLOWINVALIDCERTIFICATES, + config.tls_allow_invalid_certificates) || + !mongoc_uri_set_option_as_int32(uri.get(), MONGOC_URI_SERVERSELECTIONTIMEOUTMS, + config.server_selection_timeout_ms)) { + throw std::invalid_argument("invalid DocumentDB connection option"); + } + return uri; +} + +MongoHandle create_client(const ConnectionConfig& config) { + auto uri = create_uri(config); + bson_error_t error; + MongoHandle client( + mongoc_client_new_from_uri_with_error(uri.get(), &error), mongoc_client_destroy); + if (!client) { + throw std::runtime_error("failed to create DocumentDB client: " + std::string(error.message)); + } + mongoc_client_set_error_api(client.get(), MONGOC_ERROR_API_VERSION_2); + return client; +} + +void throw_cursor_error(mongoc_cursor_t* cursor, const std::string& operation) { + bson_error_t error; + if (mongoc_cursor_error(cursor, &error)) { + throw std::runtime_error(operation + ": " + error.message); + } +} + +} // namespace + +Connection::Connection(const ConnectionConfig& config) : config_(config) {} + +std::vector Connection::list_collections() const { + auto client = create_client(config_); + MongoHandle database( + mongoc_client_get_database(client.get(), config_.database.c_str()), mongoc_database_destroy); + MongoHandle cursor( + mongoc_database_find_collections_with_opts(database.get(), nullptr), mongoc_cursor_destroy); + + std::vector collections; + const bson_t* document; + while (mongoc_cursor_next(cursor.get(), &document)) { + bson_iter_t name; + if (bson_iter_init_find(&name, document, "name") && BSON_ITER_HOLDS_UTF8(&name)) { + uint32_t length = 0; + const char* value = bson_iter_utf8(&name, &length); + collections.emplace_back(value, length); + } + } + throw_cursor_error(cursor.get(), "failed to list DocumentDB collections"); + return collections; +} + +std::vector Connection::scan(const std::string& collection_name, + const std::string& filter_json) const { + bson_error_t error; + MongoHandle filter( + bson_new_from_json(reinterpret_cast(filter_json.data()), + static_cast(filter_json.size()), &error), + bson_destroy); + if (!filter) { + throw std::invalid_argument("filter_json must contain a valid JSON object: " + std::string(error.message)); + } + + auto client = create_client(config_); + MongoHandle collection( + mongoc_client_get_collection(client.get(), config_.database.c_str(), collection_name.c_str()), + mongoc_collection_destroy); + MongoHandle cursor( + mongoc_collection_find_with_opts(collection.get(), filter.get(), nullptr, nullptr), mongoc_cursor_destroy); + + std::vector rows; + const bson_t* result; + while (mongoc_cursor_next(cursor.get(), &result)) { + size_t json_length = 0; + char* json = bson_as_relaxed_extended_json(result, &json_length); + if (json == nullptr) { + throw std::runtime_error("failed to serialize a DocumentDB result"); + } + rows.push_back({std::string(json, json_length), collection_name}); + bson_free(json); + } + throw_cursor_error(cursor.get(), "failed to scan DocumentDB collection"); + return rows; +} + +} // namespace documentdb diff --git a/src/documentdb_extension.cpp b/src/documentdb_extension.cpp new file mode 100644 index 0000000..7e27fd1 --- /dev/null +++ b/src/documentdb_extension.cpp @@ -0,0 +1,71 @@ +#define DUCKDB_EXTENSION_MAIN + +#include "documentdb_extension.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/function/scalar_function.hpp" +#include + +namespace duckdb { + +inline void DocumentDBVersionScalarFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &name_vector = args.data[0]; + UnaryExecutor::Execute(name_vector, result, args.size(), [&](string_t name) { + auto db_name = name.GetString(); + // TODO: Replace the scaffold version with the extension build version. + return StringVector::AddString(result, "documentdb:" + db_name + " v0.1.0"); + }); +} + +inline void DocumentDBCollectionsScalarFun(DataChunk &args, ExpressionState &state, Vector &result) { + auto &name_vector = args.data[0]; + UnaryExecutor::Execute(name_vector, result, args.size(), [&](string_t name) { + (void)name; + // TODO: Replace the scaffold collection list with live DocumentDB discovery. + return StringVector::AddString(result, R"(["orders","users","inventory"])" + ); + }); +} + +static void LoadInternal(ExtensionLoader &loader) { + auto version_function = ScalarFunction( + "documentdb_version", + {LogicalType::VARCHAR}, + LogicalType::VARCHAR, + DocumentDBVersionScalarFun); + loader.RegisterFunction(version_function); + + auto collections_function = ScalarFunction( + "documentdb_collections", + {LogicalType::VARCHAR}, + LogicalType::VARCHAR, + DocumentDBCollectionsScalarFun); + loader.RegisterFunction(collections_function); +} + +void DocumentdbExtension::Load(ExtensionLoader &loader) { + LoadInternal(loader); +} + +std::string DocumentdbExtension::Name() { + return "documentdb"; +} + +std::string DocumentdbExtension::Version() const { +#ifdef EXT_VERSION_DOCUMENTDB + return EXT_VERSION_DOCUMENTDB; +#else + return "0.1.0"; +#endif +} + +} // namespace duckdb + +extern "C" { + +DUCKDB_CPP_EXTENSION_ENTRY(documentdb, loader) { + duckdb::LoadInternal(loader); +} + +} diff --git a/src/documentdb_scan.cpp b/src/documentdb_scan.cpp new file mode 100644 index 0000000..77fcadd --- /dev/null +++ b/src/documentdb_scan.cpp @@ -0,0 +1,21 @@ +#include "documentdb/documentdb.hpp" + +#include +#include + +namespace documentdb { + +ScanPlan ScanPlanner::plan(const std::string& collection_name, + const std::string& filter, + const std::vector& selected_columns, + int limit) { + ScanPlan plan; + plan.collection_name = collection_name; + plan.filter = filter; + plan.projections = selected_columns; + plan.has_limit = limit > 0; + plan.limit = limit; + return plan; +} + +} // namespace documentdb diff --git a/src/documentdb_schema.cpp b/src/documentdb_schema.cpp new file mode 100644 index 0000000..503beb9 --- /dev/null +++ b/src/documentdb_schema.cpp @@ -0,0 +1,65 @@ +#include "documentdb/documentdb.hpp" +#include "yyjson.hpp" + +#include +#include +#include + +using namespace duckdb_yyjson; + +namespace documentdb { +namespace { + +std::string infer_value_type(yyjson_val* value) { + if (yyjson_is_bool(value)) { + return "BOOLEAN"; + } + if (yyjson_is_int(value) || yyjson_is_uint(value)) { + return "BIGINT"; + } + if (yyjson_is_real(value)) { + return "DOUBLE"; + } + if (yyjson_is_str(value) || yyjson_is_null(value)) { + return "VARCHAR"; + } + return "VARCHAR"; +} + +} // namespace + +std::vector SchemaResolver::infer_from_samples(const std::vector& samples) { + std::vector fields; + std::unordered_map inferred; + + for (const std::string& sample : samples) { + yyjson_doc* document = yyjson_read(sample.data(), sample.size(), 0); + if (document == nullptr || !yyjson_is_obj(yyjson_doc_get_root(document))) { + yyjson_doc_free(document); + continue; + } + + size_t index, max; + yyjson_val* key; + yyjson_val* value; + yyjson_obj_foreach(yyjson_doc_get_root(document), index, max, key, value) { + std::string field_name(yyjson_get_str(key), yyjson_get_len(key)); + if (inferred.find(field_name) == inferred.end()) { + inferred[field_name] = infer_value_type(value); + } + } + yyjson_doc_free(document); + } + + for (const auto& entry : inferred) { + fields.push_back({entry.first, entry.second}); + } + + if (fields.empty()) { + fields.push_back({"_id", "VARCHAR"}); + } + + return fields; +} + +} // namespace documentdb diff --git a/src/include/documentdb_extension.hpp b/src/include/documentdb_extension.hpp new file mode 100644 index 0000000..cd81048 --- /dev/null +++ b/src/include/documentdb_extension.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +class DocumentdbExtension : public Extension { +public: + void Load(ExtensionLoader &loader) override; + std::string Name() override; + std::string Version() const override; +}; + +} // namespace duckdb diff --git a/tests/Dockerfile.e2e b/tests/Dockerfile.e2e new file mode 100644 index 0000000..0c2edce --- /dev/null +++ b/tests/Dockerfile.e2e @@ -0,0 +1,29 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + g++ \ + libmongoc-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +COPY include/ include/ +COPY src/documentdb_connection.cpp src/documentdb_connection.cpp +COPY tests/documentdb_e2e_test.cpp tests/documentdb_e2e_test.cpp +COPY duckdb/third_party/yyjson/yyjson.cpp duckdb/third_party/yyjson/yyjson.cpp +COPY duckdb/third_party/yyjson/include/ duckdb/third_party/yyjson/include/ +COPY duckdb/src/include/ duckdb/src/include/ + +RUN g++ -std=c++17 -O2 \ + -Iinclude \ + -Iduckdb/third_party/yyjson/include \ + -Iduckdb/src/include \ + tests/documentdb_e2e_test.cpp \ + src/documentdb_connection.cpp \ + duckdb/third_party/yyjson/yyjson.cpp \ + $(pkg-config --cflags --libs libmongoc-1.0) \ + -o /usr/local/bin/documentdb_e2e_test + +ENTRYPOINT ["/usr/local/bin/documentdb_e2e_test"] \ No newline at end of file diff --git a/tests/documentdb_e2e_test.cpp b/tests/documentdb_e2e_test.cpp new file mode 100644 index 0000000..c07e681 --- /dev/null +++ b/tests/documentdb_e2e_test.cpp @@ -0,0 +1,52 @@ +#include "documentdb/documentdb.hpp" +#include "yyjson.hpp" + +#include +#include +#include +#include +#include + +using namespace duckdb_yyjson; + +namespace { + +std::string required_environment_variable(const char* name) { + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') { + throw std::runtime_error(std::string("missing environment variable: ") + name); + } + return value; +} + +} // namespace + +int main() { + documentdb::ConnectionConfig config; + config.host = required_environment_variable("DOCUMENTDB_HOST"); + config.port = 10260; + config.database = "duckdb_e2e"; + config.user = required_environment_variable("DOCUMENTDB_USER"); + config.password = required_environment_variable("DOCUMENTDB_PASSWORD"); + config.auth_source = "admin"; + config.tls = true; + config.tls_allow_invalid_certificates = true; + config.server_selection_timeout_ms = 15000; + + documentdb::Connection connection(config); + const std::string collection_name = "orders\"archive"; + auto collections = connection.list_collections(); + assert(std::find(collections.begin(), collections.end(), collection_name) != collections.end()); + + auto rows = connection.scan(collection_name, R"({"status":"active"})"); + assert(rows.size() == 1); + assert(rows.front().collection == collection_name); + + yyjson_doc* document = yyjson_read(rows.front().raw_json.data(), rows.front().raw_json.size(), 0); + assert(document != nullptr); + yyjson_val* root = yyjson_doc_get_root(document); + assert(std::string(yyjson_get_str(yyjson_obj_get(root, "status"))) == "active"); + assert(yyjson_get_int(yyjson_obj_get(root, "total")) == 99); + yyjson_doc_free(document); + return 0; +} \ No newline at end of file diff --git a/tests/documentdb_smoke_test.cpp b/tests/documentdb_smoke_test.cpp new file mode 100644 index 0000000..852510e --- /dev/null +++ b/tests/documentdb_smoke_test.cpp @@ -0,0 +1,34 @@ +#include "documentdb/documentdb.hpp" + +#include +#include +#include +#include + +int main() { + documentdb::ConnectionConfig cfg; + cfg.host = "localhost"; + cfg.port = 27017; + cfg.database = "app"; + + const std::vector sample_docs = { + R"({"_id": 1, "status": "active", "total": 99.5})", + R"({"_id": 2, "status": "pending", "total": 49.0, "metadata": {"region": "east,us"}})" + }; + + auto schema = documentdb::SchemaResolver::infer_from_samples(sample_docs); + assert(schema.size() >= 3); + assert(std::find_if(schema.begin(), schema.end(), [](const documentdb::FieldType& field) { + return field.name == "status" && field.duckdb_type == "VARCHAR"; + }) != schema.end()); + assert(std::find_if(schema.begin(), schema.end(), [](const documentdb::FieldType& field) { + return field.name == "metadata" && field.duckdb_type == "VARCHAR"; + }) != schema.end()); + + auto plan = documentdb::ScanPlanner::plan("orders", "{\"status\": \"active\"}", {"_id", "status"}, 10); + assert(plan.collection_name == "orders"); + assert(plan.limit == 10); + assert(plan.projections.size() == 2); + + return 0; +} diff --git a/tests/run_documentdb_e2e.sh b/tests/run_documentdb_e2e.sh new file mode 100755 index 0000000..c057124 --- /dev/null +++ b/tests/run_documentdb_e2e.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +documentdb_image="ghcr.io/documentdb/documentdb/documentdb-local:latest" +documentdb_container="duckdb-documentdb-e2e-db" +client_image="duckdb-documentdb-e2e-client" +network="duckdb-documentdb-e2e" +username="duckdb_e2e_admin" +password="E2e$(date +%s)${RANDOM}${RANDOM}Aa1!" + +cleanup() { + docker rm --force "${documentdb_container}" >/dev/null 2>&1 || true + docker network rm "${network}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +cleanup +docker network create "${network}" >/dev/null +docker run --detach --name "${documentdb_container}" --network "${network}" \ + "${documentdb_image}" --username "${username}" --password "${password}" --skip-init-data >/dev/null + +ready=false +for _ in $(seq 1 80); do + if docker exec "${documentdb_container}" mongosh localhost:10260 \ + -u "${username}" -p "${password}" --authenticationDatabase admin \ + --authenticationMechanism SCRAM-SHA-256 --tls --tlsAllowInvalidCertificates \ + --quiet --eval 'db.runCommand({ping:1}).ok' >/dev/null 2>&1; then + ready=true + break + fi + sleep 3 +done + +if [[ "${ready}" != "true" ]]; then + echo "DocumentDB Local did not become ready" >&2 + docker logs "${documentdb_container}" >&2 + exit 1 +fi + +seed='const c=db.getSiblingDB("duckdb_e2e").getCollection("orders\"archive"); c.drop(); c.insertMany([{status:"active",total:99},{status:"pending",total:49}]);' +docker exec "${documentdb_container}" mongosh localhost:10260 \ + -u "${username}" -p "${password}" --authenticationDatabase admin \ + --authenticationMechanism SCRAM-SHA-256 --tls --tlsAllowInvalidCertificates \ + --quiet --eval "${seed}" >/dev/null + +docker build --file tests/Dockerfile.e2e --tag "${client_image}" . +docker run --rm --network "${network}" \ + --env DOCUMENTDB_HOST="${documentdb_container}" \ + --env DOCUMENTDB_USER="${username}" \ + --env DOCUMENTDB_PASSWORD="${password}" \ + "${client_image}" + +echo "DocumentDB Docker E2E test passed." \ No newline at end of file diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..ed0d49e --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "duckdb-documentdb", + "version-string": "0.1.0", + "dependencies": [ + { + "name": "mongo-c-driver", + "features": [ + "openssl" + ] + } + ] +} \ No newline at end of file