Add DocumentDB DuckDB extension

.gitignore

@@ -0,0 +1,11 @@
+build/
+build_output.txt
+cmake-build-debug/
+cmake-build-release/
+.vscode/
+.DS_Store
+*.log
+*.swp
+*.user
+__pycache__/
+*.pyc

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "duckdb"]
+	path = duckdb
+	url = https://github.com/duckdb/duckdb.git
+
+[submodule "extension-ci-tools"]
+	path = extension-ci-tools
+	url = https://github.com/duckdb/extension-ci-tools.git

CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 3.5)
+
+set(TARGET_NAME documentdb)
+set(EXTENSION_NAME ${TARGET_NAME}_extension)
+set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
+
+project(${TARGET_NAME})
+
+set(CMAKE_CXX_STANDARD "17" CACHE STRING "C++ standard to enforce")
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_package(mongoc CONFIG REQUIRED)
+
+include_directories(src/include)
+
+set(EXTENSION_SOURCES
+    src/documentdb_extension.cpp
+    src/documentdb_connection.cpp
+    src/documentdb_schema.cpp
+    src/documentdb_scan.cpp
+)
+
+build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
+build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
+
+target_link_libraries(${EXTENSION_NAME} duckdb_yyjson mongoc::static)
+target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_yyjson mongoc::static)
+
+target_include_directories(${EXTENSION_NAME}
+    PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/include
+)
+
+target_include_directories(${LOADABLE_EXTENSION_NAME}
+    PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/include
+)
+
+include(CTest)
+if(BUILD_TESTING)
+  add_executable(documentdb_smoke_test tests/documentdb_smoke_test.cpp)
+  target_link_libraries(documentdb_smoke_test PRIVATE ${EXTENSION_NAME})
+  add_test(NAME documentdb_smoke_test COMMAND documentdb_smoke_test)
+endif()
+
+install(
+  TARGETS ${EXTENSION_NAME}
+  EXPORT "${DUCKDB_EXPORT_SET}"
+  LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
+  ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
+)
+
+install(
+  TARGETS ${LOADABLE_EXTENSION_NAME}
+  EXPORT "${DUCKDB_EXPORT_SET}"
+  LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
+  ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
+)

Makefile

@@ -0,0 +1,6 @@
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+EXT_NAME=documentdb
+EXT_CONFIG=${PROJ_DIR}extension_config.cmake
+
+include extension-ci-tools/makefiles/duckdb_extension.Makefile

README.md

@@ -1,2 +1,73 @@
 # duckdb-documentdb
-Integrates DuckDB with DocumentDB, enabling direct SQL queries over DocumentDB collections without exporting data or ETL.
+
+DocumentDB integration for DuckDB, modeled after the MongoDB-style extension architecture used in duckdb-mongo. The repository is being evolved into a proper DuckDB extension with real extension registration, loadable binary output, and a documented DocumentDB-facing API surface.
+
+## What this repo is
+
+This repository is a practical starting point for a DuckDB extension that targets the DocumentDB ecosystem. The implementation currently includes:
+
+- a public connection and schema API
+- a scan planner inspired by MongoDB-style pushdown design
+- a real extension entry point for loading functions into DuckDB
+- an extension build configuration aligned with DuckDB’s extension model
+
+The initial extension surface is read-only. It supports collection discovery, schema inference, scans, and query pushdown, but does not provide insert, update, or delete operations.
+
+## Key design goals
+
+- expose DocumentDB collections through DuckDB SQL
+- keep document access live and pushdown-friendly
+- infer schema from sample documents for DuckDB columns
+- provide extension functions that load cleanly into the DuckDB runtime
+- preserve a MongoDB-like design while adapting to DocumentDB semantics
+
+## Repository structure
+
+- [docs/architecture.md](docs/architecture.md): architecture notes and extension layout
+- [docs/usage.md](docs/usage.md): build and usage guidance
+- [include/documentdb/documentdb.hpp](include/documentdb/documentdb.hpp): public connection and planner API
+- [src/documentdb_extension.cpp](src/documentdb_extension.cpp): DuckDB extension entry point and function registration
+- [src/documentdb_connection.cpp](src/documentdb_connection.cpp): connection handling
+- [src/documentdb_schema.cpp](src/documentdb_schema.cpp): schema inference and resolution
+- [src/documentdb_scan.cpp](src/documentdb_scan.cpp): scan and pushdown planning
+- [tests/documentdb_smoke_test.cpp](tests/documentdb_smoke_test.cpp): smoke validation for the API layer
+
+## Example usage
+
+```sql
+SELECT documentdb_version('documentdb') AS version;
+SELECT documentdb_collections('app') AS collections;
+```
+
+## Build
+
+```bash
+git submodule update --init --recursive
+make build
+```
+
+## Docker end-to-end test
+
+The end-to-end test starts the official DocumentDB Local image, creates test data through `mongosh`, and verifies that the C++ connection layer discovers and filters the real collection through the MongoDB wire protocol:
+
+```bash
+./tests/run_documentdb_e2e.sh
+```
+
+The script generates an ephemeral password for each run and removes the test container and network when it finishes. Self-signed TLS certificates are accepted only by this local test configuration.
+
+## Status
+
+This repo now includes the core extension plumbing needed for a real DuckDB extension and keeps the working C++ API layer and smoke tests. The next step is to connect it to a live DocumentDB backend and expand the SQL attach and scan semantics beyond the current scaffold.
+
+## Relationship to the base project
+
+This repo uses the duckdb-mongo extension as the architectural reference and adapts the same core ideas to DocumentDB:
+
+- SQL attach semantics
+- document-to-column mapping
+- schema inference
+- pushdown-oriented scan planning
+- live collection access through DuckDB
+
+The adaptation is targeted to DocumentDB rather than MongoDB while preserving the same developer experience where possible.

docs/architecture.md

@@ -0,0 +1,40 @@
+# Architecture
+
+This repository is deliberately structured as a reusable starting point for a DuckDB integration layer for DocumentDB.
+
+## Goals
+
+- Expose MongoDB-compatible collections through SQL
+- Keep data in DocumentDB while querying through DuckDB
+- Push down filters, projections, and aggregates when supported
+- Keep the extension design compatible with the duckdb-mongo architecture
+
+## Core components
+
+### 1. Connection layer
+
+The connection layer handles endpoint configuration, authentication, and database selection.
+
+### 2. Schema inference
+
+Schema inference samples documents and converts BSON-like fields into DuckDB-friendly logical types. This follows the same pattern used in the duckdb-mongo repo and is extended to DocumentDB semantics.
+
+### 3. Scan layer
+
+The scan layer builds a query plan and carries the DocumentDB collection read path. It is responsible for generating a logical scan, mapping document fields, and sending pushdown operations.
+
+### 4. Pushdown planner
+
+This planner converts SQL predicates into DocumentDB-native operations such as filter, projection, and aggregate stages. It is intentionally modeled after the duckdb-mongo pushdown strategy.
+
+## Planned SQL surface
+
+```sql
+ATTACH 'host=localhost port=27017 dbname=app' AS documentdb (TYPE DOCUMENTDB);
+SELECT * FROM documentdb.app.orders LIMIT 10;
+SELECT status, COUNT(*) FROM documentdb.app.orders GROUP BY status;
+```
+
+## Reference base
+
+This repo borrows architecture and design ideas from the duckdb-mongo extension and re-targets them for DocumentDB.

docs/usage.md

@@ -0,0 +1,40 @@
+# Usage
+
+This project serves as a repository skeleton for a DocumentDB extension patterned after duckdb-mongo.
+
+## Build
+
+```bash
+make build
+```
+
+## Run tests
+
+```bash
+make test
+```
+
+## Example API
+
+```cpp
+#include <documentdb/documentdb.hpp>
+
+int main() {
+    documentdb::ConnectionConfig cfg;
+    cfg.host = "localhost";
+    cfg.port = 27017;
+    cfg.database = "app";
+
+    documentdb::Connection conn(cfg);
+    auto collections = conn.list_collections();
+    auto rows = conn.scan("orders", "{\"status\": \"active\"}");
+    return rows.empty() ? 0 : 1;
+}
+```
+
+## Roadmap
+
+1. Implement DocumentDB connection protocol support
+2. Add SQL attach and scan entry points
+3. Build pushdown translation for filters and aggregates
+4. Add integration tests against a DocumentDB instance

extension_config.cmake

@@ -0,0 +1,6 @@
+# This file is included by DuckDB's extension build system.
+# It tells the build which extension to compile for the repo.
+
+duckdb_extension_load(documentdb
+    SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+)

include/documentdb/documentdb.hpp

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace documentdb {
+
+struct ConnectionConfig {
+    std::string host = "localhost";
+    int port = 27017;
+    std::string database;
+    std::string user;
+    std::string password;
+    std::string auth_source;
+    bool tls = false;
+    bool tls_allow_invalid_certificates = false;
+    int server_selection_timeout_ms = 5000;
+};
+
+struct Document {
+    std::string raw_json;
+    std::string collection;
+};
+
+struct FieldType {
+    std::string name;
+    std::string duckdb_type;
+};
+
+struct ScanPlan {
+    std::string collection_name;
+    std::string filter;
+    std::vector<std::string> projections;
+    bool has_limit = false;
+    int limit = 0;
+};
+
+class Connection {
+public:
+    explicit Connection(const ConnectionConfig& config);
+
+    std::vector<std::string> list_collections() const;
+    std::vector<Document> scan(const std::string& collection_name,
+                              const std::string& filter_json = "{}") const;
+
+private:
+    ConnectionConfig config_;
+};
+
+class SchemaResolver {
+public:
+    static std::vector<FieldType> infer_from_samples(const std::vector<std::string>& samples);
+};
+
+class ScanPlanner {
+public:
+    static ScanPlan plan(const std::string& collection_name,
+                        const std::string& filter,
+                        const std::vector<std::string>& selected_columns,
+                        int limit);
+};
+
+}  // namespace documentdb