From 92222594d6f8224791bff3f81eafe2946ac4e03a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 15:01:58 -0400 Subject: [PATCH 01/19] Add database environment storage, SQL, and module bindings --- Cargo.lock | 7 + Cargo.toml | 1 + .../include/spacetimedb/abi/FFI.h | 1 + .../include/spacetimedb/abi/abi.h | 7 + .../include/spacetimedb/bsatn/reader.h | 6 +- .../include/spacetimedb/environment.h | 34 ++++ .../include/spacetimedb/procedure_context.h | 3 + .../include/spacetimedb/reducer_context.h | 3 + .../include/spacetimedb/tx_context.h | 2 + .../include/spacetimedb/view_context.h | 4 + crates/bindings-cpp/tests/unit/CMakeLists.txt | 1 + .../tests/unit/environment_unit_tests.cpp | 50 +++++ .../diag/snapshots/Module#FFI.verified.cs | 3 + .../snapshots/Module#FFI.verified.cs | 3 + .../server/snapshots/Module#FFI.verified.cs | 3 + crates/bindings-csharp/Codegen/Module.cs | 3 + .../Runtime/DatabaseEnvironment.cs | 26 +++ .../bindings-csharp/Runtime/HandlerContext.cs | 2 + .../bindings-csharp/Runtime/Internal/FFI.cs | 15 ++ .../Runtime/ProcedureContext.cs | 2 + crates/bindings-csharp/Runtime/bindings.c | 4 + crates/bindings-sys/src/lib.rs | 20 ++ .../src/lib/environment.ts | 5 + .../bindings-typescript/src/lib/reducers.ts | 2 + .../src/server/environment.ts | 6 + .../src/server/http_handlers.ts | 2 + .../bindings-typescript/src/server/index.ts | 2 + .../src/server/procedures.ts | 3 + .../bindings-typescript/src/server/runtime.ts | 5 + .../bindings-typescript/src/server/sys.d.ts | 7 + .../bindings-typescript/src/server/views.ts | 3 + crates/bindings/src/http.rs | 4 + crates/bindings/src/lib.rs | 29 +++ crates/bindings/src/rt.rs | 7 + crates/client-api/src/lib.rs | 7 +- crates/core/src/db/environment.rs | 131 +++++++++++++ crates/core/src/db/mod.rs | 2 + crates/core/src/error.rs | 4 + crates/core/src/host/instance_env.rs | 142 ++++++++++++++ crates/core/src/host/mod.rs | 1 + crates/core/src/host/v8/syscall/mod.rs | 2 + crates/core/src/host/v8/syscall/v2.rs | 18 ++ crates/core/src/host/wasm_common.rs | 4 + .../src/host/wasmtime/wasm_instance_env.rs | 41 +++- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- crates/core/src/sql/execute.rs | 109 ++++++++++- .../locking_tx_datastore/committed_state.rs | 2 + .../src/locking_tx_datastore/datastore.rs | 7 + crates/datastore/src/system_tables.rs | 12 +- .../src/system_tables/environment.rs | 45 +++++ crates/expr/src/errors.rs | 4 + crates/expr/src/statement.rs | 21 +++ crates/lib/src/environment.rs | 62 ++++++ crates/lib/src/lib.rs | 1 + crates/query/src/lib.rs | 6 +- crates/sql-parser/src/ast/sql.rs | 9 + crates/sql-parser/src/parser/sql.rs | 35 +++- crates/testing/tests/environment.rs | 177 ++++++++++++++++++ modules/environment-test/Cargo.toml | 13 ++ modules/environment-test/src/lib.rs | 69 +++++++ modules/module-test-cpp/src/lib.cpp | 12 ++ modules/module-test-cs/EnvironmentTests.cs | 25 +++ modules/module-test-ts/src/index.ts | 23 +++ 63 files changed, 1239 insertions(+), 22 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/environment.h create mode 100644 crates/bindings-cpp/tests/unit/environment_unit_tests.cpp create mode 100644 crates/bindings-csharp/Runtime/DatabaseEnvironment.cs create mode 100644 crates/bindings-typescript/src/lib/environment.ts create mode 100644 crates/bindings-typescript/src/server/environment.ts create mode 100644 crates/core/src/db/environment.rs create mode 100644 crates/datastore/src/system_tables/environment.rs create mode 100644 crates/lib/src/environment.rs create mode 100644 crates/testing/tests/environment.rs create mode 100644 modules/environment-test/Cargo.toml create mode 100644 modules/environment-test/src/lib.rs create mode 100644 modules/module-test-cs/EnvironmentTests.cs diff --git a/Cargo.lock b/Cargo.lock index 6f9455400ec..9b030bc73ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2298,6 +2298,13 @@ dependencies = [ "log", ] +[[package]] +name = "environment-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.10.0", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 9f7852863c8..afdde4e3253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/environment-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index 32990f156d4..f3ad213452f 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -73,6 +73,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; +using ::env_get; // ===== Procedure Transactions ===== using ::procedure_start_mut_tx; diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 99dc067c147..285974cff3c 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,6 +39,10 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern +// ABI10.6 is reserved for the separate invocation-authority extension. +#define STDB_IMPORT_10_7(name) \ + __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -59,6 +63,9 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +STDB_IMPORT_10_7(env_get) +Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); + // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) Status table_id_from_name(const uint8_t* name_ptr, size_t name_len, TableId* out); diff --git a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h index 5fcba095c76..a61fc3bcc59 100644 --- a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h +++ b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h @@ -124,10 +124,10 @@ namespace SpacetimeDB::bsatn { template std::optional read_optional() { uint8_t tag = read_u8(); - if (tag == 0) { - return std::nullopt; - } else if (tag == 1) { + if (tag == 0) { // Some, matching the canonical BSATN option type. return SpacetimeDB::bsatn::deserialize(*this); + } else if (tag == 1) { // None. + return std::nullopt; } else { std::abort(); // Invalid optional tag in BSATN deserialization } diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h new file mode 100644 index 00000000000..f2df08234d0 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -0,0 +1,34 @@ +#ifndef SPACETIMEDB_ENVIRONMENT_H +#define SPACETIMEDB_ENVIRONMENT_H +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { +/// Read-only database environment. Reads use the current transaction, or a +/// short snapshot in a procedure outside a transaction. Values are not cached. +class Environment { +public: + std::optional get(std::string_view key) const { + if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); + BytesSource source{0}; + if (FFI::env_get(reinterpret_cast(key.data()), static_cast(key.size()), &source) != Status(0)) + LOG_PANIC("environment read failed"); + if (source == BytesSource{0}) return std::nullopt; + std::array buffer; + std::string value; + for (;;) { + size_t len = buffer.size(); + const auto status = FFI::bytes_source_read(source, buffer.data(), &len); + if ((status != 0 && status != -1) || len > buffer.size()) LOG_PANIC("environment source read failed"); + value.append(reinterpret_cast(buffer.data()), len); + if (status == -1) return value; + if (len == 0) LOG_PANIC("environment source made no progress"); + } + } +}; +} +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index ebfd958af7c..35c93c31475 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -15,6 +15,8 @@ #include #include +#include + namespace SpacetimeDB { /** @@ -57,6 +59,7 @@ struct ProcedureContext { Identity sender_; public: + Environment env; // Timestamp when the procedure was invoked Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 41865b14f3a..14a6028c606 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -13,6 +13,8 @@ // Include database for DatabaseContext #include +#include + namespace SpacetimeDB { // Enhanced ReducerContext with database access - matches Rust pattern @@ -21,6 +23,7 @@ struct ReducerContext { Identity sender_; public: + Environment env; // Core fields - sender is exposed via sender() like Rust, other fields remain directly accessible std::optional connection_id; Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/tx_context.h b/crates/bindings-cpp/include/spacetimedb/tx_context.h index 1a04ef027e1..c874a22b4ff 100644 --- a/crates/bindings-cpp/include/spacetimedb/tx_context.h +++ b/crates/bindings-cpp/include/spacetimedb/tx_context.h @@ -56,6 +56,7 @@ struct TxContext { // In C++, we explicitly expose references where possible and provide // accessors for fields exposed as methods on ReducerContext. DatabaseContext& db; + const Environment& env; const Timestamp& timestamp; const std::optional& connection_id; @@ -63,6 +64,7 @@ struct TxContext { explicit TxContext(ReducerContext& ctx) : ctx_(ctx), db(ctx.db), + env(ctx.env), timestamp(ctx.timestamp), connection_id(ctx.connection_id) {} diff --git a/crates/bindings-cpp/include/spacetimedb/view_context.h b/crates/bindings-cpp/include/spacetimedb/view_context.h index 63102699e42..6985c2e55b0 100644 --- a/crates/bindings-cpp/include/spacetimedb/view_context.h +++ b/crates/bindings-cpp/include/spacetimedb/view_context.h @@ -7,6 +7,8 @@ #include // For ReadOnlyDatabaseContext #include +#include + namespace SpacetimeDB { /** @@ -41,6 +43,7 @@ struct ViewContext { public: // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors @@ -76,6 +79,7 @@ struct ViewContext { struct AnonymousViewContext { // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 0ced4e0194c..da7b8705e1c 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,6 +11,7 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp + environment_unit_tests.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp new file mode 100644 index 00000000000..c6bd0f58cd8 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp @@ -0,0 +1,50 @@ +#include "test_harness.h" +#include "spacetimedb/environment.h" +#include "spacetimedb/bsatn/reader.h" +#include +#include + +using namespace SpacetimeDB; + +namespace { +size_t payload_offset; +std::string payload; +} + +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, payload.size() - payload_offset); + std::memcpy(out, payload.data() + payload_offset, *len); + payload_offset += *len; + return payload_offset == payload.size() ? -1 : 0; +} + +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + payload = std::string(8192, 'x'); + ASSERT_EQ(payload, env.get("LARGE").value()); + payload = std::string("a\0b", 3); + ASSERT_EQ(payload, env.get("NUL").value()); + payload = "updated"; + ASSERT_EQ(payload, env.get("NUL").value()); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 6638bb12fc8..2eb70c4a352 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,6 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -892,6 +893,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -905,6 +907,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index 015ec80b3ad..d05060e0bd3 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,6 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -275,6 +276,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -288,6 +290,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 82e3d7bdecf..7f196c87a30 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,6 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -726,6 +727,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -739,6 +741,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index aad1bf54c02..e3f146ad3f2 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2557,6 +2557,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2755,6 +2756,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2766,6 +2768,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs new file mode 100644 index 00000000000..59ca2c26ec2 --- /dev/null +++ b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs @@ -0,0 +1,26 @@ +namespace SpacetimeDB; + +/// +/// Read-only database environment. Values are plaintext and accessible to database +/// collaborators. Procedure reads outside a transaction use a short snapshot. +/// +public readonly struct DatabaseEnvironment +{ + internal static readonly DatabaseEnvironment Instance = new(); + + /// Return null for a missing key, or an empty string for a present empty value. + public unsafe string? Get(string key) + { + ArgumentNullException.ThrowIfNull(key); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + fixed (byte* ptr = bytes) + { + Internal.FFI.env_get(ptr, checked((uint)bytes.Length), out var source); + if (source == Internal.BytesSource.INVALID) + { + return null; + } + return System.Text.Encoding.UTF8.GetString(Internal.Module.Consume(source)); + } + } +} diff --git a/crates/bindings-csharp/Runtime/HandlerContext.cs b/crates/bindings-csharp/Runtime/HandlerContext.cs index 76b13230426..9fc02d8b858 100644 --- a/crates/bindings-csharp/Runtime/HandlerContext.cs +++ b/crates/bindings-csharp/Runtime/HandlerContext.cs @@ -7,6 +7,7 @@ namespace SpacetimeDB; public abstract class HandlerContextBase { public Random Rng => txState.Rng; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => txState.Timestamp; // NOTE: The host rejects procedure HTTP requests while a mut transaction is open @@ -90,6 +91,7 @@ public abstract class HandlerTxContextBase(Internal.TxContext inner) : IRefresha void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => Inner.Timestamp; public Random Rng => Inner.Rng; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index afc56abbc8f..f498e6ed8ea 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,6 +109,21 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_7 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.7" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_7)] + public static unsafe partial CheckedStatus env_get( + byte* key, + uint keyLen, + out BytesSource source + ); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index a86711f2814..63ec0238ca0 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -3,6 +3,7 @@ namespace SpacetimeDB; #pragma warning disable STDB_UNSTABLE public abstract class ProcedureContextBase : Internal.IInternalProcedureContext { + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public static Identity Identity => Internal.IProcedureContext.GetIdentity(); public Identity Sender { get; } public ConnectionId? ConnectionId { get; } @@ -100,6 +101,7 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) : IRefres void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 6118ba6be11..f4d3635a5f2 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,6 +135,10 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.7" +IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 0295b4616e5..2a3bd77b454 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,6 +883,18 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } + // ABI10.6 is reserved for the separate invocation-authority extension. + #[link(wasm_import_module = "spacetime_10.7")] + unsafe extern "C" { + /// Read a UTF-8 environment value. Writes INVALID for a missing key; + /// present empty strings have a valid BytesSource. Returns ordinary errno. + /// Invalid keys return HOST_CALL_FAILURE. NO_SPACE means 256 byte + /// sources remain unconsumed; consume a source before retrying. + /// Calls outside a reducer/view + /// transaction or procedure return NOT_IN_TRANSACTION. + pub fn env_get(key: *const u8, key_len: usize, out: *mut BytesSource) -> u16; + } + /// What strategy does the database index use? /// /// See also: @@ -1493,6 +1505,14 @@ pub fn get_jwt(connection_id: [u8; 16]) -> Option { } } +/// Read a database environment value without exposing the system table. +#[inline] +pub fn env_get(key: &str) -> Option { + let source = unsafe { call(|out| raw::env_get(key.as_ptr(), key.len(), out)) } + .unwrap_or_else(|errno: Errno| panic!("Error reading environment: {errno}")); + (source != raw::BytesSource::INVALID).then_some(source) +} + pub struct RowIter { raw: raw::RowIter, } diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts new file mode 100644 index 00000000000..3798cfb43d4 --- /dev/null +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -0,0 +1,5 @@ +/** Read-only database environment access. Missing keys return null; empty values return "". */ +export interface Environment { + /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ + get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 27c57f0721b..eaa8de94b55 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,3 +1,4 @@ +import type { Environment } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -115,6 +116,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + env: Environment; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts new file mode 100644 index 00000000000..349e8ffed8c --- /dev/null +++ b/crates/bindings-typescript/src/server/environment.ts @@ -0,0 +1,6 @@ +import { env_get } from 'spacetime:sys@2.3'; +import type { Environment } from '../lib/environment'; + +/** Values are not cached: transaction and procedure reads retain host semantics. */ +export const environment: Environment = Object.freeze({ get: env_get }); +export type { Environment } from '../lib/environment'; diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index 68d42a267f8..0c4d9469eee 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -219,6 +220,7 @@ export type HandlerAliasViews = : {}; export interface HandlerContext { + readonly env: Environment; readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index ae084f672d4..3ac3e8f0fbb 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -35,4 +35,6 @@ export { export type { HandlerContext, HttpHandlerExport } from './http'; export { ScheduleAt } from '../lib/schedule_at'; +export type { Environment } from './environment'; + import './polyfills'; // Ensure polyfills are loaded diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 2dec68467c0..f4d57416e69 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,3 +1,4 @@ +import { environment, type Environment } from './environment'; import { AlgebraicType, ProductType, @@ -108,6 +109,7 @@ export type ProcedureAliasViews = : {}; export interface ProcedureCtx { + readonly env: Environment; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -226,6 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + readonly env = environment; #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index cf579e87631..1fac18ac95b 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,3 +1,4 @@ +import { environment } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; @@ -246,6 +247,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + readonly env = environment; as: AliasViews; constructor( @@ -627,6 +629,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = viewFns![localId!]; const ctx: ViewCtx = freeze({ + env: environment, sender: new Identity(sender), db: dbView!, from: from!, @@ -677,6 +680,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = anonViewFns![localId!]; const ctx: AnonymousViewCtx = freeze({ + env: environment, db: dbView!, from: from!, }); @@ -773,6 +777,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { + readonly env = environment; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index f0315867cb3..1f74debd2fc 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -123,3 +123,10 @@ declare module 'spacetime:sys@2.0' { declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } + +// sys2.2 is reserved for the separate invocation-authority extension. + +declare module 'spacetime:sys@2.3' { + /** Null means missing; an empty string is a present value. */ + export function env_get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index c7cb9ca0b4d..528c8319063 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -81,11 +82,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index 4c8ea487c47..3638d35f9e2 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -98,6 +98,9 @@ pub use spacetimedb_bindings_macro::http_router as router; #[cfg(feature = "unstable")] #[non_exhaustive] pub struct HandlerContext { + /// Read-only access to this database's environment store. + pub env: crate::Environment, + /// The time at which the handler was started. pub timestamp: Timestamp, @@ -117,6 +120,7 @@ pub struct HandlerContext { impl HandlerContext { pub(crate) fn new(timestamp: Timestamp) -> Self { Self { + env: crate::Environment::default(), timestamp, http: HttpClient {}, #[cfg(feature = "rand08")] diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 2c7faa78c6b..d1548bca052 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -919,10 +919,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Read-only access to this database's environment store. +/// +/// Reads use the current transaction. In a procedure outside a transaction, +/// each read uses a short snapshot; use `with_tx` to read related keys together. +/// Values are stored in plaintext and may be read by database collaborators. +#[derive(Clone, Copy, Debug, Default)] +pub struct Environment { + _private: (), +} + +impl Environment { + /// Return None for a missing key and Some("") for a present empty value. + /// Keys must be POSIX environment names of at most 256 bytes. + pub fn get(&self, key: &str) -> Option { + rt::env_get(key) + } +} + /// One of two possible types that can be passed as the first argument to a `#[view]`. /// The other is [`ViewContext`]. /// Use this type if the view does not depend on the caller's identity. pub struct AnonymousViewContext { + pub env: Environment, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -930,6 +949,7 @@ pub struct AnonymousViewContext { impl Default for AnonymousViewContext { fn default() -> Self { Self { + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -939,6 +959,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { + pub env: Environment, sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, @@ -948,6 +969,7 @@ impl ViewContext { pub fn new(sender: Identity) -> Self { Self { sender, + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -978,6 +1000,8 @@ impl ViewContext { /// Implements the `DbContext` trait for accessing views into a database. #[non_exhaustive] pub struct ReducerContext { + /// Read-only access to the database environment in this transaction. + pub env: Environment, /// The `Identity` of the client that invoked the reducer. sender: Identity, @@ -1042,6 +1066,7 @@ impl ReducerContext { #[doc(hidden)] pub fn __dummy() -> Self { Self { + env: Environment::default(), db: Local {}, sender: Identity::__dummy(), timestamp: Timestamp::UNIX_EPOCH, @@ -1057,6 +1082,7 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { Self { + env: Environment::default(), db, sender, timestamp, @@ -1254,6 +1280,8 @@ fn with_tx(body: impl Fn(&TxContext) -> T, identity: Identity, connection_id: /// and exposes methods for running transactions and performing side-effecting operations. #[non_exhaustive] pub struct ProcedureContext { + /// Read-only access to the database environment. + pub env: Environment, /// The `Identity` of the client that invoked the procedure. sender: Identity, @@ -1285,6 +1313,7 @@ impl ProcedureContext { sender, timestamp, connection_id, + env: Environment::default(), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 7d456d73fde..b09c998f5c3 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1319,6 +1319,13 @@ pub fn get_jwt(connection_id: ConnectionId) -> Option { Some(std::str::from_utf8(&buf).unwrap().to_string()) } +pub(crate) fn env_get(key: &str) -> Option { + let source = sys::env_get(key)?; + let mut buf = IterBuf::take(); + read_bytes_source_into(source, &mut buf); + Some(String::from_utf8(buf.to_vec()).expect("host environment values are UTF-8")) +} + /// Read `source` from the host fully into `buf`. pub(crate) fn read_bytes_source_into(source: BytesSource, buf: &mut Vec) { const INVALID: i16 = NO_SUCH_BYTES as i16; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index e8999e1e617..68fd0a61bb5 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -146,7 +146,8 @@ impl Host { .await .map_err(|_| (StatusCode::NOT_FOUND, "module not found".to_string()))?; - tracing::debug!(sql = body); + // Environment SQL contains values; routine request logs must omit them. + tracing::debug!(sql_bytes = body.len(), "executing SQL"); let mut header = vec![]; let sql_start = std::time::Instant::now(); let sql_span = tracing::trace_span!("execute_sql", total_duration = tracing::field::Empty,); @@ -164,8 +165,8 @@ impl Host { ) .await .map_err(|e| { - // TODO: Review log level after user SQL errors can be distinguished from internal database failures. - log::warn!("{e}"); + // Parser diagnostics can quote values. Return them only to the caller. + log::warn!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs new file mode 100644 index 00000000000..87b7a7231f7 --- /dev/null +++ b/crates/core/src/db/environment.rs @@ -0,0 +1,131 @@ +//! Dedicated access to the private environment store. +//! +//! Mutation callers must authorize owner/admin access before calling these +//! helpers and commit through the normal module transaction machinery so +//! dependent views refresh. Helpers never acquire a second transaction. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_sats::AlgebraicValue; +use std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error(transparent)] + Validation(#[from] EnvironmentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Read from exactly the caller's snapshot, preserving missing versus empty. +pub fn get(state: &impl StateView, key: &str) -> Result, EnvironmentError> { + validate_key(key)?; + state + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| Ok(StEnvRow::try_from(row)?.value)) + .transpose() +} + +pub fn snapshot(state: &impl StateView) -> Result, EnvironmentError> { + state + .iter(ST_ENV_ID)? + .map(|row| { + let row = StEnvRow::try_from(row)?; + Ok((row.key, row.value)) + }) + .collect() +} + +/// Insert or replace one key. Validation occurs before any mutation. +pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { + validate_key(key)?; + validate_value(value)?; + let previous = get(tx, key)?; + if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { + return Err(EnvironmentValidationError::TooManyVariables.into()); + } + if previous.as_deref() == Some(value) { + return Ok(()); + } + delete(db, tx, key)?; + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + )?; + Ok(()) +} + +pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { + validate_key(key)?; + let pointer = tx + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| row.pointer()); + if let Some(pointer) = pointer { + db.delete(tx, ST_ENV_ID, [pointer]); + return Ok(true); + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use spacetimedb_datastore::execution_context::Workload; + + #[test] + fn missing_empty_nul_update_and_rollback() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + assert_eq!(get(tx, "EMPTY")?, None); + set(&db, tx, "EMPTY", "")?; + set(&db, tx, "NUL", "a\0b")?; + assert_eq!(get(tx, "EMPTY")?, Some(String::new())); + assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); + Ok(()) + }) + .unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + set(&db, tx, "EMPTY", "changed")?; + delete(&db, tx, "NUL")?; + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(result.is_err()); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + snapshot(tx).unwrap(), + BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) + ); + }); + } + + #[test] + fn capacity_and_value_limits_precede_mutation() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + for i in 0..MAX_ENV_VARS { + set(&db, tx, &format!("K{i}"), "")?; + } + assert!(set(&db, tx, "EXTRA", "").is_err()); + set(&db, tx, "K0", "updated")?; + assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); + assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); + assert!(delete(&db, tx, "K1")?); + assert!(!delete(&db, tx, "MISSING")?); + set(&db, tx, "EXTRA", "")?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 6b1d2f6700b..a7117db5b71 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -1,3 +1,5 @@ +pub mod environment; + pub mod persistence { pub use spacetimedb_engine::persistence::*; } diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index efee537dbbe..0f8d5621b3f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -30,6 +30,10 @@ impl From for DBError { #[derive(Error, Debug)] pub enum NodesError { + #[error("invalid environment variable name")] + InvalidEnvironmentKey, + #[error("too many outstanding byte sources for environment read")] + EnvironmentSourceLimit, #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 9bf807ebf7a..1bfb4f7f7c7 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -17,6 +17,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOrRange, MutTxId}; +use spacetimedb_datastore::system_tables::{is_module_restricted_index, is_module_restricted_table}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_metrics::utils::IntGaugeExt; @@ -281,6 +282,25 @@ impl InstanceEnv { self.replica_ctx.relational_db() } + /// Dedicated read-only environment access. Missing reads also register a + /// dependency so a later insert refreshes a view that observed absence. + pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { + use crate::db::environment; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; + let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if let Ok(mut tx) = self.get_tx() { + tx.record_table_scan(&self.func_type, ST_ENV_ID); + return read(&*tx); + } + if !matches!(self.func_type, FuncCallType::Procedure) { + return Err(NodesError::NotInTransaction); + } + self.relational_db().with_read_only(Workload::Internal, |tx| { + environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) + }) + } + pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { let tx = &mut *self.get_tx()?; Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) @@ -355,9 +375,19 @@ impl InstanceEnv { count } + /// Environment values are reachable only through their dedicated host interface. + fn require_module_table(table_id: TableId) -> Result<(), NodesError> { + if is_module_restricted_table(table_id) { + Err(NodesError::TableNotFound) + } else { + Ok(()) + } + } + pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, insert_flags) = stdb .insert(tx, table_id, buffer) @@ -436,6 +466,7 @@ impl InstanceEnv { pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, update_flags) = stdb .update(tx, table_id, index_id, buffer) @@ -479,6 +510,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, _, iter) = stdb.index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = iter.map(|row_ref| row_ref.pointer()).collect::>(); @@ -499,6 +531,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, iter) = stdb.index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = match iter { IndexScanPointOrRange::Point(_, iter) => iter.map(|row_ref| row_ref.pointer()).collect(), @@ -540,6 +573,7 @@ impl InstanceEnv { pub fn datastore_delete_all_by_eq_bsatn(&self, table_id: TableId, relation: &[u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Track the number of bytes coming from the caller tx.metrics.bytes_scanned += relation.len(); @@ -563,6 +597,7 @@ impl InstanceEnv { pub fn clear(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let rows_deleted = stdb.clear_table(tx, table_id).map_err(NodesError::from)?; @@ -584,6 +619,7 @@ impl InstanceEnv { // Query the table id from the name. stdb.table_id_from_name_mut(tx, table_name)? + .filter(|id| !is_module_restricted_table(*id)) .ok_or(NodesError::TableNotFound) } @@ -598,6 +634,7 @@ impl InstanceEnv { // Query the index id from the name. stdb.index_id_from_name_mut(tx, index_name)? + .filter(|id| !is_module_restricted_index(*id)) .ok_or(NodesError::IndexNotFound) } @@ -609,6 +646,7 @@ impl InstanceEnv { pub fn datastore_table_row_count(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Query the row count for id. stdb.table_row_count_mut(tx, table_id) @@ -625,6 +663,7 @@ impl InstanceEnv { table_id: TableId, ) -> Result>, NodesError> { let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Open the iterator. let iter = self.relational_db().iter_mut(tx, table_id)?; @@ -652,6 +691,7 @@ impl InstanceEnv { // Open index iterator let (table_id, point, iter) = self.relational_db().index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (chunks, rows_scanned, bytes_scanned) = ChunkedWriter::collect_iter(pool, iter); @@ -682,6 +722,7 @@ impl InstanceEnv { let (table_id, iter) = self.relational_db() .index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (point, (chunks, rows_scanned, bytes_scanned)) = match iter { @@ -1446,6 +1487,107 @@ mod test { Ok(db) } + #[test] + fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; + use spacetimedb_primitives::ViewId; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("A")?, None); + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "A", "uncommitted")?; + env.tx.set_raw(tx); + assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + let view = ViewCallInfo::anonymous(ViewId(88)); + env.func_type = FuncCallType::View(view.clone()); + assert_eq!(env.env_get("MISSING")?, None); + let tx = env.tx.take()?; + db.commit_tx(tx)?; + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "MISSING", "")?; + assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("MISSING")?, None); + assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); + Ok(()) + } + + #[test] + fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::ST_ENV_ID; + let db = relational_db()?; + let (env, _runtime) = instance_env(db.clone())?; + let mut slot = env.tx.clone(); + let protected = [(ST_ENV_ID, "st_env", to_vec("TOKEN")?)]; + let tx = begin_mut_tx(&db); + let (tx, result) = slot.set(tx, || -> Result<()> { + for (table, name, point) in &protected { + // Host lookup remains available, independently of module lookup. + let (index, index_name) = { + let tx = env.get_tx()?; + let schema = db.schema_for_table_mut(&tx, *table)?; + let index = &schema.indexes[0]; + (index.index_id, index.index_name.to_string()) + }; + assert!(matches!(env.table_id_from_name(name), Err(NodesError::TableNotFound))); + assert!(matches!( + env.index_id_from_name(&index_name), + Err(NodesError::IndexNotFound) + )); + assert!(matches!(env.insert(*table, &mut []), Err(NodesError::TableNotFound))); + assert!(matches!( + env.update(*table, index, &mut []), + Err(NodesError::TableNotFound) + )); + assert!(matches!(env.clear(*table), Err(NodesError::TableNotFound))); + assert!(matches!( + env.datastore_table_row_count(*table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_table_scan_bsatn_chunks(&mut ChunkPool::default(), *table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_all_by_eq_bsatn(*table, &[]), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_index_scan_point_bsatn_chunks(&mut ChunkPool::default(), index, point), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_point_bsatn(index, point), + Err(NodesError::TableNotFound) + )); + let bound = to_vec(&Bound::::Unbounded)?; + assert!(matches!( + env.datastore_index_scan_range_bsatn_chunks( + &mut ChunkPool::default(), + index, + &[], + 0.into(), + &bound, + &bound + ), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_range_bsatn(index, &[], 0.into(), &bound, &bound), + Err(NodesError::TableNotFound) + )); + } + Ok(()) + }); + let _ = db.rollback_mut_tx(tx); + result + } + /// Generate a `ProductValue` for use in [create_table_with_index] fn product_row(i: usize) -> ProductValue { let str = i.to_string(); diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 31aa62ec6bc..f28a515c910 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -191,6 +191,7 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index a09e7cbba0c..029d5836282 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,6 +62,8 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), + // sys2.2 is reserved for invocation authority. + (2, 3) => Ok(v2::sys_v2_3(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index f49d2260549..8fb376ec7af 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,24 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +} + +fn env_get<'s>( + scope: &mut PinScope<'s, '_>, + args: FunctionCallbackArguments<'s>, +) -> SysCallResult> { + let key: String = deserialize_js(scope, args.get(0))?; + match get_env(scope)?.instance_env.env_get(&key)? { + Some(value) => Ok(value + .into_string(scope) + .map_err(|_| RangeError("environment value could not be represented").throw(scope))? + .into()), + None => Ok(v8::null(scope).into()), + } +} + /// Registers a function in `module` /// where the function has `name` and does `body`. fn register_module_fun( diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7fd18b5e1..dc8baa44227 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -362,6 +362,8 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::DecodeRow(_) => errno::BSATN_DECODE_ERROR, NodesError::DecodeValue(_) => errno::BSATN_DECODE_ERROR, NodesError::TableNotFound => errno::NO_SUCH_TABLE, + NodesError::InvalidEnvironmentKey => errno::HOST_CALL_FAILURE, + NodesError::EnvironmentSourceLimit => errno::NO_SPACE, NodesError::IndexNotFound => errno::NO_SUCH_INDEX, NodesError::IndexNotUnique => errno::INDEX_NOT_UNIQUE, NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, @@ -442,6 +444,8 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, + // ABI10.6 is reserved for invocation authority. + "spacetime_10.7"::env_get, } $link_async! { diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index da4a2f3987f..23d33983440 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -31,6 +31,10 @@ use std::sync::Arc; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; +/// Env reads may retain at most 2 MiB of value bytes outside the Wasm heap. +/// Other outstanding byte sources count against this interface's handle limit. +const MAX_OUTSTANDING_ENV_SOURCES: usize = 256; + /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -253,7 +257,14 @@ impl WasmInstanceEnv { // This allows the module to avoid allocating and make a system call in those cases. if bytes.is_empty() { Ok(BytesSourceId::INVALID) - } else if bytes.len() > u32::MAX as usize { + } else { + self.create_present_bytes_source(bytes) + } + } + + /// Allocate a valid source even for an empty value when zero means absence. + fn create_present_bytes_source(&mut self, bytes: bytes::Bytes) -> RtResult { + if bytes.len() > u32::MAX as usize { // There's no inherent reason we need to error here, // other than that it makes it impossible to report the length in `bytes_source_remaining_length` // and that all of our usage of `BytesSource`s as of writing (pgoldman 2025-09-26) @@ -1572,6 +1583,34 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + /// Finds the JWT payload associated with `connection_id`. /// A `[ByteSourceId]` for the payload will be written to `target_ptr`. /// If nothing is found for the connection, `[ByteSourceId::INVALID]` (zero) is written to `target_ptr`. diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index e34367ee0de..a5316ceb95c 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 5); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index ddd57d780c5..e927921c886 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -88,7 +88,23 @@ fn run_inner( // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth) + let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + // Check mutation authority while the automatic rollback guard owns + // the transaction, including rejected administrative statements. + if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + return Err(anyhow!( + "Caller {} is not authorized to run SQL mutations", + auth.caller() + )); + } + if let Statement::DML(dml) = &stmt + && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID + { + return Err(anyhow!( + "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + )); + } + Ok(stmt) })?; let mut metrics = ExecutionMetrics::default(); @@ -141,14 +157,21 @@ fn run_inner( trapped, )) } - Statement::DML(stmt) => { - // An extra layer of auth is required for DML - if !auth.has_write_access() { - return Err(anyhow!("Caller {} is not authorized to run SQL DML statements", auth.caller()).into()); - } - + stmt @ (Statement::DML(_) | Statement::Environment(_)) => { // Evaluate the mutation - let (mut tx, _) = db.with_auto_rollback(tx, |tx| execute_dml_stmt(&auth, stmt, tx, &mut metrics))?; + let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + match stmt { + Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, + Statement::Environment(environment) => match environment.value { + Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, + None => { + crate::db::environment::delete(&db, tx, &environment.key)?; + } + }, + Statement::Select(_) => unreachable!(), + } + Ok(()) + })?; // Update transaction metrics tx.metrics.merge(metrics); @@ -243,6 +266,76 @@ pub(crate) mod tests { use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_schema::table_name::TableName; + #[test] + fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + use spacetimedb_lib::identity::SqlPermission; + let db = TestDB::in_memory().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let owner = AuthCtx::for_current(Identity::ZERO); + let viewer = AuthCtx::with_permissions( + Identity::ONE, + Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), + ); + let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let execute = |statement: &str, auth: AuthCtx| { + runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) + }; + let value = "it's \\quoted;\nUTF-8 é\0tail"; + execute( + &format!( + "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", + value.replace('\'', "''") + ), + owner.clone(), + ) + .unwrap(); + execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); + let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) + .unwrap() + .rows; + assert_eq!(rows, vec![product![value]]); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [viewer, outsider] { + assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); + assert!(execute("DELETE env.Mixed_Key", auth).is_err()); + } + for statement in [ + "SET env.EMPTY = 5".to_string(), + "SET env.\"BAD-KEY\" = 'value'".to_string(), + format!("SET env.EMPTY = '{}'", "x".repeat(8193)), + "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), + "DELETE env.Mixed_Key WHERE true".to_string(), + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), + "UPDATE st_env SET value = 'bypass'".to_string(), + "DELETE FROM st_env".to_string(), + ] { + assert!( + execute(&statement, owner.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + // Every rejected path released its transaction and preserved old data. + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + .unwrap() + .rows, + vec![product![""]] + ); + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + .unwrap() + .rows, + vec![product![value]] + ); + execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) + .unwrap() + .rows + .is_empty()); + } + /// Short-cut for simplify test execution pub(crate) fn run_for_testing(db: &Arc, sql_text: &str) -> Result, DBError> { let (subs, runtime) = ModuleSubscriptions::for_test_new_runtime(db.clone()); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 0ce629524c4..9169a96b45c 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -357,6 +357,8 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + let env = crate::system_tables::st_env_schema(); + self.create_table(env.table_id, env.into()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 7293a81fba0..cf5fd662c63 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1079,6 +1079,7 @@ pub(crate) mod tests { use super::*; use crate::error::IndexError; use crate::locking_tx_datastore::tx_state::PendingSchemaChange; + use crate::system_tables::ST_ENV_ID; use crate::system_tables::{ system_tables, StColumnRow, StConnectionCredentialsFields, StConstraintData, StConstraintFields, StConstraintRow, StEventTableFields, StIndexAlgorithm, StIndexFields, StIndexRow, StRowLevelSecurityFields, @@ -1558,6 +1559,7 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, + TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1655,6 +1657,8 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 0, name: "table_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 1, name: "col_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1687,6 +1691,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1732,6 +1737,7 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 23, table_id: ST_INDEX_ACCESSOR_ID.into(), unique_columns: col(1), constraint_name: "st_index_accessor_accessor_name_key", }, ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, + ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col_list![0], constraint_name: "st_env_key_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2165,6 +2171,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 2635093cc2d..d39cd14923c 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -207,7 +207,7 @@ pub enum SystemTable { st_event_table = ST_EVENT_TABLE_ID.0 as _, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 21] { [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -230,6 +230,7 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + st_env_schema(), ] } @@ -313,6 +314,9 @@ macro_rules! st_fields_enum { } } +mod environment; +pub use environment::*; + // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, @@ -670,6 +674,8 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + environment::register_table(&mut builder); + let result = builder .finish() .try_into() @@ -694,6 +700,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_EVENT_TABLE_NAME); validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); + environment::validate_table(&result); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); result @@ -743,6 +750,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.insert("st_env_key_key", ConstraintId(26)); m }; } @@ -781,6 +789,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.insert("st_env_key_idx_btree", IndexId(30)); m }; } @@ -970,6 +979,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), + ST_ENV_ID => Some(st_env_schema()), _ => None, } } diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs new file mode 100644 index 00000000000..82395f4bd77 --- /dev/null +++ b/crates/datastore/src/system_tables/environment.rs @@ -0,0 +1,45 @@ +//! Private database environment state. Values follow ordinary table durability. +use super::*; +pub const ST_ENV_ID: TableId = TableId(21); +pub const ST_ENV_NAME: &str = "st_env"; +st_fields_enum!(enum StEnvFields { "key", Key = 0, "value", Value = 1, }); +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StEnvRow { + pub key: String, + pub value: String, +} +impl TryFrom> for StEnvRow { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } +} +impl From for ProductValue { + fn from(row: StEnvRow) -> Self { + to_product_value(&row) + } +} +pub(super) fn register_table(builder: &mut RawModuleDefV9Builder) { + let ty = builder.add_type::(); + builder + .build_table(ST_ENV_NAME, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); +} +pub(super) fn validate_table(def: &ModuleDef) { + validate_system_table::(def, ST_ENV_NAME); +} +pub(crate) fn st_env_schema() -> TableSchema { + st_schema(ST_ENV_NAME, ST_ENV_ID) +} +/// Module code must use env_get even when it guesses numeric identifiers. +pub fn is_module_restricted_table(table: TableId) -> bool { + table == ST_ENV_ID +} +pub fn is_module_restricted_index(index: IndexId) -> bool { + index == IndexId(30) +} diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index 1e7315cb4eb..d61ade64f54 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,6 +132,10 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { + #[error(transparent)] + Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), + #[error("environment values must be SQL string literals")] + EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index 9fbdb869a20..3293553f1b6 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,6 +31,12 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), + Environment(EnvironmentWrite), +} + +pub struct EnvironmentWrite { + pub key: Box, + pub value: Option>, } pub enum DML { @@ -454,6 +460,21 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, _auth: &AuthCtx) -> T SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), + SqlAst::Environment(environment) => { + // Resolve through the normal private-table visibility check. + tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; + let key = &*environment.key.0; + spacetimedb_lib::environment::validate_key(key)?; + let value = match environment.value { + None => None, + Some(SqlLiteral::Str(value)) => { + spacetimedb_lib::environment::validate_value(&value)?; + Some(value) + } + Some(_) => return Err(TypingError::EnvironmentValueType), + }; + Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) + } } } diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs new file mode 100644 index 00000000000..17b5c3f3293 --- /dev/null +++ b/crates/lib/src/environment.rs @@ -0,0 +1,62 @@ +//! Limits shared by the database environment store and its clients. + +pub const MAX_ENV_KEY_BYTES: usize = 256; +pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; +pub const MAX_ENV_VARS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentValidationError { + InvalidKey, + ValueTooLarge, + TooManyVariables, +} + +impl std::fmt::Display for EnvironmentValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::InvalidKey => "invalid POSIX environment variable name (maximum 256 bytes)", + Self::ValueTooLarge => "environment value exceeds 8192 UTF-8 bytes", + Self::TooManyVariables => "environment store exceeds 256 variables", + }) + } +} + +impl std::error::Error for EnvironmentValidationError {} + +pub fn validate_key(key: &str) -> Result<(), EnvironmentValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > MAX_ENV_KEY_BYTES + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(EnvironmentValidationError::InvalidKey); + } + Ok(()) +} + +/// NUL is representable in the database. Container launch separately rejects it. +pub fn validate_value(value: &str) -> Result<(), EnvironmentValidationError> { + if value.len() > MAX_ENV_VALUE_BYTES { + return Err(EnvironmentValidationError::ValueTooLarge); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_utf8_byte_limits_and_posix_keys_without_container_policy() { + for key in ["", "1FIRST", "A=B", "A\0B", "é", "A-B"] { + assert_eq!(validate_key(key), Err(EnvironmentValidationError::InvalidKey)); + } + assert!(validate_key(&"A".repeat(256)).is_ok()); + assert!(validate_key(&"A".repeat(257)).is_err()); + assert!(validate_key("SPACETIMEDB_USER_DATA").is_ok()); + assert!(validate_value("\0").is_ok()); + assert!(validate_value(&"é".repeat(4096)).is_ok()); + assert!(validate_value(&"é".repeat(4097)).is_err()); + } +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index aedded78ad1..55d5621a16d 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -13,6 +13,7 @@ use std::collections::{btree_map, BTreeMap}; pub mod connection_id; pub mod db; mod direct_index_key; +pub mod environment; pub mod error; mod filterable_value; pub mod http; diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 9225393eb34..69168032a0a 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,11 +59,11 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } match parse_and_type_sql(sql, tx, auth)? { - stmt @ Statement::DML(_) => Ok(stmt), + stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index be7b753f395..a7593cc4582 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,6 +19,15 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), + /// Administrative environment mutation, distinct from generic table DML. + Environment(SqlEnvironment), +} + +#[derive(Debug)] +pub struct SqlEnvironment { + pub key: SqlIdent, + /// None is DELETE; a string literal, including empty, is SET. + pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index 0508baae5b2..a70ea4d5b39 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,11 +133,13 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, + keywords::Keyword, parser::Parser, + tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -148,7 +150,36 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; + // DELETE env.KEY is a SpacetimeDB administrative statement, not the + // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression + // parser, including comments and SQL string escaping, for this extension. + let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; + let verb = parser.peek_token().token; + let environment_prefix = matches!(parser.peek_nth_token(1).token, + Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) + && parser.peek_nth_token(2).token == Token::Period; + if environment_prefix + && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) + { + parser.next_token(); + parser.next_token(); + parser.next_token(); + let key = SqlIdent(parser.parse_identifier()?.value.into()); + let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { + if !parser.parse_keyword(Keyword::TO) { + parser.expect_token(&Token::Eq)?; + } + Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) + } else { + None + }; + let _ = parser.consume_token(&Token::SemiColon); + if parser.peek_token().token != Token::EOF { + return Err(SqlUnsupported::MultiStatement.into()); + } + return Ok(SqlAst::Environment(SqlEnvironment { key, value })); + } + let mut stmts = parser.parse_statements()?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs new file mode 100644 index 00000000000..7cdf4fecf0e --- /dev/null +++ b/crates/testing/tests/environment.rs @@ -0,0 +1,177 @@ +//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +use serial_test::serial; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; + +async fn sql(module: &ModuleHost, statement: String) -> Vec { + spacetimedb::sql::execute::run( + module.relational_db().clone(), + statement, + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await + .unwrap() + .rows +} + +async fn set_environment(module: &ModuleHost, key: &str, value: &str) { + sql(module, format!("SET env.{key} = '{}'", value.replace('\'', "''"))).await; +} + +fn exercise_fixture(name: &str) { + CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { + let module = handle.client.module(); + for (key, expected) in [ + ("MISSING", None), + ("EMPTY", Some("".to_string())), + ("UTF8", Some("héllo 🌍".to_string())), + ("NUL", Some("before\0after".to_string())), + ("MAXIMUM", Some("é".repeat(4096))), + ] { + if let Some(value) = &expected { + set_environment(&module, key, value).await; + } + let args = product![key, expected.clone()]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + let result = result + .map_err(anyhow::Error::from) + .and_then(|r| r.outcome.into_result()); + assert!( + result.is_ok(), + "{name} {key}: {result:?}; module log: {}", + handle.read_log(None).await + ); + let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(expected.clone())); + if expected.is_some() { + set_environment(&module, key, "updated").await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); + sql(&module, format!("DELETE env.{key}")).await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(None::)); + } + } + if name == "environment-test" { + set_environment(&module, "HANDLER", "handler snapshot").await; + let (_, body) = module + .call_http_handler( + module.info.module_def.http_handler_ids_and_defs().next().unwrap().0, + spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: "/environment".into(), + version: spacetimedb_lib::http::Version::Http11, + }, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(&body[..], b"handler snapshot"); + // This view first reads a missing key. Its dependency must survive + // absence, and normal SQL mutations must invalidate its cached row. + let read_view = || "SELECT * FROM environment_value".to_string(); + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "WATCHED", "first").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("first".to_string())]] + ); + set_environment(&module, "WATCHED", "second").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("second".to_string())]] + ); + sql(&module, "DELETE env.WATCHED".into()).await; + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "LIMIT", &"x".repeat(8192)).await; + for _ in 0..2 { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "bounded_environment_sources", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + } + } + let args = product!["A=B", None::]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + module.exit().await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); + }); +} + +#[test] +#[serial] +fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("environment-test"); +} + +#[test] +#[serial] +fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-ts"); +} + +#[test] +#[serial] +fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cpp"); +} + +#[test] +#[serial] +fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cs"); +} diff --git a/modules/environment-test/Cargo.toml b/modules/environment-test/Cargo.toml new file mode 100644 index 00000000000..9ae6466b734 --- /dev/null +++ b/modules/environment-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "environment-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs new file mode 100644 index 00000000000..07c8c6aedd8 --- /dev/null +++ b/modules/environment-test/src/lib.rs @@ -0,0 +1,69 @@ +use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.get(&key), expected); + assert_eq!(ctx.as_read_only().env.get(&key), expected); + assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} + +#[derive(SpacetimeType)] +pub struct EnvironmentValue { + pub value: Option, +} + +#[spacetimedb::view(accessor = environment_value, public)] +pub fn environment_value(ctx: &AnonymousViewContext) -> Option { + Some(EnvironmentValue { + value: ctx.env.get("WATCHED"), + }) +} + +/// Hand-written ABI callers cannot retain unbounded host allocations. +#[spacetimedb::reducer] +pub fn bounded_environment_sources(_ctx: &ReducerContext) { + use spacetimedb::sys::raw::{self, BytesSource}; + let mut sources = Vec::new(); + for i in 0..=256 { + let mut source = BytesSource::INVALID; + let status = unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }; + if i == 256 { + assert_eq!(status, 9); // NO_SPACE + } else { + assert_eq!(status, 0); + assert!(source != BytesSource::INVALID); + sources.push(source); + } + } + let mut buffer = [0u8; 8192]; + let mut len = buffer.len(); + let status = unsafe { raw::bytes_source_read(sources[0], buffer.as_mut_ptr(), &mut len) }; + assert_eq!(status, -1); + assert_eq!(len, buffer.len()); + let mut source = BytesSource::INVALID; + assert_eq!(unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }, 0); + assert!(source != BytesSource::INVALID); + // The remaining sources are released when this invocation ends. +} + +#[spacetimedb::http::handler] +pub fn handler_environment( + ctx: &mut spacetimedb::http::HandlerContext, + _request: spacetimedb::http::Request, +) -> spacetimedb::http::Response { + let outside = ctx.env.get("HANDLER"); + ctx.with_tx(|tx| assert_eq!(tx.env.get("HANDLER"), outside)); + spacetimedb::http::Response::new(spacetimedb::http::Body::from_bytes(outside.unwrap())) +} + +#[spacetimedb::http::router] +pub fn router() -> spacetimedb::http::Router { + spacetimedb::http::Router::new().get("/environment", handler_environment) +} diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index 3a93f0b20ec..1102ef8be5f 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -720,3 +720,15 @@ SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { SPACETIMEDB_HTTP_ROUTER(router) { return Router().get("/get", get_simple); } + +SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); + return Ok(); +} +SPACETIMEDB_PROCEDURE(std::optional, read_environment, ProcedureContext ctx, std::string key) { + const auto outside = ctx.env.get(key); + ctx.with_tx([&](TxContext& tx) { + if (tx.env.get(key) != outside) LOG_PANIC("transaction environment value mismatch"); + }); + return outside; +} diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs new file mode 100644 index 00000000000..1798d7bda44 --- /dev/null +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -0,0 +1,25 @@ +#pragma warning disable STDB_UNSTABLE +namespace SpacetimeDB.Modules.ModuleTestCs; + +using SpacetimeDB; + +public static partial class EnvironmentTests +{ + [Reducer] + public static void expect_environment(ReducerContext ctx, string key, string? expected) + { + if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); + } + + [Procedure] + public static string? read_environment(ProcedureContext ctx, string key) + { + var outside = ctx.Env.Get(key); + ctx.WithTx(tx => + { + if (tx.Env.Get(key) != outside) throw new Exception("transaction environment value mismatch"); + return true; + }); + return outside; + } +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 0eba467a7d0..10520d2b72c 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -552,3 +552,26 @@ export const libHello = spacetimedb.httpHandler((ctx, req) => { export const router = spacetimedb.httpRouter( new Router().get('/get', getSimple).get('/lib-hello', libHello) ); + +// Dedicated environment ABI integration exercised by crates/testing. +export const expect_environment = spacetimedb.reducer( + { key: t.string(), expected: t.option(t.string()) }, + (ctx, { key, expected }) => { + if (ctx.env.get(key) !== (expected ?? null)) { + throw new Error('environment value mismatch'); + } + } +); +export const read_environment = spacetimedb.procedure( + { key: t.string() }, + t.option(t.string()), + (ctx, { key }) => { + const outside = ctx.env.get(key); + ctx.withTx(tx => { + if (tx.env.get(key) !== outside) { + throw new Error('transaction environment value mismatch'); + } + }); + return outside ?? undefined; + } +); From 4613001fad5b6f4cf8ce44ed1a473746ea8883b3 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 19:23:28 -0400 Subject: [PATCH 02/19] Implement typed publish-only database environments --- Cargo.lock | 2 +- crates/bindings-cpp/CMakeLists.txt | 27 + crates/bindings-cpp/README.md | 48 ++ .../include/spacetimedb/environment.h | 75 ++- .../spacetimedb/environment_declaration.h | 153 ++++++ .../spacetimedb/environment_prelude.h.in | 7 + .../autogen/EnvironmentConstraint.g.h | 7 + .../autogen/EnvironmentDeclaration.g.h | 15 + .../autogen/RawModuleDefV10Section.g.h | 3 +- crates/bindings-cpp/src/abi/wasi_shims.cpp | 17 +- .../bindings-cpp/src/internal/v10_builder.cpp | 5 + .../tests/environment/CMakeLists.txt | 15 + .../tests/environment/declarations.h | 11 + .../tests/environment/fallback.cpp | 8 + .../bindings-cpp/tests/environment/main.cpp | 56 +++ .../bindings-cpp/tests/environment/other.cpp | 6 + .../Codegen.Tests/EnvironmentTests.cs | 137 ++++++ crates/bindings-csharp/Codegen.Tests/Tests.cs | 11 +- .../diag/snapshots/Module#FFI.verified.cs | 12 +- .../snapshots/Module#FFI.verified.cs | 12 +- .../server/snapshots/Module#FFI.verified.cs | 12 +- crates/bindings-csharp/Codegen/Environment.cs | 158 ++++++ crates/bindings-csharp/Codegen/Module.cs | 10 +- crates/bindings-csharp/README.md | 27 + crates/bindings-csharp/Runtime/Attrs.cs | 11 + .../Autogen/EnvironmentConstraint.g.cs | 11 + .../Autogen/EnvironmentDeclaration.g.cs | 29 ++ .../Autogen/RawModuleDefV10Section.g.cs | 3 +- .../bindings-csharp/Runtime/Internal/FFI.cs | 3 +- .../Runtime/Internal/Module.cs | 6 + .../Runtime/build/SpacetimeDB.Runtime.targets | 3 + crates/bindings-macro/src/environment.rs | 252 ++++++++++ crates/bindings-macro/src/lib.rs | 7 + crates/bindings-typescript/README.md | 26 + .../src/lib/autogen/types.ts | 29 ++ .../src/lib/environment.ts | 59 ++- .../bindings-typescript/src/lib/reducers.ts | 4 +- crates/bindings-typescript/src/lib/schema.ts | 2 + .../src/server/environment.ts | 86 +++- .../src/server/http_handlers.ts | 4 +- .../src/server/procedures.ts | 6 +- .../bindings-typescript/src/server/runtime.ts | 6 +- .../bindings-typescript/src/server/schema.ts | 29 +- .../bindings-typescript/src/server/views.ts | 6 +- .../tests/__mocks__/spacetime-sys.ts | 4 + .../tests/environment.test.ts | 149 ++++++ crates/bindings-typescript/vitest.config.ts | 1 + crates/bindings/src/lib.rs | 29 +- crates/bindings/src/rt.rs | 9 + crates/bindings/tests/environment.rs | 4 + crates/bindings/tests/pass/environment.rs | 24 + crates/cli/src/lib.rs | 2 + crates/cli/src/spacetime_config.rs | 57 ++- .../cli/src/spacetime_config/environment.rs | 286 +++++++++++ crates/cli/src/subcommands/env.rs | 284 +++++++++++ crates/cli/src/subcommands/mod.rs | 1 + crates/cli/src/subcommands/publish.rs | 41 +- .../src/subcommands/publish/environment.rs | 180 +++++++ .../subcommands/publish/environment/tests.rs | 307 ++++++++++++ crates/client-api-messages/src/lib.rs | 2 + crates/client-api-messages/src/publish.rs | 117 +++++ crates/client-api/src/lib.rs | 23 + crates/client-api/src/routes/database.rs | 24 +- .../routes/database/publish_environment.rs | 139 ++++++ crates/core/src/db/environment.rs | 139 ++++-- crates/core/src/host/host_controller.rs | 236 ++++++--- crates/core/src/host/instance_env.rs | 207 +++++++- crates/core/src/host/mod.rs | 4 +- crates/core/src/host/module_host.rs | 169 ++++++- crates/core/src/host/v8/mod.rs | 49 +- crates/core/src/host/v8/syscall/common.rs | 8 +- .../src/host/wasm_common/module_host_actor.rs | 433 ++++++++++++++--- .../src/host/wasmtime/wasm_instance_env.rs | 14 +- .../core/src/host/wasmtime/wasmtime_module.rs | 4 +- crates/core/src/sql/execute.rs | 97 ++-- crates/expr/src/errors.rs | 4 - crates/expr/src/statement.rs | 21 - crates/lib/src/db/raw_def/v10.rs | 24 + crates/lib/src/environment.rs | 321 ++++++++++++ crates/query/src/lib.rs | 2 +- crates/schema/src/def.rs | 36 +- crates/schema/src/def/validate/v10.rs | 110 ++++- crates/schema/src/def/validate/v9.rs | 2 + crates/schema/src/error.rs | 8 + crates/smoketests/modules/Cargo.lock | 7 + crates/smoketests/modules/Cargo.toml | 1 + .../modules/environment-publish/Cargo.toml | 11 + .../modules/environment-publish/src/lib.rs | 46 ++ .../tests/standalone/cli/environment.rs | 439 +++++++++++++++++ crates/smoketests/tests/standalone/cli/mod.rs | 1 + crates/sql-parser/src/ast/sql.rs | 9 - crates/sql-parser/src/parser/sql.rs | 35 +- crates/standalone/src/control_db.rs | 50 +- .../standalone/src/control_db/environment.rs | 316 ++++++++++++ crates/standalone/src/environment_tests.rs | 190 ++++++++ crates/standalone/src/lib.rs | 459 +++++++++++------ crates/testing/src/modules.rs | 85 +++- crates/testing/tests/environment.rs | 460 +++++++++++++++--- .../00300-spacetime-publish.md | 6 + .../00700-environment-variables.md | 235 +++++++++ .../00100-cli-reference.md | 66 ++- docs/docusaurus.config.ts | 2 + modules/environment-test/src/lib.rs | 60 ++- modules/module-test-cpp/CMakeLists.txt | 3 + modules/module-test-cpp/environment.h | 9 + modules/module-test-cpp/src/lib.cpp | 2 + modules/module-test-cs/EnvironmentTests.cs | 11 + .../module-test-ts/src/environment_sys.d.ts | 4 + modules/module-test-ts/src/index.ts | 26 +- modules/module-test-ts/src/lib_submodule.ts | 36 ++ 110 files changed, 6836 insertions(+), 720 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/environment_declaration.h create mode 100644 crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h create mode 100644 crates/bindings-cpp/tests/environment/CMakeLists.txt create mode 100644 crates/bindings-cpp/tests/environment/declarations.h create mode 100644 crates/bindings-cpp/tests/environment/fallback.cpp create mode 100644 crates/bindings-cpp/tests/environment/main.cpp create mode 100644 crates/bindings-cpp/tests/environment/other.cpp create mode 100644 crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs create mode 100644 crates/bindings-csharp/Codegen/Environment.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs create mode 100644 crates/bindings-macro/src/environment.rs create mode 100644 crates/bindings-typescript/tests/environment.test.ts create mode 100644 crates/bindings/tests/environment.rs create mode 100644 crates/bindings/tests/pass/environment.rs create mode 100644 crates/cli/src/spacetime_config/environment.rs create mode 100644 crates/cli/src/subcommands/env.rs create mode 100644 crates/cli/src/subcommands/publish/environment.rs create mode 100644 crates/cli/src/subcommands/publish/environment/tests.rs create mode 100644 crates/client-api-messages/src/publish.rs create mode 100644 crates/client-api/src/routes/database/publish_environment.rs create mode 100644 crates/smoketests/modules/environment-publish/Cargo.toml create mode 100644 crates/smoketests/modules/environment-publish/src/lib.rs create mode 100644 crates/smoketests/tests/standalone/cli/environment.rs create mode 100644 crates/standalone/src/control_db/environment.rs create mode 100644 crates/standalone/src/environment_tests.rs create mode 100644 docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md create mode 100644 modules/module-test-cpp/environment.h create mode 100644 modules/module-test-ts/src/environment_sys.d.ts diff --git a/Cargo.lock b/Cargo.lock index 9b030bc73ff..384c70585bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2302,7 +2302,7 @@ dependencies = [ name = "environment-test" version = "0.0.0" dependencies = [ - "spacetimedb 2.10.0", + "spacetimedb", ] [[package]] diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index dc5d1433fce..695c91f5b5f 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -28,6 +28,33 @@ add_library(spacetimedb::spacetimedb_cpp_library ALIAS spacetimedb_cpp_library) target_sources(spacetimedb_cpp_library PRIVATE ${LIBRARY_SOURCES}) +# A declared Environment adds named methods to the context member type. Every +# SDK and module translation unit must see the same prelude before any context. +if(NOT DEFINED SPACETIMEDB_ENV_HEADER) + set(SPACETIMEDB_ENV_HEADER "" CACHE FILEPATH "Absolute module environment declaration header") +endif() +if(SPACETIMEDB_ENV_HEADER) + if(NOT IS_ABSOLUTE "${SPACETIMEDB_ENV_HEADER}" OR NOT EXISTS "${SPACETIMEDB_ENV_HEADER}") + message(FATAL_ERROR "SPACETIMEDB_ENV_HEADER must name an existing absolute declaration header") + endif() + # The standalone WASI ABI shims have no context or Environment type. Their + # hand-written ABI declarations must not include the C++ standard library's + # WASI declarations through the module prelude. + set_source_files_properties(src/abi/wasi_shims.cpp PROPERTIES + COMPILE_DEFINITIONS SPACETIMEDB_WASI_SHIMS=1) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/include/spacetimedb/environment_prelude.h.in + ${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h + @ONLY) + set(SPACETIMEDB_ENV_PRELUDE "${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h") + target_compile_definitions(spacetimedb_cpp_library PUBLIC SPACETIMEDB_ENV_HEADER_ACTIVE=1) + if(MSVC) + target_compile_options(spacetimedb_cpp_library PUBLIC "/FI${SPACETIMEDB_ENV_PRELUDE}") + else() + target_compile_options(spacetimedb_cpp_library PUBLIC -include "${SPACETIMEDB_ENV_PRELUDE}") + endif() +endif() + # Require C++20 for consumers of this library without forcing global flags target_compile_features(spacetimedb_cpp_library PUBLIC cxx_std_20) target_compile_definitions(spacetimedb_cpp_library PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..155fafedfda 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -275,3 +275,51 @@ See the `modules/*-cpp/src/` directory for example modules: This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. + +### Declared environment + +Declare the complete environment schema in a dedicated header, before other SDK +includes. Values are supplied at publish time and are never embedded in this +header. For example, `environment.h`: + +```cpp +#pragma once +#include +SPACETIMEDB_ENV( + (API_URL, std::string), + (MODE, std::string, ("prod", "dev")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +Select it in your project's `CMakeLists.txt` **before** adding the SDK directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +add_subdirectory(path/to/bindings-cpp sdk) +add_executable(my_module src/lib.cpp) +target_link_libraries(my_module PRIVATE spacetimedb_cpp_library) +``` + +The SDK's CMake target includes the declaration header before every SDK and module +translation unit that uses contexts. The standalone WASI ABI shim is excluded +because it has no SDK context types. This keeps the context type consistent; +including the header in only one source file is insufficient. The normal module +source can then use the usual umbrella include: + +```cpp +#include +void example(SpacetimeDB::ReducerContext ctx) { + std::string mode = ctx.env.MODE(); + std::optional level = ctx.env.LOG_LEVEL(); +} +``` + +An empty `SPACETIMEDB_ENV()` or an omitted declaration selects an empty schema. +`get`, C++ keywords, and names colliding with the accessor type do not create named +methods; use the checked `ctx.env.get("get")` form for such declared names. Unknown +keys fail at runtime, including when the module declares no environment variables. +Only the host's root module entry may read environment values. Ordinary C++ helper +calls retain their caller's host scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h index f2df08234d0..06020dcee2f 100644 --- a/crates/bindings-cpp/include/spacetimedb/environment.h +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -6,11 +6,14 @@ #include #include #include +#include +#include +#include namespace SpacetimeDB { /// Read-only database environment. Reads use the current transaction, or a /// short snapshot in a procedure outside a transaction. Values are not cached. -class Environment { +class EnvironmentBase { public: std::optional get(std::string_view key) const { if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); @@ -30,5 +33,75 @@ class Environment { } } }; + +namespace Internal { +inline std::vector& environment_declarations() { + static std::vector declarations; + return declarations; +} + +template +inline constexpr bool environment_string = std::is_same_v || std::is_same_v>; + +template +T read_environment(std::string_view key) { + static_assert(environment_string, "Environment declarations require string or optional"); + auto value = EnvironmentBase{}.get(key); + if constexpr (std::is_same_v) { + if (!value) LOG_PANIC("required environment value is absent"); + return std::move(*value); + } else { return value; } +} + +template +EnvironmentDeclaration declare_environment(std::string name) { + static_assert(environment_string, "Environment declarations require string or optional"); + EnvironmentConstraint constraint; + constraint.set<0>(std::monostate{}); + return {std::move(name), std::move(constraint), std::is_same_v>}; +} + +// Preserve the complete source literal, including embedded NUL bytes. Implicit +// conversion through std::string(const char*) would truncate those constraints. +struct EnvironmentLiteral { + std::string value; + template + EnvironmentLiteral(const char (&text)[N]) : value(text, N - 1) {} + EnvironmentLiteral(std::string text) : value(std::move(text)) {} +}; + +template +EnvironmentDeclaration declare_environment(std::string name, std::initializer_list allowed) { + auto declaration = declare_environment(std::move(name)); + if (allowed.size() == 0) LOG_PANIC("environment literal union cannot be empty"); + std::unordered_set unique; + std::vector values; + values.reserve(allowed.size()); + for (const auto& literal : allowed) { + const auto& value = literal.value; + if (value.size() > 8192 || !unique.insert(value).second) LOG_PANIC("invalid environment literal union"); + values.push_back(value); + } + if (values.size() == 1) declaration.constraint.template set<1>(std::move(values.front())); + else declaration.constraint.template set<2>(std::move(values)); + return declaration; +} + +inline void validate_environment_declarations() { + const auto& declarations = environment_declarations(); + if (declarations.size() > 256) LOG_PANIC("too many environment declarations"); + std::unordered_set keys; + const auto initial = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; }; + for (const auto& declaration : declarations) { + const auto& key = declaration.name; + if (key.empty() || key.size() > 256 || !initial(key[0]) || !keys.insert(key).second) LOG_PANIC("invalid environment declaration name"); + for (char c : key) if (!initial(c) && !(c >= '0' && c <= '9')) LOG_PANIC("invalid environment declaration name"); + } +} +} + +#ifndef SPACETIMEDB_ENV_DECLARATION +class Environment : public EnvironmentBase {}; +#endif } #endif diff --git a/crates/bindings-cpp/include/spacetimedb/environment_declaration.h b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h new file mode 100644 index 00000000000..b3df8028d5b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h @@ -0,0 +1,153 @@ +#pragma once +// Put SPACETIMEDB_ENV in a module declaration header selected by CMake's +// SPACETIMEDB_ENV_HEADER, before including any SDK context or umbrella header. +#ifndef SPACETIMEDB_ENV_HEADER_ACTIVE +#error "Configure SPACETIMEDB_ENV_HEADER before adding the SDK CMake directory" +#endif +#ifdef SPACETIMEDB_ENVIRONMENT_H +#error "The environment declaration header must precede every SDK include" +#endif +#define SPACETIMEDB_ENV_DECLARATION 1 +#include + +#define STDB_ENV_CAT_I(a, b) a##b +#define STDB_ENV_CAT(a, b) STDB_ENV_CAT_I(a, b) +#define STDB_ENV_SECOND(a, b, ...) b +#define STDB_ENV_PROBE() ignored, 1 +#define STDB_ENV_CHECK(...) STDB_ENV_SECOND(__VA_ARGS__, 0) +#define STDB_ENV_RESERVED(name) STDB_ENV_CHECK(STDB_ENV_CAT(STDB_ENV_RESERVED_, name)) +#define STDB_ENV_KEEP_0(...) __VA_ARGS__ +#define STDB_ENV_KEEP_1(...) +#define STDB_ENV_RESERVED_alignas STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_alignof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_asm STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_cancel STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_commit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_auto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitand STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bool STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_break STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_case STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_catch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char8_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char16_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char32_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_class STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_compl STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_concept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_consteval STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constinit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_continue STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_await STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_yield STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_decltype STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_default STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_delete STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_do STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_double STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_dynamic_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_else STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_enum STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_explicit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_export STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_extern STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_false STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_float STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_for STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_friend STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_goto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_if STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_inline STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_int STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_long STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_mutable STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_namespace STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_new STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_nullptr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_operator STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_private STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_protected STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_public STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reflexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_register STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reinterpret_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_requires STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_short STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_signed STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_sizeof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_assert STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_struct STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_switch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_synchronized STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_template STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_this STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_thread_local STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_throw STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_true STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_try STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typedef STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typeid STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typename STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_union STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_unsigned STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_using STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_virtual STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_void STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_volatile STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_wchar_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_while STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_get STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_Environment STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_EnvironmentBase STDB_ENV_PROBE() + +#define STDB_ENV_PARENS () +#define STDB_ENV_EVAL1(...) __VA_ARGS__ +#define STDB_ENV_EVAL2(...) STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(__VA_ARGS__)))) +#define STDB_ENV_EVAL3(...) STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(__VA_ARGS__)))) +#define STDB_ENV_EVAL4(...) STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(__VA_ARGS__)))) +#define STDB_ENV_EVAL(...) STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(__VA_ARGS__)))) +#define STDB_ENV_EACH(macro, ...) __VA_OPT__(STDB_ENV_EVAL(STDB_ENV_EACH_I(macro, __VA_ARGS__))) +#define STDB_ENV_EACH_I(macro, tuple, ...) macro tuple __VA_OPT__(STDB_ENV_AGAIN STDB_ENV_PARENS (macro, __VA_ARGS__)) +#define STDB_ENV_AGAIN() STDB_ENV_EACH_I +#define STDB_ENV_UNPAREN(...) __VA_ARGS__ +#define STDB_ENV_MEMBER(name, type, ...) \ + STDB_ENV_CAT(STDB_ENV_KEEP_, STDB_ENV_RESERVED(name))( \ + type name() const { return ::SpacetimeDB::Internal::read_environment(#name); } \ + ) +#define STDB_ENV_METADATA(name, type, ...) \ + ::SpacetimeDB::Internal::declare_environment(#name __VA_OPT__(, { STDB_ENV_UNPAREN __VA_ARGS__ })), + +/// Declare the complete schema, never live values. One declaration per module. +/// Empty SPACETIMEDB_ENV() is supported. Reserved names retain generic get(). +#define SPACETIMEDB_ENV(...) \ + namespace SpacetimeDB { \ + class Environment : public EnvironmentBase { \ + public: STDB_ENV_EACH(STDB_ENV_MEMBER, __VA_ARGS__) \ + }; \ + namespace Internal { \ + inline const bool environment_schema_registered = [] { \ + environment_declarations() = { STDB_ENV_EACH(STDB_ENV_METADATA, __VA_ARGS__) }; \ + validate_environment_declarations(); \ + return true; \ + }(); \ + } \ + } diff --git a/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in new file mode 100644 index 00000000000..4d812bb6d05 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in @@ -0,0 +1,7 @@ +#pragma once + +// The ABI shim translation unit has no SDK contexts. All context-bearing SDK +// and module translation units see the same declared Environment definition. +#ifndef SPACETIMEDB_WASI_SHIMS +#include "@SPACETIMEDB_ENV_HEADER@" +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h new file mode 100644 index 00000000000..0b2e45c8ae6 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h @@ -0,0 +1,7 @@ +#pragma once +#include "../autogen_base.h" +#include +#include +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_TAGGED_ENUM(EnvironmentConstraint, std::monostate, std::string, std::vector) +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h new file mode 100644 index 00000000000..2d83f3c9d3b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h @@ -0,0 +1,15 @@ +#pragma once +#include "EnvironmentConstraint.g.h" +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(EnvironmentDeclaration) { + std::string name; + EnvironmentConstraint constraint; + bool optional; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, name); + ::SpacetimeDB::bsatn::serialize(writer, constraint); + ::SpacetimeDB::bsatn::serialize(writer, optional); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(name, constraint, optional) +}; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index ea2e4b5ec85..551fb3bc636 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -27,8 +27,9 @@ #include "RawScheduleDefV10.g.h" #include "RawViewPrimaryKeyDefV10.g.h" #include "RawHttpHandlerDefV10.g.h" +#include "EnvironmentDeclaration.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index 430a1f9a858..ec461c8d51d 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -4,13 +4,14 @@ #include #include - -// SpacetimeDB imports we need for console output -// Import from spacetime_10.0 module as required by SpacetimeDB ABI +// Keep this pure ABI translation unit independent of SDK opaque types: their +// standard-library helpers include wasi/api.h in Emscripten, which conflicts +// with these standalone shim definitions. Use a distinct C++ name for the raw +// host logging import; its WebAssembly signature is the same eight i32 values. extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void console_log(uint8_t log_level, const uint8_t* target, uint32_t target_len, - const uint8_t* filename, uint32_t filename_len, uint32_t line_number, - const uint8_t* message, uint32_t message_len); +void wasi_console_log(uint8_t level, const uint8_t* target_ptr, size_t target_len, + const uint8_t* filename_ptr, size_t filename_len, uint32_t line_number, + const uint8_t* message_ptr, size_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 @@ -150,7 +151,7 @@ __wasi_errno_t __wasi_fd_write(__wasi_fd_t fd, const __wasi_ciovec_t* iovs, // Make a single console_log call with the complete message uint8_t log_level = (fd == STDERR_FILENO) ? 1 : 2; // 1=WARN, 2=INFO - console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, + wasi_console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, buffer, offset); // Clean up heap allocation if needed @@ -190,4 +191,4 @@ void emscripten_notify_memory_growth(int32_t) { // No-op - memory growth is handled by the runtime } -} // extern "C" \ No newline at end of file +} // extern "C" diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index a931b79a7c2..be5bbd65035 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -1,3 +1,4 @@ +#include "spacetimedb/environment.h" #include "spacetimedb/internal/v10_builder.h" #include "spacetimedb/internal/autogen/AlgebraicType.g.h" #include "spacetimedb/internal/autogen/ProductType.g.h" @@ -315,6 +316,10 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { v10_module.sections.push_back(std::move(section_rls)); } + validate_environment_declarations(); + RawModuleDefV10Section section_environment; + section_environment.set<15>(environment_declarations()); + v10_module.sections.push_back(std::move(section_environment)); return v10_module; } diff --git a/crates/bindings-cpp/tests/environment/CMakeLists.txt b/crates/bindings-cpp/tests/environment/CMakeLists.txt new file mode 100644 index 00000000000..34223081b4a --- /dev/null +++ b/crates/bindings-cpp/tests/environment/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.16) +project(environment_declaration_tests LANGUAGES CXX) +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/declarations.h") +add_subdirectory(../.. sdk) +add_executable(environment_declaration_tests main.cpp other.cpp) +target_link_libraries(environment_declaration_tests PRIVATE spacetimedb_cpp_library) +enable_testing() +add_test(NAME environment_declaration_tests COMMAND environment_declaration_tests) +# This separate target deliberately has no forced prelude, modeling an existing +# module with no declarations and the generic checked accessor only. +add_executable(environment_fallback_tests fallback.cpp) +target_include_directories(environment_fallback_tests PRIVATE ../../include) +target_compile_features(environment_fallback_tests PRIVATE cxx_std_20) +add_test(NAME environment_fallback_tests COMMAND environment_fallback_tests) +set_tests_properties(environment_declaration_tests environment_fallback_tests PROPERTIES TIMEOUT 10) diff --git a/crates/bindings-cpp/tests/environment/declarations.h b/crates/bindings-cpp/tests/environment/declarations.h new file mode 100644 index 00000000000..ab7e2bba1b5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/declarations.h @@ -0,0 +1,11 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (FOOBAR, std::string), + (ENABLE_EMAIL, std::string, ("true", "false")), + (LOG_LEVEL, std::optional, ("debug", "info", "error")), + (DEPLOYMENT_KIND, std::string, ("production")), + (get, std::optional), + (class, std::string), + (NUL_LITERAL, std::string, ("a\0b")) +) diff --git a/crates/bindings-cpp/tests/environment/fallback.cpp b/crates/bindings-cpp/tests/environment/fallback.cpp new file mode 100644 index 00000000000..27d94126011 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/fallback.cpp @@ -0,0 +1,8 @@ +#include +#include +#include + +int main() { + static_assert(std::is_same_v>); + assert(SpacetimeDB::Internal::environment_declarations().empty()); +} diff --git a/crates/bindings-cpp/tests/environment/main.cpp b/crates/bindings-cpp/tests/environment/main.cpp new file mode 100644 index 00000000000..7d0273a8bc5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/main.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +using namespace SpacetimeDB; +std::string from_another_translation_unit(); +namespace { std::string payload; size_t position; unsigned calls; } +extern "C" Status env_get(const uint8_t* key, uint32_t length, BytesSource* out) { + const std::string name(reinterpret_cast(key), length); + ++calls; + if (name == "LOG_LEVEL") { *out = BytesSource{0}; return Status{0}; } + if (name == "FOOBAR") payload = calls == 1 ? "first" : "updated"; + else if (name == "ENABLE_EMAIL") payload = "false"; + else if (name == "DEPLOYMENT_KIND") payload = "production"; + else if (name == "get") payload = "reserved"; + else if (name == "class") payload = "keyword"; + else if (name == "NUL_LITERAL") payload = std::string("a\0b", 3); + else return Status{1}; + position = 0; + *out = BytesSource{1}; + return Status{0}; +} +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* size) { + *size = std::min(*size, payload.size() - position); + std::memcpy(out, payload.data() + position, *size); + position += *size; + return position == payload.size() ? -1 : 0; +} +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, uint32_t, const uint8_t*, size_t) {} + +int main() { + Environment env; + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + assert(env.FOOBAR() == "first"); + assert(from_another_translation_unit() == "updated"); + assert(env.ENABLE_EMAIL() == "false"); + assert(!env.LOG_LEVEL()); + assert(env.DEPLOYMENT_KIND() == "production"); + assert(env.get("get") == "reserved"); + assert(env.get("class") == "keyword"); + assert(env.NUL_LITERAL() == std::string("a\0b", 3)); + const auto& entries = Internal::environment_declarations(); + assert(entries.size() == 7); + assert(entries[0].name == "FOOBAR" && entries[0].constraint.get_tag() == 0 && !entries[0].optional); + assert(entries[1].constraint.get_tag() == 2 && entries[1].constraint.get<2>() == std::vector({"true", "false"})); + assert(entries[2].optional); + assert(entries[3].constraint.get_tag() == 1 && entries[3].constraint.get<1>() == "production"); + assert(entries[6].constraint.get<1>() == std::string("a\0b", 3)); + Internal::RawModuleDefV10Section section; + section.set<15>(entries); + assert(section.get_tag() == 15); + assert(section.get<15>() == entries); +} diff --git a/crates/bindings-cpp/tests/environment/other.cpp b/crates/bindings-cpp/tests/environment/other.cpp new file mode 100644 index 00000000000..c9ffe40f9fd --- /dev/null +++ b/crates/bindings-cpp/tests/environment/other.cpp @@ -0,0 +1,6 @@ +#include +#include +std::string from_another_translation_unit() { + return SpacetimeDB::Environment{}.FOOBAR(); +} +static_assert(std::is_same_v().env.FOOBAR()), std::string>); diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs new file mode 100644 index 00000000000..d7dd4d4382e --- /dev/null +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -0,0 +1,137 @@ +namespace SpacetimeDB.Codegen.Tests; + +using System.Reflection; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +public static class EnvironmentTests +{ + // A controlled host seam allows the generated accessor code to execute. + // Live value access remains delegated to DatabaseEnvironment.Get. + private const string Host = """ + #nullable enable + namespace SpacetimeDB { + [System.AttributeUsage(System.AttributeTargets.Struct)] + public sealed class EnvAttribute : System.Attribute {} + [System.AttributeUsage(System.AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : System.Attribute {} + public readonly struct DatabaseEnvironment { + public string? Get(string key) => Host.Get(key); + } + public static class Host { + public static int Reads; + public static string? Get(string key) { + Reads++; + return key switch { + "REQUIRED" => Reads.ToString(), "OPTIONAL" => null, + "MODE" => "prod", "Get" => "reserved", "class" => "keyword", + _ => throw new System.InvalidOperationException("undeclared environment key") + }; + } + } + } + namespace SpacetimeDB.Internal { + public abstract record EnvironmentConstraint { + public sealed record AnyString(System.ValueTuple Value) : EnvironmentConstraint; + public sealed record Literal(string Value) : EnvironmentConstraint; + public sealed record OneOf(System.Collections.Generic.List Value) : EnvironmentConstraint; + } + public sealed record EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional); + public static class Module { + public static System.Collections.Generic.List Declarations = new(); + public static void RegisterEnvironment(EnvironmentDeclaration value) => Declarations.Add(value); + } + } + """; + + private static (Compilation Compilation, GeneratorDriverRunResult Result) Generate( + string declaration + ) + { + var references = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!) + .Split(Path.PathSeparator) + .Select(path => MetadataReference.CreateFromFile(path)); + var parse = new CSharpParseOptions(LanguageVersion.Preview); + var compilation = CSharpCompilation.Create( + "EnvironmentFixture" + Guid.NewGuid().ToString("N"), + [CSharpSyntaxTree.ParseText(Host + declaration, parse)], + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) + ); + GeneratorDriver driver = CSharpGeneratorDriver.Create( + [new EnvironmentGenerator().AsSourceGenerator()], + parseOptions: parse + ); + driver = driver.RunGeneratorsAndUpdateCompilation(compilation, out var output, out _); + return (output, driver.GetRunResult()); + } + + [Fact] + public static void NamedAccessorsKeepCheckedReadsAndRegisterCanonicalConstraints() + { + var (compilation, result) = Generate( + """ + [SpacetimeDB.Env] public struct Declarations { + public string REQUIRED; + public string? OPTIONAL; + [SpacetimeDB.EnvValues("prod", "dev")] public string MODE; + [SpacetimeDB.EnvValues("reserved")] public string Get; + public string @class; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.REQUIRED == env.REQUIRED || env.OPTIONAL != null || env.MODE != "prod" || + env.Get("Get") != "reserved" || env.@class != "keyword") throw new System.Exception("bad accessor"); + try { env.Get("UNKNOWN"); throw new System.Exception("unchecked generic read"); } + catch (System.InvalidOperationException) {} + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 5 || declarations[0].Optional || !declarations[1].Optional || + declarations[0].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.AnyString || + declarations[2].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.OneOf { Value.Count: 2 } || + declarations[3].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.Literal { Value: "reserved" }) + throw new System.Exception("bad metadata"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty("Get")); + } + + [Theory] + [InlineData("public int BAD;")] + [InlineData("public static string BAD;")] + [InlineData("[SpacetimeDB.EnvValues()] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(\"x\", \"x\")] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(null)] public string BAD;")] + public static void InvalidDeclarationsAreCompileErrors(string field) + { + var (_, result) = Generate("[SpacetimeDB.Env] public struct Declarations {" + field + "}"); + Assert.Contains( + result.Diagnostics, + diagnostic => + diagnostic.Id == "STDBENV001" && diagnostic.Severity == DiagnosticSeverity.Error + ); + } + + [Fact] + public static void EmptySchemaRetainsOnlyGenericAccess() + { + var (compilation, result) = Generate(""); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => diagnostic.Severity == DiagnosticSeverity.Error + ); + Assert.Contains( + "public string? Get(string key)", + result.GeneratedTrees.Single().ToString() + ); + } +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 4133819ef9c..5ecccedb31b 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -88,9 +88,13 @@ public async Task RunAndCheckGenerators( params IIncrementalGenerator[] generators ) => SampleCompilation.AddSyntaxTrees( - (await Task.WhenAll(generators.Select(RunAndCheckGenerator))).SelectMany(output => - output - ) + (await Task.WhenAll(generators.Select(RunAndCheckGenerator))) + .SelectMany(output => output) + .Concat( + generators.Any(generator => generator is SpacetimeDB.Codegen.Module) + ? RunGeneratorAndGetResult(new EnvironmentGenerator()).GeneratedTrees + : [] + ) ); } @@ -333,6 +337,7 @@ public static void @params(ProcedureContext ctx) [ new SpacetimeDB.Codegen.Type().AsSourceGenerator(), new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + new EnvironmentGenerator().AsSourceGenerator(), ], driverOptions: new( disabledOutputs: IncrementalGeneratorOutputKind.None, diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 2eb70c4a352..e81f122020d 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,7 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -732,6 +732,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -810,6 +811,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -850,6 +852,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -859,6 +863,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -893,7 +899,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -907,7 +913,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index d05060e0bd3..ad729dadac1 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,7 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -134,6 +134,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -212,6 +213,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -252,6 +254,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -261,6 +265,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -276,7 +282,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -290,7 +296,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 7f196c87a30..95bbf9c0516 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,7 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -576,6 +576,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -654,6 +655,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -694,6 +696,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -703,6 +707,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -727,7 +733,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -741,7 +747,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs new file mode 100644 index 00000000000..e049abb5fda --- /dev/null +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -0,0 +1,158 @@ +namespace SpacetimeDB.Codegen; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +/// Compile-time declarations and read-only accessors, never live values. +[Generator] +public sealed class EnvironmentGenerator : IIncrementalGenerator +{ + private static readonly DiagnosticDescriptor InvalidDeclaration = + new( + "STDBENV001", + "Invalid environment declaration", + "{0}", + "SpacetimeDB", + DiagnosticSeverity.Error, + isEnabledByDefault: true + ); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var declarations = context + .SyntaxProvider.ForAttributeWithMetadataName( + "SpacetimeDB.EnvAttribute", + (_, _) => true, + (ctx, _) => (INamedTypeSymbol)ctx.TargetSymbol + ) + .Collect(); + context.RegisterSourceOutput(declarations, Generate); + } + + private static void Generate( + SourceProductionContext context, + ImmutableArray types + ) + { + void Report(ISymbol symbol, string message) => + context.ReportDiagnostic( + Diagnostic.Create(InvalidDeclaration, symbol.Locations.FirstOrDefault(), message) + ); + if (types.Length > 1) + { + foreach (var type in types) + Report(type, "A module may have only one [SpacetimeDB.Env] declaration struct."); + } + var fields = types + .SelectMany(type => type.GetMembers().OfType()) + .Where(field => !field.IsImplicitlyDeclared) + .ToArray(); + if (fields.Length > 256 && types.Length != 0) + Report(types[0], "An environment schema may declare at most 256 variables."); + + var keys = new HashSet(StringComparer.Ordinal); + var properties = new List(); + var registrations = new List(); + foreach (var field in fields) + { + var name = field.Name; + if (field.IsStatic || field.Type.SpecialType != SpecialType.System_String) + { + Report( + field, + "Environment fields must be instance string or nullable string declarations." + ); + continue; + } + if ( + !Regex.IsMatch(name, "^[A-Za-z_][A-Za-z0-9_]*$") + || Encoding.UTF8.GetByteCount(name) > 256 + || !keys.Add(name) + ) + { + Report( + field, + "Environment names must be unique POSIX identifiers of at most 256 UTF-8 bytes." + ); + continue; + } + var optional = field.NullableAnnotation == NullableAnnotation.Annotated; + var attr = field + .GetAttributes() + .FirstOrDefault(a => + a.AttributeClass?.ToDisplayString() == "SpacetimeDB.EnvValuesAttribute" + ); + var constraint = + "new global::SpacetimeDB.Internal.EnvironmentConstraint.AnyString(default)"; + if (attr is not null) + { + var values = attr.ConstructorArguments.FirstOrDefault(); + if ( + values.Kind != TypedConstantKind.Array + || values.IsNull + || values.Values.Length == 0 + || values.Values.Any(value => + value.Value is not string text || Encoding.UTF8.GetByteCount(text) > 8192 + ) + ) + { + Report( + field, + "EnvValues requires a nonempty list of string literals of at most 8192 UTF-8 bytes each." + ); + continue; + } + var strings = values.Values.Select(value => (string)value.Value!).ToArray(); + if (strings.Distinct(StringComparer.Ordinal).Count() != strings.Length) + { + Report(field, "EnvValues must not repeat an allowed literal."); + continue; + } + constraint = + strings.Length == 1 + ? $"new global::SpacetimeDB.Internal.EnvironmentConstraint.Literal({Literal(strings[0])})" + : $"new global::SpacetimeDB.Internal.EnvironmentConstraint.OneOf(new global::System.Collections.Generic.List {{ {string.Join(", ", strings.Select(Literal))} }})"; + } + registrations.Add( + $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" + ); + // Preserve the checked generic method, including a key literally + // named Get. Keywords are escaped without renaming stored keys. + if (name is "Get" or "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString") + continue; + var read = $"Get({Literal(name)})"; + if (!optional) + read += + " ?? throw new global::System.InvalidOperationException(\"Required environment value is absent\")"; + properties.Add($"public string{(optional ? "?" : "")} @{name} => {read};"); + } + context.AddSource( + "Environment.g.cs", + $$""" + // + #nullable enable + #pragma warning disable CS0436 + namespace SpacetimeDB { + public readonly struct ModuleEnvironment { + public string? Get(string key) => default(global::SpacetimeDB.DatabaseEnvironment).Get(key); + {{string.Join("\n", properties)}} + } + internal static class EnvironmentRegistration { + [global::System.Runtime.CompilerServices.ModuleInitializer] + internal static void Register() { + {{string.Join("\n", registrations)}} + } + } + } + """ + ); + } + + private static string Literal(string value) => SymbolDisplay.FormatLiteral(value, quote: true); +} diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index e3f146ad3f2..29fc16d2cfc 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2557,7 +2557,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2629,6 +2629,7 @@ public Uuid NewUuidV7() } public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext(Identity identity, ConnectionId? connectionId, Random random, Timestamp time) @@ -2699,6 +2700,7 @@ public Uuid NewUuidV7() } public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -2736,6 +2738,7 @@ public Uuid NewUuidV7() } public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2743,6 +2746,7 @@ internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal HandlerTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2756,7 +2760,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2768,7 +2772,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { - public global::SpacetimeDB.DatabaseEnvironment Env => default; + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..a97596d4ec9 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -20,3 +20,30 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. + +### Declared environment + +A module may declare one `[SpacetimeDB.Env]` struct. `string` is required and +`string?` is optional. An optional `EnvValues` attribute restricts the allowed +strings; one value is a literal constraint. Values are supplied on publish, never +in module source: + +```csharp +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_URL; + [SpacetimeDB.EnvValues("prod", "dev")] + public string MODE; + public string? LOG_LEVEL; +} +``` + +Context access is read-only: `ctx.Env.MODE` returns `string`, while +`ctx.Env.LOG_LEVEL` returns `string?`. `ctx.Env.Get("MODE")` uses the same checked +host read. A key named `Get` keeps generic access rather than replacing the method; +C# keywords are escaped, for example `ctx.Env.@class`. Empty or absent declarations +allow no environment keys. Undeclared reads and reads from host-dispatched +submodules fail at runtime. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 3865f925a2a..afcfcc0688e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -1,5 +1,16 @@ namespace SpacetimeDB { + /// Declares the complete environment schema for this module. + [AttributeUsage(AttributeTargets.Struct)] + public sealed class EnvAttribute : Attribute { } + + /// Restricts one declared string to these exact permitted values. + [AttributeUsage(AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : Attribute + { + public string[] Values { get; } = values; + } + namespace Internal { [Flags] diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs new file mode 100644 index 00000000000..f4226020799 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs @@ -0,0 +1,11 @@ +// Canonical module-definition metadata; declaration constraints contain no runtime values. +#nullable enable +namespace SpacetimeDB.Internal; + +[SpacetimeDB.Type] +public partial record EnvironmentConstraint + : SpacetimeDB.TaggedEnum<( + SpacetimeDB.Unit AnyString, + string Literal, + System.Collections.Generic.List OneOf + )>; diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs new file mode 100644 index 00000000000..554d77ba55a --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace SpacetimeDB.Internal; + +[SpacetimeDB.Type] +[System.Runtime.Serialization.DataContract] +public sealed partial class EnvironmentDeclaration +{ + [System.Runtime.Serialization.DataMember(Name = "name")] + public string Name; + + [System.Runtime.Serialization.DataMember(Name = "constraint")] + public EnvironmentConstraint Constraint; + + [System.Runtime.Serialization.DataMember(Name = "optional")] + public bool Optional; + + public EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional) + { + this.Name = Name; + this.Constraint = Constraint; + this.Optional = Optional; + } + + public EnvironmentDeclaration() + { + Name = ""; + Constraint = new EnvironmentConstraint.AnyString(default); + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 61212c98e89..52f750e3903 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -23,6 +23,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, System.Collections.Generic.List ViewPrimaryKeys, - System.Collections.Generic.List Submodules + System.Collections.Generic.List Submodules, + System.Collections.Generic.List Environment )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index f498e6ed8ea..cc3e8cf624b 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -110,13 +110,14 @@ internal static partial class FFI ; const string StdbNamespace10_7 = -#if EXPERIMENTAL_WASM_AOT +#if EXPERIMENTAL_WASM_AOT || NET10_0_OR_GREATER "spacetime_10.7" #else "bindings" #endif ; + [WasmImportLinkage] [LibraryImport(StdbNamespace10_7)] public static unsafe partial CheckedStatus env_get( byte* key, diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 177fddd785a..035291a2f0d 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -20,6 +20,7 @@ partial class RawModuleDefV10 private readonly List httpRouteDefs = []; private readonly List viewDefs = []; private readonly List viewPrimaryKeyDefs = []; + private readonly List environment = []; private readonly List rowLevelSecurityDefs = []; private readonly Dictionary> defaultValuesByTable = new(StringComparer.Ordinal); @@ -86,6 +87,8 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); + internal void RegisterEnvironment(EnvironmentDeclaration declaration) => environment.Add(declaration); + internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => viewPrimaryKeyDefs.Add(new RawViewPrimaryKeyDefV10(viewSourceName, [.. columns])); @@ -162,6 +165,7 @@ internal RawModuleDefV10 BuildModuleDefinition() var sections = new List { new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Environment(environment), }; if (typeDefs.Count > 0) @@ -427,6 +431,8 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } + public static void RegisterEnvironment(EnvironmentDeclaration declaration) => moduleDef.RegisterEnvironment(declaration); + public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => moduleDef.RegisterViewPrimaryKey(viewSourceName, columns); diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index e3b4bd0e942..95c2f1cfdd5 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -65,6 +65,9 @@ + + + diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs new file mode 100644 index 00000000000..8377037082e --- /dev/null +++ b/crates/bindings-macro/src/environment.rs @@ -0,0 +1,252 @@ +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::ext::IdentExt as _; +use syn::punctuated::Punctuated; +use syn::{Fields, GenericArgument, ItemStruct, LitStr, PathArguments, Token, Type}; + +pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result { + if !args.is_empty() { + return Err(syn::Error::new_spanned(args, "env does not accept arguments")); + } + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment declarations cannot be generic", + )); + } + if matches!(item.fields, Fields::Unnamed(_)) { + return Err(syn::Error::new_spanned( + &item.fields, + "environment declarations require named fields", + )); + } + let mut declarations = Vec::new(); + let mut signatures = Vec::new(); + let mut methods = Vec::new(); + for field in &mut item.fields { + let ident = field.ident.as_ref().expect("named fields checked"); + let name = ident.unraw().to_string(); + if name.len() > 256 || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(syn::Error::new_spanned( + ident, + "environment keys must be POSIX names of at most 256 bytes", + )); + } + let optional = optional_string(&field.ty)?; + let mut values: Option> = None; + for attr in field.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("values") || values.is_some() { + return Err(meta.error("expected one values(\"literal\", ...) constraint")); + } + let content; + syn::parenthesized!(content in meta.input); + let parsed = Punctuated::::parse_terminated(&content)?; + if parsed.is_empty() { + return Err(meta.error("environment string unions must not be empty")); + } + for value in &parsed { + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + } + values = Some(parsed.into_iter().collect()); + Ok(()) + })?; + } + field.attrs.retain(|attr| !attr.path().is_ident("env")); + let constraint = match values.as_deref() { + None => quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::AnyString), + Some([value]) => { + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())) + } + Some(values) => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf( + ::std::vec![#(#values.into()),*] + ) + ), + }; + declarations.push( + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentDeclaration { + name: #name.into(), constraint: #constraint, optional: #optional, + }), + ); + if name == "get" { + continue; + } + let (return_type, body) = if optional { + ( + quote!(::std::option::Option<::std::string::String>), + quote!(::spacetimedb::Environment::get(self, #name)), + ) + } else { + ( + quote!(::std::string::String), + quote!(::spacetimedb::Environment::get(self, #name).expect(concat!("required environment key is missing: ", #name))), + ) + }; + signatures.push(quote! { + #[doc = concat!("Read the declared environment key `", #name, "` through the checked host ABI.")] + fn #ident(&self) -> #return_type; + }); + methods.push(quote!(fn #ident(&self) -> #return_type { #body })); + } + let vis = &item.vis; + let access = format_ident!("{}Access", item.ident.unraw()); + let symbol = format!("__preinit__20_register_environment_{}", item.ident.unraw()); + Ok(quote! { + #[allow(non_snake_case)] + #item + + /// Named read-only accessors for this module's environment declaration. + #[allow(non_snake_case)] + #vis trait #access { + #(#signatures)* + } + #[allow(non_snake_case)] + impl #access for ::spacetimedb::Environment { + #(#methods)* + } + const _: () = { + #[unsafe(export_name = #symbol)] + extern "C" fn __register_environment() { + ::spacetimedb::rt::register_environment(|| ::std::vec![#(#declarations),*]); + } + }; + }) +} + +fn optional_string(ty: &Type) -> syn::Result { + let Type::Path(path) = ty else { + return Err(syn::Error::new_spanned(ty, "expected String or Option")); + }; + if path.qself.is_none() { + let segments: Vec<_> = path + .path + .segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect(); + let names: Vec<_> = segments.iter().map(String::as_str).collect(); + if matches!(names.as_slice(), ["String"] | ["std" | "alloc", "string", "String"]) + && path + .path + .segments + .iter() + .all(|segment| matches!(segment.arguments, PathArguments::None)) + { + return Ok(false); + } + if matches!(names.as_slice(), ["Option"] | ["std" | "core", "option", "Option"]) + && path + .path + .segments + .iter() + .rev() + .skip(1) + .all(|segment| matches!(segment.arguments, PathArguments::None)) + && let PathArguments::AngleBracketed(arguments) = &path.path.segments.last().unwrap().arguments + && let [GenericArgument::Type(inner)] = arguments.args.iter().collect::>().as_slice() + && !optional_string(inner)? + { + return Ok(true); + } + } + Err(syn::Error::new_spanned( + ty, + "expected String or Option; aliases and other types are not env constraints", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_string_nested_optional_empty_union_and_invalid_names() { + for item in [ + quote!( + struct Env { + VALUE: bool, + } + ), + quote!( + struct Env { + VALUE: Option>, + } + ), + quote!( + struct Env { + #[env(values())] + VALUE: String, + } + ), + quote!( + struct Env { + #[env(values("a"), values("b"))] + VALUE: String, + } + ), + quote!( + struct Env { + 雪: String, + } + ), + quote!( + struct Env(String); + ), + quote!( + struct Env { + VALUE: T, + } + ), + ] { + assert!(expand(TokenStream::new(), syn::parse2(item).unwrap()).is_err()); + } + } + + #[test] + fn accepts_strings_optionals_constraints_and_generic_accessor_collision() { + let output = expand( + TokenStream::new(), + syn::parse_quote! { + pub struct Env { + VALUE: String, + #[env(values("false", "true"))] ENABLED: std::string::String, + #[env(values(""))] OPTIONAL: Option, + get: Option, + r#type: String, + } + }, + ) + .unwrap(); + let parsed: syn::File = syn::parse2(output).unwrap(); + let trait_item = parsed + .items + .iter() + .find_map(|item| match item { + syn::Item::Trait(item) => Some(item), + _ => None, + }) + .unwrap(); + let names: Vec<_> = trait_item + .items + .iter() + .filter_map(|item| match item { + syn::TraitItem::Fn(method) => Some(method.sig.ident.unraw().to_string()), + _ => None, + }) + .collect(); + assert_eq!(names, ["VALUE", "ENABLED", "OPTIONAL", "type"]); + assert!(expand( + TokenStream::new(), + syn::parse_quote!( + pub struct Empty {} + ) + ) + .is_ok()); + } +} diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index a3efdd3d2f4..02aaacc4028 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -8,6 +8,13 @@ // // (private documentation for the macro authors is totally fine here and you SHOULD write that!) +mod environment; + +#[proc_macro_attribute] +pub fn env(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::expand(args.into(), syn::parse(item)?)) +} + mod http; mod procedure; diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 48e7cfd1535..a006b8489fd 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -209,3 +209,29 @@ To run the tests, do: ```sh pnpm build && pnpm test ``` + +### Declared environment + +Pass the complete environment schema to `schema`. Values are supplied at publish +time, not embedded in the module: + +```ts +const db = schema(tables, { + env: { + API_URL: t.string(), + MODE: t.enum('Mode', ['prod', 'dev']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, +}); +``` + +`ctx.env.MODE` has type `'prod' | 'dev'`; `ctx.env.LOG_LEVEL` also permits +`undefined`. Simple enum cases mean allowed strings only in this declaration; +ordinary enum values elsewhere retain their tagged representation. Payload enums +are rejected. The checked `ctx.env.get('LOG_LEVEL')` returns `null` for an omitted +optional value. A declared key named `get` remains accessible through the generic +method. Omitted or empty schemas allow no keys. Undeclared reads and reads entered +by the host in a submodule fail at runtime; ordinary helpers retain their caller's +scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 3cee51f03d5..642c0431285 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -393,9 +393,38 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get Submodules() { return __t.array(RawSubmoduleV10); }, + get Environment() { + return __t.array(EnvironmentDeclaration); + }, }); export type RawModuleDefV10Section = __Infer; +export const EnvironmentConstraint = __t.enum('EnvironmentConstraint', { + get AnyString() { + return __t.unit(); + }, + get Literal() { + return __t.string(); + }, + get OneOf() { + return __t.array(__t.string()); + }, +}); +export type EnvironmentConstraint = __Infer; + +export const EnvironmentDeclaration = __t.object('EnvironmentDeclaration', { + get name() { + return __t.string(); + }, + get constraint() { + return EnvironmentConstraint; + }, + get optional() { + return __t.bool(); + }, +}); +export type EnvironmentDeclaration = __Infer; + export const RawModuleDefV8 = __t.object('RawModuleDefV8', { get typespace() { return Typespace; diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts index 3798cfb43d4..d065d5bb0aa 100644 --- a/crates/bindings-typescript/src/lib/environment.ts +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -1,5 +1,56 @@ -/** Read-only database environment access. Missing keys return null; empty values return "". */ -export interface Environment { - /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ - get(key: string): string | null; +import type { + OptionBuilder, + StringBuilder, + TypeBuilder, + t, +} from './type_builders'; + +/** Only strings and simple, payload-free enums constrain environment strings. */ +export type EnvironmentString = + | StringBuilder + | (TypeBuilder & { + readonly variants: Record>; + }); +export type EnvironmentSchema = Record< + string, + EnvironmentString | OptionBuilder +>; + +export type EnvironmentValue = + T extends OptionBuilder + ? EnvironmentValue | undefined + : T extends { readonly variants: infer Variants } + ? keyof Variants & string + : string; + +/** Values are read from the host on each access. Undeclared names are errors. */ +export type Environment< + Declarations extends EnvironmentSchema | undefined = undefined, +> = Declarations extends EnvironmentSchema + ? { + readonly [Key in keyof Declarations as Key extends 'get' + ? never + : Key]: EnvironmentValue; + } & { + get( + key: Key & + (string extends Key + ? unknown + : Key extends keyof Declarations + ? unknown + : never) + ): Key extends keyof Declarations + ? + | Exclude, undefined> + | (undefined extends EnvironmentValue + ? null + : never) + : string | null; + } + : { get(key: string): string | null }; + +export type EnvironmentFor = Schema extends { + env: infer Declarations extends EnvironmentSchema; } + ? Environment + : Environment; diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index eaa8de94b55..f2d470152ec 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,4 +1,4 @@ -import type { Environment } from './environment'; +import type { EnvironmentFor } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -116,7 +116,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; - env: Environment; + env: EnvironmentFor; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index eb3821f6606..67b263d8397 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -205,6 +205,7 @@ export class ModuleContext { entries: [], }, submodules: [], + environment: [], }; get moduleDef(): ModuleDef { @@ -275,6 +276,7 @@ export class ModuleContext { value: module.submodules, } ); + push({ tag: 'Environment', value: module.environment }); return { sections }; } diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts index 349e8ffed8c..203c608e08c 100644 --- a/crates/bindings-typescript/src/server/environment.ts +++ b/crates/bindings-typescript/src/server/environment.ts @@ -1,6 +1,86 @@ import { env_get } from 'spacetime:sys@2.3'; -import type { Environment } from '../lib/environment'; +import type { Environment, EnvironmentSchema } from '../lib/environment'; +import type { + EnvironmentDeclaration, + EnvironmentConstraint, + AlgebraicType, +} from '../lib/autogen/types'; +import { OptionBuilder, StringBuilder } from '../lib/type_builders'; /** Values are not cached: transaction and procedure reads retain host semantics. */ -export const environment: Environment = Object.freeze({ get: env_get }); -export type { Environment } from '../lib/environment'; +export const environment: Environment = new Proxy( + Object.freeze( + Object.assign(Object.create(null), { get: (key: string) => env_get(key) }) + ), + { + get(target, key) { + if (key === 'get') return target.get; + if (typeof key !== 'string') return undefined; + // The host rejects undeclared keys and missing required values. Optional + // named access uses undefined; the generic ABI accessor retains null. + return env_get(key) ?? undefined; + }, + } +); +export type { + Environment, + EnvironmentFor, + EnvironmentSchema, +} from '../lib/environment'; + +/** Produce metadata only. No environment value is embedded in the artifact. */ +export function environmentDeclarations( + schema: EnvironmentSchema +): EnvironmentDeclaration[] { + const entries = Object.entries(schema); + if (entries.length > 256) + throw new TypeError('Too many environment declarations'); + const bytes = new TextEncoder(); + return entries.map(([name, definition]) => { + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || + bytes.encode(name).length > 256 + ) { + throw new TypeError('Invalid environment declaration name'); + } + const optional = definition instanceof OptionBuilder; + const inner = optional ? definition.value : definition; + let constraint: EnvironmentConstraint; + if (inner instanceof StringBuilder) { + constraint = { tag: 'AnyString' }; + } else { + const type: AlgebraicType = inner?.algebraicType; + if (type?.tag !== 'Sum' || !('variants' in inner)) { + throw new TypeError( + `Environment '${name}' must be a string or simple enum` + ); + } + const values = type.value.variants.map(variant => { + if ( + variant.algebraicType.tag !== 'Product' || + variant.algebraicType.value.elements.length !== 0 + ) { + throw new TypeError( + `Environment '${name}' cannot use an enum payload` + ); + } + if (typeof variant.name !== 'string') + throw new TypeError( + `Environment '${name}' enum cases must have names` + ); + if (bytes.encode(variant.name).length > 8192) + throw new TypeError(`Environment '${name}' literal is too long`); + return variant.name; + }); + if (values.length === 0 || new Set(values).size !== values.length) + throw new TypeError( + `Environment '${name}' needs a nonempty literal union` + ); + constraint = + values.length === 1 + ? { tag: 'Literal', value: values[0]! } + : { tag: 'OneOf', value: values }; + } + return { name, constraint, optional }; + }); +} diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index 0c4d9469eee..b8823577f4e 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,4 +1,4 @@ -import type { Environment } from '../lib/environment'; +import type { EnvironmentFor } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -220,7 +220,7 @@ export type HandlerAliasViews = : {}; export interface HandlerContext { - readonly env: Environment; + readonly env: EnvironmentFor; readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index f4d57416e69..f5b8e2aa1e2 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,4 +1,4 @@ -import { environment, type Environment } from './environment'; +import { environment, type EnvironmentFor } from './environment'; import { AlgebraicType, ProductType, @@ -109,7 +109,7 @@ export type ProcedureAliasViews = : {}; export interface ProcedureCtx { - readonly env: Environment; + readonly env: EnvironmentFor; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -228,7 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; - readonly env = environment; + readonly env = environment as EnvironmentFor; #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index 1fac18ac95b..70c6eb81401 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,4 +1,4 @@ -import { environment } from './environment'; +import { environment, type EnvironmentFor } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; @@ -247,7 +247,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; - readonly env = environment; + readonly env = environment as EnvironmentFor; as: AliasViews; constructor( @@ -777,7 +777,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { - readonly env = environment; + readonly env = environment as EnvironmentFor; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index c399a66d31e..95688b7f3cd 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -1,3 +1,4 @@ +import { environmentDeclarations, type EnvironmentSchema } from './environment'; import { moduleHooks, type ModuleDefaultExport } from 'spacetime:sys@2.0'; import { CaseConversionPolicy, @@ -322,6 +323,9 @@ export class Schema implements ModuleDefaultExport { const rawDef = this.buildRawModuleDefV10(exports, { ignoreNonModuleExports: true, }); + if (this.#ctx.moduleDef.environment.length !== 0) { + throw new TypeError('Submodules cannot declare environment variables'); + } this.#ctx.resolveHttpRoutes(); return { rawDef, @@ -747,7 +751,11 @@ export type InferSchema> = /** * Module-level settings that can be passed to `schema()`. */ -export interface ModuleSettings { +export interface ModuleSettings< + E extends EnvironmentSchema = EnvironmentSchema, +> { + /** Declared strings installed only through publishing; omitted means empty. */ + env?: E; /** * The case conversion policy for this module. * Defaults to `SnakeCase` if not specified. @@ -825,11 +833,17 @@ function registerModuleExports( } } -export function schema>( +export function schema< + const H extends Record, + const E extends EnvironmentSchema = {}, +>( entries: H, - moduleSettings?: ModuleSettings -): Schema> { - const ctx = new SchemaInner>(ctx => { + moduleSettings?: ModuleSettings +): Schema & { env: E }> { + const ctx = new SchemaInner & { env: E }>(ctx => { + ctx.moduleDef.environment = environmentDeclarations( + moduleSettings?.env ?? {} + ); // Apply module settings. if (moduleSettings?.CASE_CONVERSION_POLICY != null) { ctx.setCaseConversionPolicy(moduleSettings.CASE_CONVERSION_POLICY); @@ -884,7 +898,10 @@ export function schema>( }); } } - return { tables: tableSchemas } as SchemaDefForEntries; + return { + tables: tableSchemas, + env: moduleSettings?.env ?? {}, + } as SchemaDefForEntries & { env: E }; }); return new Schema(ctx); diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 528c8319063..f089de2f0c1 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,4 +1,4 @@ -import type { Environment } from '../lib/environment'; +import type { EnvironmentFor } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -82,13 +82,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; - env: Environment; + env: EnvironmentFor; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; - env: Environment; + env: EnvironmentFor; from: QueryBuilder; }>; diff --git a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts index 47cbf9d7039..257321cfe8e 100644 --- a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts +++ b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts @@ -91,3 +91,7 @@ export const procedure_http_request = ( export const procedure_start_mut_tx = (): bigint => 0n; export const procedure_commit_mut_tx = (): void => {}; export const procedure_abort_mut_tx = (): void => {}; + +export const env_get = (_key: string): string | null => { + throw new Error('mock environment read is not configured'); +}; diff --git a/crates/bindings-typescript/tests/environment.test.ts b/crates/bindings-typescript/tests/environment.test.ts new file mode 100644 index 00000000000..b78713d49e9 --- /dev/null +++ b/crates/bindings-typescript/tests/environment.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, expectTypeOf, it, vi } from 'vitest'; +import { schema } from '../src/server/schema'; +import { t } from '../src/lib/type_builders'; +import { + environment, + environmentDeclarations, +} from '../src/server/environment'; +import type { EnvironmentSchema } from '../src/lib/environment'; +import { env_get } from 'spacetime:sys@2.3'; + +vi.mock('spacetime:sys@2.3', async importOriginal => ({ + ...(await importOriginal()), + env_get: vi.fn(), +})); + +const declarations = { + FOOBAR: t.string(), + ENABLE_EMAIL: t.enum('EnableEmail', ['true', 'false']), + LOG_LEVEL: t.enum('LogLevel', ['debug', 'info', 'error']).optional(), + DEPLOYMENT_KIND: t.enum('DeploymentKind', ['production']), + get: t.string().optional(), +}; + +describe('declared database environment', () => { + it('emits canonical constraint metadata and an explicit empty section', () => { + expect(environmentDeclarations(declarations)).toEqual([ + { name: 'FOOBAR', constraint: { tag: 'AnyString' }, optional: false }, + { + name: 'ENABLE_EMAIL', + constraint: { tag: 'OneOf', value: ['true', 'false'] }, + optional: false, + }, + { + name: 'LOG_LEVEL', + constraint: { tag: 'OneOf', value: ['debug', 'info', 'error'] }, + optional: true, + }, + { + name: 'DEPLOYMENT_KIND', + constraint: { tag: 'Literal', value: 'production' }, + optional: false, + }, + { name: 'get', constraint: { tag: 'AnyString' }, optional: true }, + ]); + const defined = schema({}, { env: declarations }); + const section = defined + .buildRawModuleDefV10({}) + .sections.find(section => section.tag === 'Environment'); + expect(section).toEqual({ + tag: 'Environment', + value: environmentDeclarations(declarations), + }); + expect(schema({}).buildRawModuleDefV10({}).sections).toContainEqual({ + tag: 'Environment', + value: [], + }); + // Enum values outside env retain their existing tagged-sum interpretation. + expect(Reflect.get(declarations.ENABLE_EMAIL, 'true')).toEqual({ + tag: 'true', + }); + }); + + it('rejects unsupported constraints and submodule declarations before upload', () => { + const invalid = (value: unknown) => () => + environmentDeclarations(value as EnvironmentSchema); + expect(invalid({ BAD: t.u32() })).toThrow('string or simple enum'); + expect(invalid({ BAD: t.enum('Payload', { value: t.string() }) })).toThrow( + 'enum payload' + ); + expect(invalid({ BAD: t.enum('Empty', []) })).toThrow( + 'nonempty literal union' + ); + expect(invalid({ BAD: t.string().optional().optional() })).toThrow(); + expect(invalid({ 'bad-name': t.string() })).toThrow('name'); + expect(invalid({ BAD: t.enum('Long', ['x'.repeat(8193)]) })).toThrow( + 'too long' + ); + expect( + invalid( + Object.fromEntries( + Array.from({ length: 257 }, (_, i) => [`K${i}`, t.string()]) + ) + ) + ).toThrow('Too many'); + expect(() => + schema({ child: { default: schema({}, { env: declarations }) } }) + ).toThrow('Submodules'); + expect(() => + schema({ child: { default: schema({}, { env: {} }) } }) + ).not.toThrow(); + }); + + it('preserves generic get, optional absence, host errors and uncached named reads', () => { + const get = vi.mocked(env_get); + get.mockReset(); + get + .mockReturnValueOnce('first') + .mockReturnValueOnce('') + .mockReturnValueOnce(null) + .mockReturnValueOnce('declared get'); + const named = environment as typeof environment & { + readonly FOOBAR: string; + readonly LOG_LEVEL: string | undefined; + }; + expect(named.FOOBAR).toBe('first'); + expect(named.FOOBAR).toBe(''); + expect(named.LOG_LEVEL).toBeUndefined(); + expect(named.get('get')).toBe('declared get'); + get.mockReturnValueOnce(null); + expect(named.get('LOG_LEVEL')).toBeNull(); + get.mockImplementationOnce(() => { + throw new Error('undeclared host key'); + }); + expect(() => named.get('UNDECLARED')).toThrow('undeclared host key'); + expect(get.mock.calls.map(([name]) => name)).toEqual([ + 'FOOBAR', + 'FOOBAR', + 'LOG_LEVEL', + 'get', + 'LOG_LEVEL', + 'UNDECLARED', + ]); + }); +}); + +// These declarations are compiled by the focused typecheck. Callback bodies +// need not execute to assert their schema-specific context types. +const typed = schema({}, { env: declarations }); +typed.reducer(ctx => { + expectTypeOf(ctx.env.FOOBAR).toEqualTypeOf(); + expectTypeOf(ctx.env.ENABLE_EMAIL).toEqualTypeOf<'true' | 'false'>(); + expectTypeOf(ctx.env.LOG_LEVEL).toEqualTypeOf< + 'debug' | 'info' | 'error' | undefined + >(); + expectTypeOf(ctx.env.DEPLOYMENT_KIND).toEqualTypeOf<'production'>(); + expectTypeOf(ctx.env.get('FOOBAR')).toEqualTypeOf(); + expectTypeOf(ctx.env.get('get')).toEqualTypeOf(); + // @ts-expect-error Undeclared literal names have no checked accessor. + ctx.env.get('UNDECLARED'); + // @ts-expect-error No named access to undeclared keys. + void ctx.env.UNKNOWN; + // @ts-expect-error Environment access is read-only. + ctx.env.FOOBAR = 'changed'; +}); +function rejectPayloadType() { + // @ts-expect-error Payload enums cannot declare environment strings. + schema({}, { env: { BAD: t.enum('Payload', { value: t.string() }) } }); +} +void rejectPayloadType; diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index 7cfc858d47c..e5503037be6 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, + { find: 'spacetime:sys@2.3', replacement: sysMock }, ], }, test: { diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index d1548bca052..1589eb54ad5 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -919,6 +919,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Declare the complete publish-time environment schema and generate named accessors. +/// +/// Fields must be `String` or `Option`; `#[env(values("a", "b"))]` +/// constrains exact strings. Values are supplied on every publish, never in metadata. +/// The macro generates an `EnvAccess` extension trait for a struct named `Env`. +/// Import that trait when the declaration lives in a different Rust module. +/// The name `get` is reserved for generic checked access. +/// +/// ```no_run +/// #[spacetimedb::env] +/// pub struct Env { +/// pub API_KEY: String, +/// #[env(values("debug", "info"))] +/// pub LOG_LEVEL: Option, +/// } +/// fn read(ctx: &spacetimedb::ReducerContext) { +/// let _: String = ctx.env.API_KEY(); +/// let _: Option = ctx.env.LOG_LEVEL(); +/// } +/// ``` +#[doc(inline)] +pub use spacetimedb_bindings_macro::env; + /// Read-only access to this database's environment store. /// /// Reads use the current transaction. In a procedure outside a transaction, @@ -930,8 +953,12 @@ pub struct Environment { } impl Environment { - /// Return None for a missing key and Some("") for a present empty value. + /// Return `None` for an absent declared optional key and `Some("")` for a present empty value. /// Keys must be POSIX environment names of at most 256 bytes. + /// + /// # Panics + /// + /// Panics if the key is undeclared, invalid, or inaccessible in the current host call. pub fn get(&self, key: &str) -> Option { rt::env_get(key) } diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index b09c998f5c3..bc5b0d8fae1 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,6 +917,14 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } +/// Register declarative ENV metadata without reading any environment values. +#[doc(hidden)] +pub fn register_environment(declarations: fn() -> Vec) { + register_describer(move |module| { + module.inner.add_environment(declarations()); + }); +} + /// A builder for a module. #[derive(Default)] pub struct ModuleBuilder { @@ -983,6 +991,7 @@ extern "C" fn __describe_module__(description: BytesSink) { } // Serialize the module to bsatn. + module.inner.ensure_environment(); let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs new file mode 100644 index 00000000000..5bffb7924ff --- /dev/null +++ b/crates/bindings/tests/environment.rs @@ -0,0 +1,4 @@ +#[test] +fn environment_declaration_accessors_compile_with_exact_types() { + trybuild::TestCases::new().pass("tests/pass/environment.rs"); +} diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs new file mode 100644 index 00000000000..87a53d23aea --- /dev/null +++ b/crates/bindings/tests/pass/environment.rs @@ -0,0 +1,24 @@ +#![deny(warnings)] + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: String, + #[env(values("false", "true"))] + pub FLAG: String, + #[env(values(""))] + pub OPTIONAL: Option, + pub get: Option, + pub r#type: String, +} + +fn reads(env: spacetimedb::Environment) { + let _: String = env.REQUIRED(); + let _: String = env.FLAG(); + let _: Option = env.OPTIONAL(); + let _: Option = env.get("get"); + let _: String = env.r#type(); +} + +fn main() { + let _ = reads as fn(spacetimedb::Environment); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9411e6f5e5e..96bb1034bfe 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -22,6 +22,7 @@ pub use tasks::build; pub fn get_subcommands() -> Vec { vec![ publish::cli(), + env::cli(), delete::cli(), logs::cli(), call::cli(), @@ -57,6 +58,7 @@ pub async fn exec_subcommand( "describe" => describe::exec(config, args).await, "dev" => dev::exec(config, args).await, "publish" => publish::exec(config, args).await, + "env" => env::exec(config, args).await, "delete" => delete::exec(config, args).await, "logs" => logs::exec(config, args).await, "sql" => sql::exec(config, args).await, diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..cce271510ee 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use clap::{ArgMatches, Command}; use path_clean::PathClean; @@ -183,6 +185,14 @@ impl SpacetimeConfig { let mut fields = self.additional_fields.clone(); if let Some(parent) = parent_fields { for (key, value) in parent { + if key == "env" + && let Some(child) = fields.get_mut(key) + { + let mut combined = value.clone(); + environment::overlay(&mut combined, child); + *child = combined; + continue; + } if fields.contains_key(key) { continue; } @@ -310,6 +320,9 @@ impl CommandSchemaBuilder { // Check that all the defined keys exist in clap for key in &self.keys { + if key.config_only { + continue; + } if !clap_arg_names.contains(key.clap_arg_name()) { return Err(CommandConfigError::InvalidClapReference { config_name: key.config_name().to_string(), @@ -343,6 +356,9 @@ impl CommandSchemaBuilder { let mut config_to_alias_map = HashMap::new(); for key in &self.keys { + if key.config_only { + continue; + } let config_name = key.config_name().to_string(); let clap_name = key.clap_arg_name().to_string(); @@ -398,6 +414,13 @@ impl CommandSchema { matches: &ArgMatches, config_name: &str, ) -> Result, CommandConfigError> { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return Ok(None); + } // Check clap with mapped name (if from_clap was used, use that name, otherwise use config name) let clap_name = self .config_to_clap @@ -428,6 +451,13 @@ impl CommandSchema { /// Check if a value was provided via CLI (not from config). /// Only returns true if the user explicitly provided the value, not if it came from a default. pub fn is_from_cli(&self, matches: &ArgMatches, config_name: &str) -> bool { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return false; + } // Check clap with mapped name let clap_name = self .config_to_clap @@ -567,6 +597,8 @@ impl CommandSchema { pub struct Key { /// The key name in the config file (e.g., "module-path") config_name: String, + /// This field has no value-bearing CLI argument (even if a same-named selector exists). + config_only: bool, /// The corresponding clap argument name (e.g., "project-path"), if different clap_name: Option, /// Alias for a clap argument, useful for example if we have to deprecate a clap @@ -585,6 +617,7 @@ impl Key { pub fn new(name: impl Into) -> Self { Self { config_name: name.into(), + config_only: false, clap_name: None, clap_alias: None, module_specific: false, @@ -593,6 +626,12 @@ impl Key { } } + /// Read this key exclusively from project configuration. + pub fn config_only(mut self) -> Self { + self.config_only = true; + self + } + /// Map this config key to a different clap argument name. When fetching values /// the key that is defined should be used. /// Example: Key::new("module-path").from_clap("project-path") @@ -813,8 +852,10 @@ impl SpacetimeConfig { let content = std::fs::read_to_string(path).with_context(|| format!("Failed to read config file: {}", path.display()))?; - let config: Self = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; + let config: Self = environment::decode_config(value) + .map_err(|_| anyhow::anyhow!("Invalid configuration structure in {}", path.display()))?; Ok(config) } @@ -911,8 +952,8 @@ fn load_json_value(path: &Path) -> anyhow::Result> { // comments and formatting since json5 crate doesn't support serialization. remove_source_config_from_text(path, &content); - let value: serde_json::Value = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; Ok(Some(value)) } @@ -1000,6 +1041,12 @@ fn overlay_json(base: &mut serde_json::Value, mut overlay: serde_json::Value, so base_obj.insert(key.clone(), other); } } + } else if key == "env" { + if let Some(base_env) = base_obj.get_mut(key) { + environment::overlay(base_env, value); + } else { + base_obj.insert(key.clone(), value_owned); + } } else { base_obj.insert(key.clone(), value_owned); } @@ -1068,7 +1115,7 @@ pub fn find_and_load_with_env_from(env: Option<&str>, start_dir: PathBuf) -> any } } - let config: SpacetimeConfig = serde_json::from_value(merged).context("Failed to deserialize merged config")?; + let config = environment::decode_config(merged)?; Ok(Some(LoadedConfig { config, diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs new file mode 100644 index 00000000000..ecaa3218684 --- /dev/null +++ b/crates/cli/src/spacetime_config/environment.rs @@ -0,0 +1,286 @@ +//! Preserve JSON numeric values before the JSON5 deserializer can round them. +//! This also permits existing comments, unquoted names and trailing commas. +use serde_json::Value; + +pub(super) fn parse(content: &str) -> anyhow::Result { + let bytes = content.as_bytes(); + let mut numbers = Vec::new(); + let mut replacements = Vec::new(); + let mut strings: Vec = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let start = i; + match bytes[i] { + b'\'' | b'"' => { + let quote = bytes[i]; + i += 1; + while i < bytes.len() { + if bytes[i] == b'\\' { + i = (i + 2).min(bytes.len()); + } else if bytes[i] == quote { + i += 1; + break; + } else { + i += 1; + } + } + } + b'/' if bytes.get(i + 1) == Some(&b'/') => { + i += 2; + while i < bytes.len() { + let ch = content[i..].chars().next().unwrap(); + if matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}') { + break; + } + i += ch.len_utf8(); + } + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + i = (i + 2).min(bytes.len()); + } + b'{' | b'}' | b'[' | b']' | b':' | b',' => i += 1, + _ if is_space(content[i..].chars().next().unwrap()) => i += content[i..].chars().next().unwrap().len_utf8(), + _ => { + while i < bytes.len() + && !is_space(content[i..].chars().next().unwrap()) + && !matches!(bytes[i], b'{' | b'}' | b'[' | b']' | b':' | b',' | b'/' | b'\'' | b'"') + { + i += content[i..].chars().next().unwrap().len_utf8(); + } + if i == start { + i += content[i..].chars().next().unwrap().len_utf8(); + } + let token = &content[start..i]; + if (token.starts_with(|c: char| c.is_ascii_digit() || matches!(c, '-' | '+' | '.')) + || matches!(token, "Infinity" | "NaN")) + && !content[i..].trim_start_matches(is_space).starts_with(':') + { + replacements.push((start, i)); + numbers.push(token); + continue; + } + } + } + if matches!(bytes[start], b'\'' | b'"') { + strings.push(json5::from_str(&content[start..i]).map_err(|_| anyhow::anyhow!("Invalid JSON5 string"))?); + } + } + // Check decoded strings as well: Unicode escapes must not manufacture an + // internal marker and cause a string to be interpreted as a number. + let mut prefix = "__spacetime_numeric_".to_owned(); + while content.contains(&prefix) || strings.iter().any(|s| s.contains(&prefix)) { + prefix.push('_'); + } + let mut text = String::with_capacity(content.len()); + let mut previous = 0; + for (index, (start, end)) in replacements.into_iter().enumerate() { + text.push_str(&content[previous..start]); + text.push_str(&format!("\"{prefix}{index}\"")); + previous = end; + } + text.push_str(&content[previous..]); + // Parser diagnostics may quote the source line, which can contain secrets. + let mut value: Value = json5::from_str(&text).map_err(|_| anyhow::anyhow!("Invalid JSON5 configuration"))?; + restore(&mut value, &prefix, &numbers, None)?; + Ok(value) +} + +/// Deserialize through JSON text so Serde's flattened-field buffer does not +/// receive visit_u128 from Value's deserializer. That buffer cannot represent +/// u128, while the arbitrary-precision JSON parser preserves its decimal token. +/// Diagnostics deliberately discard the original error, which may quote values. +pub(super) fn decode_config(value: Value) -> anyhow::Result { + let encoded = serde_json::to_vec(&value).map_err(|_| anyhow::anyhow!("Invalid configuration structure"))?; + serde_json::from_slice(&encoded).map_err(|_| anyhow::anyhow!("Invalid configuration structure")) +} + +fn is_space(ch: char) -> bool { + ch.is_whitespace() || ch == '\u{feff}' +} + +fn restore(value: &mut Value, prefix: &str, numbers: &[&str], env_key: Option<&str>) -> anyhow::Result<()> { + match value { + Value::String(s) => { + if let Some(index) = s.strip_prefix(prefix).and_then(|s| s.parse::().ok()) { + let token = numbers[index]; + *value = if let Some(env_key) = env_key { + Value::Number(token.parse().map_err(|_| { + anyhow::anyhow!( + "Environment key {:?}: numeric config input must use JSON number syntax", + env_key + ) + })?) + } else { + // Preserve existing JSON5 conveniences outside env. JSON numbers retain + // arbitrary precision throughout layering and config serialization. + token + .parse() + .map(Value::Number) + .or_else(|_| json5::from_str(token)) + .map_err(|_| anyhow::anyhow!("Invalid numeric configuration input"))? + }; + } + } + Value::Object(object) => { + for (key, value) in object { + if key == "env" && env_key.is_none() { + if let Value::Object(env) = value { + for (name, value) in env { + restore(value, prefix, numbers, Some(name))?; + } + } else { + restore(value, prefix, numbers, Some("env"))?; + } + } else { + restore(value, prefix, numbers, env_key)?; + } + } + } + Value::Array(values) => { + for value in values { + restore(value, prefix, numbers, env_key)?; + } + } + _ => {} + } + Ok(()) +} + +/// Merge only object-valued env maps. Invalid higher precedence input remains +/// invalid instead of silently falling back to the lower layer. +pub(super) fn overlay(base: &mut Value, higher: &Value) { + if let (Some(base), Some(higher)) = (base.as_object_mut(), higher.as_object()) { + base.extend(higher.clone()); + } else { + *base = higher.clone(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}; + #[test] + fn numeric_input_is_lossless_with_json5_comments_and_strings() { + let value = parse( + r#"{ // comment 111 + env: { HUGE: 9007199254740993123456789, DECIMAL: 0.1234567890123456789012345, + EXP: 1.000000000000000000001e+300, STRING: '12 // 99', BOOL: false, }, + 'server': 'http://127.0.0.1:9', /* 222 */ 'num-replicas': 3, + }"#, + ) + .unwrap(); + assert_eq!(value["env"]["HUGE"].to_string(), "9007199254740993123456789"); + assert_eq!(value["env"]["DECIMAL"].to_string(), "0.1234567890123456789012345"); + assert_eq!(value["env"]["EXP"].to_string(), "1.000000000000000000001e+300"); + assert_eq!(value["env"]["STRING"], "12 // 99"); + assert_eq!(value["num-replicas"], 3); + let unicode_lines = parse("{ // comment\u{2028} env: { NUMBER: 9007199254740993123456789\u{feff}} }").unwrap(); + assert_eq!(unicode_lines["env"]["NUMBER"].to_string(), "9007199254740993123456789"); + } + #[test] + fn numeric_extensions_and_parse_errors_do_not_quote_secret_input() { + for input in ["NaN", "Infinity", "-Infinity", "0xFF", "+2", ".5"] { + assert!(parse(&format!("{{env: {{KEY: {input}}}}}")).is_err()); + } + let error = parse("{env: {KEY: 'generated-secret-sentinel', broken }").unwrap_err(); + assert!(!format!("{error:#}").contains("generated-secret-sentinel")); + let value = parse("{env: {A: '__spacetime_numeric_0', B: 2}}").unwrap(); + assert_eq!(value["env"]["A"], "__spacetime_numeric_0"); + assert_eq!(value["env"]["B"], 2); + let escaped = parse(r#"{env: {A: '\u005f_spacetime_numeric_999', B: 2}}"#).unwrap(); + assert_eq!(escaped["env"]["A"], "__spacetime_numeric_999"); + } + #[test] + fn four_layers_and_parent_child_merge_individual_keys() { + let dir = tempfile::tempdir().unwrap(); + for (file, json) in [ + ( + "spacetime.json", + r#"{database:'parent', env:{A:'base',B:1},children:[{database:'child',env:{B:2,C:'child'}}]}"#, + ), + ( + "spacetime.local.json", + r#"{env:{A:'local'},children:[{env:{D:'local-child'}}]}"#, + ), + ( + "spacetime.prod.json", + r#"{env:{A:'prod',E:true},children:[{env:{B:3}}]}"#, + ), + ( + "spacetime.prod.local.json", + r#"{env:{A:'prod-local'},children:[{env:{}}]}"#, + ), + ] { + std::fs::write(dir.path().join(file), json).unwrap(); + } + let config = find_and_load_with_env_from(Some("prod"), dir.path().to_owned()) + .unwrap() + .unwrap(); + let targets = config.config.collect_all_targets_with_inheritance(); + assert_eq!( + targets[0].fields["env"], + serde_json::json!({"A":"prod-local","B":1,"E":true}) + ); + assert_eq!( + targets[1].fields["env"], + serde_json::json!({"A":"prod-local","B":3,"C":"child","D":"local-child","E":true}) + ); + // Direct loads use the same lossless parser. + let base = SpacetimeConfig::load(&dir.path().join("spacetime.json")).unwrap(); + assert_eq!(base.additional_fields["env"]["B"], 1); + } + #[test] + fn loaded_flattened_configuration_preserves_full_precision_and_redacts_structure_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + for number in [ + "9007199254740993123456789", + "0.1234567890123456789012345", + "1.000000000000000000001e+300", + ] { + std::fs::write( + &path, + format!(r#"{{"database":"owned","env":{{"NUMBER":{number}}},"children":[{{"database":"child"}}]}}"#), + ) + .unwrap(); + let direct = SpacetimeConfig::load(&path).unwrap(); + let layered = find_and_load_with_env_from(None, dir.path().to_owned()) + .unwrap() + .unwrap() + .config; + for config in [direct, layered] { + for target in config.collect_all_targets_with_inheritance() { + assert_eq!(target.fields["env"]["NUMBER"].to_string(), number); + } + } + } + std::fs::write( + &path, + r#"{"children":"private-structure-sentinel","env":{"NUMBER":9007199254740993123456789}}"#, + ) + .unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let error = format!("{error:#}"); + assert!(!error.contains("private-structure-sentinel")); + assert!(!error.contains("9007199254740993123456789")); + } + } + + #[test] + fn invalid_higher_layer_is_not_an_empty_map_or_fallback() { + let mut base = serde_json::json!({"A":"base"}); + overlay(&mut base, &Value::Null); + assert!(base.is_null()); + let config: SpacetimeConfig = + serde_json::from_value(serde_json::json!({"env":{"A":1},"children":[{"env":null}]})).unwrap(); + assert!(config.collect_all_targets_with_inheritance()[1].fields["env"].is_null()); + } +} diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs new file mode 100644 index 00000000000..58876664f48 --- /dev/null +++ b/crates/cli/src/subcommands/env.rs @@ -0,0 +1,284 @@ +//! Read-only conveniences over the private st_env SQL table. +use anyhow::{ensure, Context}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_lib::environment::{validate_key, MAX_ENV_KEY_BYTES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS}; + +use super::{ + db_arg_resolution::{load_config_db_targets, resolve_database_arg}, + sql, +}; +use crate::{api::ClientApi, common_args, Config}; + +pub fn cli() -> Command { + let target = |command: Command| { + command + .arg( + Arg::new("database") + .index(1) + .required(true) + .help("The database name, identity, or configured target"), + ) + .arg(common_args::server()) + .arg(common_args::anonymous()) + .arg(common_args::yes()) + .arg(common_args::confirmed()) + .arg( + Arg::new("no_config") + .long("no-config") + .action(ArgAction::SetTrue) + .help("Ignore project configuration when resolving the database target"), + ) + }; + Command::new("env") + .about("Inspect published database environment variables") + .subcommand_required(true) + .subcommand(target( + Command::new("get").about("Read one published environment value").arg( + Arg::new("key") + .index(2) + .required(true) + .help("The declared environment key to read"), + ), + )) + .subcommand(target( + Command::new("list").about("List published environment keys (never values)"), + )) +} + +#[derive(Clone)] +enum Query { + List, + Get(String), +} +impl Query { + fn sql(&self) -> anyhow::Result { + match self { + Self::List => Ok("SELECT key FROM st_env".into()), + Self::Get(key) => { + // POSIX names cannot contain quotes or SQL syntax. + validate_key(key).map_err(|_| anyhow::anyhow!("Invalid environment key name"))?; + Ok(format!("SELECT value FROM st_env WHERE key = '{key}'")) + } + } + } +} + +pub async fn exec(config: Config, args: &ArgMatches) -> anyhow::Result<()> { + let (command, args) = args.subcommand().context("Expected env get or list")?; + let query = match command { + "list" => Query::List, + "get" => Query::Get( + args.get_one::("key") + .context("Expected environment key")? + .clone(), + ), + _ => anyhow::bail!("Environment values can only be changed by publishing"), + }; + query.sql()?; + let targets = load_config_db_targets(args.get_flag("no_config"))?; + let database = resolve_database_arg( + args.get_one::("database").map(String::as_str), + targets.as_deref(), + "spacetime env get/list ", + )?; + let con = sql::parse_req(config, args, &database.database, database.server.as_deref()).await?; + let mut request = ClientApi::new(con).sql(); + if let Some(confirmed) = args.get_one::("confirmed") { + request = request.query(&[("confirmed", confirmed)]); + } + print!("{}", fetch(request, query).await?); + Ok(()) +} + +async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result { + use futures::StreamExt; + let response = request + .timeout(std::time::Duration::from_secs(30)) + .body(query.sql()?) + .send() + .await?; + ensure!( + response.status().is_success(), + "Environment read failed with HTTP {}", + response.status() + ); + let mut body = Vec::new(); + let limit = MAX_ENV_VARS * (MAX_ENV_KEY_BYTES + MAX_ENV_VALUE_BYTES) * 6 + 64 * 1024; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + ensure!( + body.len().saturating_add(chunk.len()) <= limit, + "Environment read response exceeds limit" + ); + body.extend_from_slice(&chunk); + } + render(&body, &query) +} + +fn render(body: &[u8], query: &Query) -> anyhow::Result { + // Only project the requested single string column; do not dump an error or + // unexpected response which could contain unrequested secret values. + let results: Vec>> = + serde_json::from_slice(body).map_err(|_| anyhow::anyhow!("Invalid environment read response"))?; + ensure!(results.len() == 1, "Invalid environment read result count"); + let result = &results[0]; + let expected = match query { + Query::List => "key", + Query::Get(_) => "value", + }; + ensure!( + result.schema.elements.len() == 1 + && result.schema.elements[0].name.as_deref() == Some(expected) + && result.schema.elements[0].algebraic_type == spacetimedb_lib::AlgebraicType::String, + "Invalid environment read projection" + ); + let mut values = Vec::new(); + for row in &result.rows { + ensure!(row.len() == 1, "Invalid environment read row"); + if matches!(query, Query::List) { + validate_key(&row[0]).context("Invalid environment key in response")?; + } + ensure!( + row[0].len() <= MAX_ENV_VALUE_BYTES, + "Environment read value exceeds limit" + ); + values.push(row[0].as_str()); + } + match query { + Query::List => { + ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); + values.sort_unstable(); + } + Query::Get(_) => ensure!(values.len() == 1, "Environment key is absent"), + } + Ok(values.into_iter().map(|v| format!("{v}\n")).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn body(column: &'static str, rows: Vec>) -> Vec { + serde_json::to_vec(&[spacetimedb_client_api_messages::http::SqlStmtResult { + schema: spacetimedb_lib::sats::ProductType::from([(column, spacetimedb_lib::AlgebraicType::String)]), + rows, + total_duration_micros: 0, + stats: Default::default(), + }]) + .unwrap() + } + #[test] + fn read_only_commands_and_safe_queries() { + for command in ["set", "del", "delete", "update"] { + assert!(cli().try_get_matches_from(["env", command, "db", "KEY"]).is_err()); + } + let matches = cli() + .try_get_matches_from([ + "env", + "get", + "db", + "KEY", + "--server", + "http://127.0.0.1:9", + "--no-config", + ]) + .unwrap(); + let get = matches.subcommand_matches("get").unwrap(); + assert_eq!(get.get_one::("database").unwrap(), "db"); + assert_eq!(get.get_one::("key").unwrap(), "KEY"); + assert_eq!(Query::List.sql().unwrap(), "SELECT key FROM st_env"); + assert_eq!( + Query::Get("KEY".into()).sql().unwrap(), + "SELECT value FROM st_env WHERE key = 'KEY'" + ); + assert!(Query::Get("x';DELETE FROM st_env;--".into()).sql().is_err()); + } + #[test] + fn list_projects_keys_and_rejects_unexpected_secret_columns() { + assert_eq!( + render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List).unwrap(), + "A\nZ\n" + ); + let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); + assert!(!format!("{err:#}").contains("generated-secret-sentinel")); + assert_eq!( + render(&body("value", vec![vec![""]]), &Query::Get("A".into())).unwrap(), + "\n" + ); + assert!(render(&body("value", vec![]), &Query::Get("A".into())).is_err()); + } + #[tokio::test] + async fn actual_loopback_queries_and_error_redaction() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (query, status, response, expected) in [ + (Query::List, "200 OK", body("key", vec![vec!["KEY"]]), Some("KEY\n")), + ( + Query::Get("KEY".into()), + "200 OK", + body("value", vec![vec!["generated-read-sentinel"]]), + Some("generated-read-sentinel\n"), + ), + ( + Query::Get("KEY".into()), + "403 Forbidden", + b"generated-error-secret".to_vec(), + None, + ), + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let expected_sql = query.sql().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut input = Vec::new(); + let (head, length) = loop { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + assert!(input.len() <= 16384); + if let Some(end) = input.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = std::str::from_utf8(&input[..end]).unwrap(); + assert!(headers.starts_with("POST /v1/database/owned/sql HTTP/1.1\r\n")); + let length: usize = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length: ") + .map(str::to_owned) + }) + .unwrap() + .parse() + .unwrap(); + break (end + 4, length); + } + }; + while input.len() < head + length { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + } + assert_eq!(&input[head..head + length], expected_sql.as_bytes()); + let headers = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", response.len()); + stream.write_all(headers.as_bytes()).await.unwrap(); + stream.write_all(&response).await.unwrap(); + stream.shutdown().await.unwrap(); + }); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + let result = fetch(client.post(format!("http://{address}/v1/database/owned/sql")), query).await; + match expected { + Some(expected) => assert_eq!(result.unwrap(), expected), + None => assert!(!format!("{:#}", result.unwrap_err()).contains("generated-error-secret")), + } + tokio::time::timeout(std::time::Duration::from_secs(5), server) + .await + .unwrap() + .unwrap(); + } + } +} diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index af0d2e364c3..9c659a5880f 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -5,6 +5,7 @@ pub mod delete; pub mod describe; pub mod dev; pub mod dns; +pub mod env; pub mod generate; pub mod init; pub mod list; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..b734b37cba8 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::{ensure, Context}; use clap::Arg; use clap::ArgAction::{self, Set, SetTrue}; @@ -6,8 +8,8 @@ use reqwest::{StatusCode, Url}; use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, PublishResult}; use spacetimedb_client_api_messages::name::{DatabaseNameError, PrePublishResult, PrettyPrintStyle, PublishOp}; use std::collections::HashMap; +use std::env; use std::path::PathBuf; -use std::{env, fs}; use crate::common_args::parse_optional_dotnet_version; use crate::common_args::ClearMode; @@ -87,6 +89,7 @@ pub fn build_publish_schema(command: &clap::Command) -> Result( ) .await? }; - let program_bytes = fs::read(path_to_program)?; + let program_bytes = environment::read_program(&path_to_program)?; + let module_schema = environment::inspect(&program_bytes, host_type).await?; + let environment = environment::resolve( + module_schema.environment(), + command_config.get_config_value("env"), + |key| std::env::var_os(key), + )?; + print!("{}", environment.display()); let server_address = { let url = Url::parse(&database_host)?; @@ -583,7 +593,10 @@ async fn execute_publish_configs<'a>( database_host ); - let client = reqwest::Client::new(); + // The body contains secrets. Never replay it to a redirect destination. + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; // If a name was given, ensure to percent-encode it. // We also use PUT with a name or identity, and POST otherwise. let mut builder = if let Some(name_or_identity) = name_or_identity { @@ -634,8 +647,24 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let res = builder.body(program_bytes).send().await?; - let response: PublishResult = res.json_or_error().await?; + let payload = spacetimedb_client_api_messages::publish::PublishRequest { + module: program_bytes, + environment: environment.values, + } + .encode()?; + let res = builder + .header( + reqwest::header::CONTENT_TYPE, + spacetimedb_client_api_messages::publish::CONTENT_TYPE, + ) + .body(payload) + .send() + .await?; + anyhow::ensure!(res.status().is_success(), "Publish failed with HTTP {}", res.status()); + let response: PublishResult = res + .json() + .await + .map_err(|_| anyhow::anyhow!("Invalid publish response"))?; match response { PublishResult::Success { domain, diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs new file mode 100644 index 00000000000..17fc67efb62 --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -0,0 +1,180 @@ +//! Resolve a complete, declared environment without consulting stored values. +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{ensure, Context}; +use serde_json::Value; +use spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; +use spacetimedb_schema::def::ModuleDef; +use tokio::io::AsyncReadExt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Source { + Config, + Shell, +} +impl std::fmt::Display for Source { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Config => "config", + Self::Shell => "shell", + }) + } +} + +// Deliberately no Debug: values are credentials, not diagnostics. +pub(super) struct Resolved { + pub values: BTreeMap, + pub sources: BTreeMap, +} +impl Resolved { + pub fn display(&self) -> String { + use std::fmt::Write; + let mut output = String::new(); + for (name, source) in &self.sources { + let _ = writeln!(output, "Environment {name} ({source})"); + } + output + } +} + +pub(super) fn resolve( + schema: &EnvironmentSchema, + config: Option<&Value>, + mut shell: impl FnMut(&str) -> Option, +) -> anyhow::Result { + let mut resolved = Resolved { + values: BTreeMap::new(), + sources: BTreeMap::new(), + }; + if let Some(config) = config { + let config = config.as_object().context("Environment config must be an object")?; + for (name, value) in config { + ensure!( + schema.get(name).is_some(), + "Environment key {name:?}: key is not declared" + ); + let value = match value { + Value::String(value) => value.clone(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + _ => anyhow::bail!("Environment key {name:?}: config input must be a string, boolean or JSON number"), + }; + resolved.values.insert(name.clone(), value); + resolved.sources.insert(name.clone(), Source::Config); + } + } + // Lookup only the new artifact's declared names, never enumerate ambient values. + for declaration in schema.declarations() { + if let Some(value) = shell(&declaration.name) { + let value = value + .into_string() + .map_err(|_| anyhow::anyhow!("Environment key {:?}: shell input must be UTF-8", declaration.name))?; + resolved.values.insert(declaration.name.clone(), value); + resolved.sources.insert(declaration.name.clone(), Source::Shell); + } + } + schema.validate_values(&resolved.values)?; + Ok(resolved) +} + +pub(super) fn read_program(path: &std::path::Path) -> anyhow::Result> { + use std::io::Read; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, + "Module exceeds publish size limit" + ); + Ok(bytes) +} + +const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; +const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Inspect exactly the artifact bytes that will be uploaded. The private copy +/// prevents path replacement between inspection and upload, including --bin-path. +/// This invokes only local extraction, never a server or a saved CLI context. +pub(super) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { + let extractor = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone"))?; + inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await +} + +async fn inspect_with( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, +) -> anyhow::Result { + let (mut send, mut recv) = tokio::sync::oneshot::channel(); + // This owner retains the child and private file until actual reaping, even + // when its caller drops while extraction or stdout reading is in progress. + tokio::spawn(async move { + let result = async { + let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; + let module = dir.path().join("module"); + tokio::fs::write(&module, program) + .await + .context("Cannot prepare module inspection input")?; + let mut child = tokio::process::Command::new(extractor) + .arg("extract-schema") + .arg(&module) + .arg("--host-type") + .arg(host_type.to_ascii_lowercase()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("Cannot start local module schema inspection")?; + let mut output = Vec::new(); + let mut stdout = child + .stdout + .take() + .context("Module inspection stdout unavailable")? + .take(MAX_SCHEMA_BYTES + 1); + let result = tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), + result = tokio::time::timeout(deadline, async { + stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; + ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); + let status = child.wait().await.context("Cannot reap local module inspector")?; + ensure!(status.success(), "Local module schema inspection failed"); + Ok(()) + }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), + }; + if result.is_err() { + // Queue termination, then retain ownership through positive reaping. + let _ = child.start_kill(); + child + .wait() + .await + .context("Cannot reap failed local module inspector")?; + } + result?; + // Neither parser nor validation diagnostics may echo schema literals. + let SerdeWrapper::(raw) = serde_json::from_slice(&output) + .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; + let schema = + ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; + Ok(schema) + } + .await; + let _ = send.send(result); + }); + (&mut recv).await.context("Local module inspection owner failed")? +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs new file mode 100644 index 00000000000..5af04aa6877 --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -0,0 +1,307 @@ +use super::*; +use spacetimedb_lib::environment::{EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +#[test] +fn declared_shell_overrides_are_complete_and_redacted() { + let mut checked = Vec::new(); + let resolved = resolve( + &schema(), + Some(&serde_json::json!({"A":"config-sentinel","B":false})), + |name| { + checked.push(name.to_owned()); + match name { + "C" => Some("shell-sentinel".into()), + "A" => Some("".into()), + _ => None, + } + }, + ) + .unwrap(); + assert_eq!( + resolved.values, + BTreeMap::from([ + ("A".into(), "".into()), + ("B".into(), "false".into()), + ("C".into(), "shell-sentinel".into()) + ]) + ); + assert_eq!(checked, vec!["A", "B", "C", "OPTIONAL"]); + assert_eq!( + resolved.display(), + "Environment A (shell)\nEnvironment B (config)\nEnvironment C (shell)\n" + ); + assert!(!resolved.display().contains("sentinel")); + // No declaration means no ambient lookup, including PATH or credentials. + let empty = resolve(&EnvironmentSchema::default(), None, |_| panic!("ambient access")).unwrap(); + assert!(empty.values.is_empty()); +} + +#[test] +fn missing_required_never_reuses_old_values_and_optional_disappears() { + let config = serde_json::json!({"A":"first","B":true,"C":"first","OPTIONAL":"old"}); + let first = resolve(&schema(), Some(&config), |_| None).unwrap(); + assert!(first.values.contains_key("OPTIONAL")); + let second = resolve( + &schema(), + Some(&serde_json::json!({"A":"next","B":false,"C":"next"})), + |_| None, + ) + .unwrap(); + assert!(!second.values.contains_key("OPTIONAL")); + let error = resolve(&schema(), Some(&serde_json::json!({"A":"first","B":true})), |_| None) + .err() + .unwrap(); + assert!(error.to_string().contains('C')); +} + +#[test] +fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { + for input in [Value::Null, serde_json::json!([]), serde_json::json!({})] { + let config = serde_json::json!({"A":input,"B":true,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |_| None).err().unwrap(); + assert!(!format!("{error:#}").contains("private-sentinel")); + } + let config = serde_json::json!({"A":"private-sentinel","B":false,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |name| { + (name == "B").then(|| "invalid-shell-secret".into()) + }) + .err() + .unwrap(); + let error = format!("{error:#}"); + assert!(error.contains('B')); + assert!(!error.contains("invalid-shell-secret") && !error.contains("private-sentinel")); + let error = resolve(&schema(), Some(&serde_json::json!({"UNDECLARED":"secret"})), |_| { + panic!("must fail first") + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UNDECLARED")); +} + +#[test] +fn env_layer_selector_is_not_a_value_source() { + let command = super::super::cli(); + let args = command + .clone() + .try_get_matches_from(["publish", "db", "--env", "prod"]) + .unwrap(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let config = crate::spacetime_config::CommandConfig::new( + &schema, + std::collections::HashMap::from([("env".into(), serde_json::json!({"A":"from-config"}))]), + &args, + ) + .unwrap(); + assert_eq!(config.get_config_value("env").unwrap()["A"], "from-config"); + assert!(!config.is_from_cli("env")); +} + +#[test] +fn number_boolean_and_empty_string_conversion_has_no_float_rounding() { + let schema = EnvironmentSchema::new( + ["NUMBER", "BOOL", "EMPTY"] + .map(|name| Declaration { + name: name.into(), + constraint: Constraint::AnyString, + optional: false, + }) + .to_vec(), + ) + .unwrap(); + let config = serde_json::from_str(r#"{"NUMBER":9007199254740993123456789,"BOOL":false,"EMPTY":""}"#).unwrap(); + let resolved = resolve(&schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values["NUMBER"], "9007199254740993123456789"); + assert_eq!(resolved.values["BOOL"], "false"); + assert_eq!(resolved.values["EMPTY"], ""); +} + +#[cfg(unix)] +#[test] +fn non_utf8_declared_shell_value_is_rejected_without_bytes() { + use std::os::unix::ffi::OsStringExt; + let error = resolve(&schema(), None, |name| { + (name == "A").then(|| OsString::from_vec(vec![0xff, 0xfe])) + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UTF-8")); +} + +// These fixtures invoke only a locally generated executable in an owned tempdir. +// No CLI config, server credentials or user environment are imported. +#[cfg(unix)] +fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("inspector"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + (dir, path) +} + +#[cfg(unix)] +#[tokio::test] +async fn local_inspection_passes_exact_bytes_host_and_requires_success() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let result = inspect_with( + extractor, + b"exact-artifact".to_vec(), + "Js".into(), + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(result.environment(), &schema()); + drop(dir); + let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); + assert!( + inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .is_err() + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { + for script in [ + "printf 'generated-inspector-secret'", + "exec /usr/bin/head -c 16777217 /dev/zero", + ] { + let (_dir, extractor) = inspector(script); + let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!format!("{error:#}").contains("generated-inspector-secret")); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn inspection_timeout_and_dropped_waiter_reap_exact_child() { + // A shell-only busy loop has no descendants and records the exact child PID. + // kill(0) via the owned process is not used for proof: wait for /bin/kill -0 + // to report ESRCH after our owner has called wait, including cancellation. + async fn wait_pid(path: &std::path::Path) -> String { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(pid) = tokio::fs::read_to_string(path).await + && !pid.trim().is_empty() + { + break pid; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap() + } + async fn gone(pid: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let status = tokio::process::Command::new("/bin/kill") + .args(["-0", pid.trim()]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + .unwrap(); + if !status.success() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + } + for cancel in [false, true] { + let (dir, extractor) = inspector("placeholder"); + let pid_file = dir.path().join("pid"); + std::fs::write( + &extractor, + format!( + "#!/bin/sh\nprintf '%s' \"$$\" > '{}'\nwhile :; do :; done\n", + pid_file.display() + ), + ) + .unwrap(); + let operation = tokio::spawn(inspect_with( + extractor, + b"input".to_vec(), + "Wasm".into(), + if cancel { + Duration::from_secs(30) + } else { + Duration::from_millis(300) + }, + )); + let pid = wait_pid(&pid_file).await; + if cancel { + operation.abort(); + let _ = operation.await; + } else { + assert!(operation.await.unwrap().is_err()); + } + gone(&pid).await; + } +} + +#[tokio::test] +#[ignore = "requires explicit locally built ENV-aware standalone and declared Wasm fixture paths"] +async fn actual_precompiled_declarations_are_inspected_without_server_or_values() { + let extractor = + PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_EXTRACTOR").expect("explicit inspector path")); + let module = PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_MODULE").expect("explicit module path")); + assert!(extractor.is_absolute() && module.is_absolute()); + let program = read_program(&module).unwrap(); + let inspected = inspect_with(extractor, program, "Wasm".into(), INSPECT_TIMEOUT) + .await + .unwrap(); + let schema = inspected.environment(); + assert!(inspected.environment_declared()); + assert!(!schema.get("REQUIRED").unwrap().optional); + assert_eq!( + schema.get("MODE").unwrap().constraint, + Constraint::OneOf(vec!["other".into(), "ready".into()]) + ); + let config = serde_json::json!({"REQUIRED":"generated-local-inspection-sentinel","MODE":"ready"}); + let resolved = resolve(schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values.len(), 2); + assert!(!resolved.display().contains("generated-local-inspection-sentinel")); + assert!(resolve(schema, None, |_| None).is_err()); +} diff --git a/crates/client-api-messages/src/lib.rs b/crates/client-api-messages/src/lib.rs index bf2f1dc9ca5..67b58de659f 100644 --- a/crates/client-api-messages/src/lib.rs +++ b/crates/client-api-messages/src/lib.rs @@ -4,3 +4,5 @@ pub mod energy; pub mod http; pub mod name; pub mod websocket; + +pub mod publish; diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs new file mode 100644 index 00000000000..fea29732139 --- /dev/null +++ b/crates/client-api-messages/src/publish.rs @@ -0,0 +1,117 @@ +//! Complete publish input. Environment values travel only in the request body. +use serde::{Deserialize, Deserializer, Serialize}; +use serde_with::{base64::Base64, serde_as}; +use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; +use std::collections::BTreeMap; + +pub const CONTENT_TYPE: &str = "application/vnd.spacetimedb.publish+json"; +pub const MAX_MODULE_BYTES: usize = 128 * 1024 * 1024; +/// Includes base64 module expansion and worst-case JSON escaping of configuration. +pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; + +/// Values deliberately have no Debug representation. Omission is an empty map, +/// including for a publish of an unchanged module. +#[serde_as] +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + #[serde_as(as = "Base64")] + pub module: Vec, + #[serde(default, deserialize_with = "deserialize_environment")] + pub environment: BTreeMap, +} + +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum PublishRequestError { + #[error("invalid publish request body")] + Invalid, + #[error("publish request exceeds size limit")] + TooLarge, +} + +impl PublishRequest { + pub fn decode(body: &[u8]) -> Result { + if body.len() > MAX_REQUEST_BYTES { + return Err(PublishRequestError::TooLarge); + } + // Never expose serde's error text: it can quote a supplied secret. + let request: Self = serde_json::from_slice(body).map_err(|_| PublishRequestError::Invalid)?; + request.validate()?; + Ok(request) + } + + pub fn encode(&self) -> Result, PublishRequestError> { + self.validate()?; + serde_json::to_vec(self).map_err(|_| PublishRequestError::Invalid) + } + + fn validate(&self) -> Result<(), PublishRequestError> { + if self.module.len() > MAX_MODULE_BYTES || self.environment.len() > MAX_ENV_VARS { + return Err(PublishRequestError::TooLarge); + } + for (key, value) in &self.environment { + validate_key(key).map_err(|_| PublishRequestError::Invalid)?; + validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; + } + Ok(()) + } +} + +fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = BTreeMap; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a complete map of environment strings") + } + fn visit_map>(self, mut map: A) -> Result { + use serde::de::Error; + let mut values = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { + return Err(A::Error::custom("invalid environment keys")); + } + let value = map.next_value::()?; + if validate_value(&value).is_err() { + return Err(A::Error::custom("environment value exceeds size limit")); + } + values.insert(key, value); + } + Ok(values) + } + } + de.deserialize_map(Visitor) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn roundtrip_and_omission_preserve_complete_string_input() { + let request = PublishRequest { + module: vec![0, 1, 255], + environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "雪\0false".into())]), + }; + let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); + assert_eq!(decoded.module, request.module); + assert_eq!(decoded.environment, request.environment); + assert!(PublishRequest::decode(br#"{"module":""}"#) + .unwrap() + .environment + .is_empty()); + } + #[test] + fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { + for body in [ + r#"{"module":"","environment":{"KEY":true}}"#, + r#"{"module":"","environment":{"KEY":null}}"#, + r#"{"module":"","environment":{"KEY":"first","KEY":"secret-marker"}}"#, + r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#, + r#"{"module":"secret-marker"}"#, + r#"{"module":"","unknown":"secret-marker"}"#, + ] { + let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + } +} diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 68fd0a61bb5..9423849ddf8 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -206,6 +206,26 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + pub async fn update_with_environment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_environment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + environment, + ) + .await + } } /// Parameters for publishing a database. /// @@ -215,6 +235,8 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, + /// Complete publish input, never persisted in the public Database record. + pub environment: std::collections::BTreeMap, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -232,6 +254,7 @@ pub struct DatabaseDef { pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, + pub environment: std::collections::BTreeMap, pub num_replicas: Option, pub host_type: Option, } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index f170f9b290e..5d98a065976 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,3 +1,6 @@ +mod publish_environment; +use publish_environment::{ModuleBody, PublishBody}; + use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; @@ -835,7 +838,10 @@ pub async fn reset( host_type, }): Query, Extension(auth): Extension, - program_bytes: Option, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -856,6 +862,7 @@ pub async fn reset( DatabaseResetDef { database_identity, program_bytes, + environment, num_replicas, host_type: Some(host_type), }, @@ -933,7 +940,10 @@ pub async fn publish( update_confirmation_timeout: confirmation_timeout, }): Query, Extension(auth): Extension, - program_bytes: Bytes, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. @@ -963,13 +973,17 @@ pub async fn publish( host_type, }), Extension(auth), - Some(program_bytes), + PublishBody { + program_bytes, + environment, + }, ) .await; } } } + let program_bytes = program_bytes.unwrap_or_default(); let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; let maybe_parent_database_identity = match parent.as_ref() { None => None, @@ -1033,6 +1047,7 @@ pub async fn publish( DatabaseDef { database_identity, program_bytes, + environment, num_replicas, host_type, parent, @@ -1217,7 +1232,7 @@ pub async fn pre_publish Extension(ResolvedDatabase(database)): Extension, Query(PrePublishQueryParams { style, host_type }): Query, Extension(auth): Extension, - program_bytes: Bytes, + ModuleBody(program_bytes): ModuleBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -1236,6 +1251,7 @@ pub async fn pre_publish DatabaseDef { database_identity, program_bytes, + environment: Default::default(), num_replicas: None, host_type, parent: None, diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs new file mode 100644 index 00000000000..0287d226688 --- /dev/null +++ b/crates/client-api/src/routes/database/publish_environment.rs @@ -0,0 +1,139 @@ +//! Bounded publish extraction. Neither errors nor Debug output retain configuration values. +use axum::body::{to_bytes, Bytes}; +use axum::extract::{FromRequest, Request}; +use axum::response::{IntoResponse, Response}; +use http::{header, StatusCode}; +use spacetimedb_client_api_messages::publish::{PublishRequest, CONTENT_TYPE, MAX_MODULE_BYTES, MAX_REQUEST_BYTES}; +use std::collections::BTreeMap; + +pub struct PublishBody { + pub program_bytes: Option, + pub environment: BTreeMap, +} + +async fn bounded_body(request: Request, limit: usize) -> Result { + if request + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|len| len > limit as u64) + { + return Err((StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()); + } + to_bytes(request.into_body(), limit) + .await + .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()) +} + +#[async_trait::async_trait] +impl FromRequest for PublishBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let envelope = request + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .unwrap_or_default() + .trim() + .eq_ignore_ascii_case(CONTENT_TYPE) + }); + let bytes = bounded_body(request, if envelope { MAX_REQUEST_BYTES } else { MAX_MODULE_BYTES }).await?; + if envelope { + let request = PublishRequest::decode(&bytes) + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; + Ok(Self { + program_bytes: (!request.module.is_empty()).then_some(request.module.into()), + environment: request.environment, + }) + } else { + // An absent reset body retains the program, but never retains env values. + Ok(Self { + program_bytes: (!bytes.is_empty()).then_some(bytes), + environment: BTreeMap::new(), + }) + } + } +} + +pub struct ModuleBody(pub Bytes); + +#[async_trait::async_trait] +impl FromRequest for ModuleBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + bounded_body(request, MAX_MODULE_BYTES).await.map(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + + #[tokio::test] + async fn legacy_envelope_and_empty_reset_have_complete_input_semantics() { + let legacy = PublishBody::from_request(Request::new(Body::from("module")), &()) + .await + .unwrap(); + assert_eq!(legacy.program_bytes.unwrap(), "module"); + assert!(legacy.environment.is_empty()); + let empty = PublishBody::from_request(Request::new(Body::empty()), &()) + .await + .unwrap(); + assert!(empty.program_bytes.is_none()); + assert!(empty.environment.is_empty()); + let input = PublishRequest { + module: vec![1, 2, 3], + environment: BTreeMap::from([("TOKEN".into(), "雪\0".into())]), + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(input.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert_eq!(extracted.environment, input.environment); + assert_eq!(extracted.program_bytes.unwrap(), input.module); + let reset = PublishRequest { + module: vec![], + environment: input.environment, + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(reset.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert!(extracted.program_bytes.is_none()); + assert_eq!(extracted.environment, reset.environment); + } + + #[tokio::test] + async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { + let stream = futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from_static(b"12345")), + Ok(Bytes::from_static(b"67890")), + ]); + let error = bounded_body(Request::new(Body::from_stream(stream)), 8) + .await + .unwrap_err(); + assert_eq!(error.into_response().status(), StatusCode::PAYLOAD_TOO_LARGE); + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#)) + .unwrap(); + let error = PublishBody::from_request(request, &()) + .await + .err() + .unwrap() + .into_response(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(error.into_body(), 1024).await.unwrap(); + assert!(!String::from_utf8_lossy(&body).contains("secret-marker")); + } +} diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs index 87b7a7231f7..10718637d13 100644 --- a/crates/core/src/db/environment.rs +++ b/crates/core/src/db/environment.rs @@ -1,15 +1,16 @@ //! Dedicated access to the private environment store. //! -//! Mutation callers must authorize owner/admin access before calling these -//! helpers and commit through the normal module transaction machinery so -//! dependent views refresh. Helpers never acquire a second transaction. +//! Publishing replaces the complete environment inside the program transaction. +//! Helpers never acquire a second transaction or expose individual mutation APIs. use super::relational_db::{MutTx, RelationalDB}; use crate::error::DBError; use spacetimedb_datastore::error::DatastoreError; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; -use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_lib::environment::{ + validate_key, EnvironmentSchema, EnvironmentSchemaError, EnvironmentValidationError, +}; use spacetimedb_sats::AlgebraicValue; use std::collections::BTreeMap; @@ -18,6 +19,8 @@ pub enum EnvironmentError { #[error(transparent)] Validation(#[from] EnvironmentValidationError), #[error(transparent)] + Schema(#[from] EnvironmentSchemaError), + #[error(transparent)] Datastore(#[from] DatastoreError), #[error(transparent)] Database(#[from] DBError), @@ -43,29 +46,37 @@ pub fn snapshot(state: &impl StateView) -> Result, Envi .collect() } -/// Insert or replace one key. Validation occurs before any mutation. -pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { - validate_key(key)?; - validate_value(value)?; - let previous = get(tx, key)?; - if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { - return Err(EnvironmentValidationError::TooManyVariables.into()); +/// Apply a complete publish configuration in the caller's program transaction. +/// Validate every input before modifying any row, even if the caller chooses to +/// recover from a validation error and commit other transaction work. +pub fn replace( + db: &RelationalDB, + tx: &mut MutTx, + schema: &EnvironmentSchema, + values: &BTreeMap, +) -> Result<(), EnvironmentError> { + schema.validate_values(values)?; + let previous = snapshot(tx)?; + for (key, value) in &previous { + if values.get(key) != Some(value) { + delete(db, tx, key)?; + } } - if previous.as_deref() == Some(value) { - return Ok(()); + for (key, value) in values { + if previous.get(key) != Some(value) { + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.clone(), + value: value.clone(), + }, + )?; + } } - delete(db, tx, key)?; - tx.insert_via_serialize_bsatn( - ST_ENV_ID, - &StEnvRow { - key: key.into(), - value: value.into(), - }, - )?; Ok(()) } -pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { +fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { validate_key(key)?; let pointer = tx .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? @@ -83,49 +94,73 @@ mod tests { use super::*; use crate::db::relational_db::tests_utils::TestDB; use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + EnvironmentDeclaration { + name: "REQUIRED".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }, + EnvironmentDeclaration { + name: "OPTIONAL".into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap() + } #[test] - fn missing_empty_nul_update_and_rollback() { + fn replacement_preserves_empty_and_nul_and_removes_omitted_values() { let db = TestDB::in_memory().unwrap(); - db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - assert_eq!(get(tx, "EMPTY")?, None); - set(&db, tx, "EMPTY", "")?; - set(&db, tx, "NUL", "a\0b")?; - assert_eq!(get(tx, "EMPTY")?, Some(String::new())); - assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); - Ok(()) + let initial = BTreeMap::from([("REQUIRED".into(), "".into()), ("OPTIONAL".into(), "a\0b".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &initial)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); + let next = BTreeMap::from([("REQUIRED".into(), "new".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &next)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), next)); + db.with_auto_commit(Workload::ForTests, |tx| { + replace(&db, tx, &EnvironmentSchema::default(), &BTreeMap::new()) }) .unwrap(); - let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - set(&db, tx, "EMPTY", "changed")?; - delete(&db, tx, "NUL")?; - Err(EnvironmentValidationError::InvalidKey.into()) - }); - assert!(result.is_err()); - db.with_read_only(Workload::ForTests, |tx| { - assert_eq!( - snapshot(tx).unwrap(), - BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) - ); - }); + db.with_read_only(Workload::ForTests, |tx| assert!(snapshot(tx).unwrap().is_empty())); } #[test] - fn capacity_and_value_limits_precede_mutation() { + fn invalid_complete_input_does_not_reuse_stored_values_or_mutate() { let db = TestDB::in_memory().unwrap(); + let initial = BTreeMap::from([("REQUIRED".into(), "old".into()), ("OPTIONAL".into(), "keep".into())]); db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { - for i in 0..MAX_ENV_VARS { - set(&db, tx, &format!("K{i}"), "")?; + replace(&db, tx, &schema(), &initial)?; + for invalid in [ + BTreeMap::new(), + BTreeMap::from([ + ("REQUIRED".into(), "new".into()), + ("UNKNOWN".into(), "secret-marker".into()), + ]), + BTreeMap::from([("REQUIRED".into(), "x".repeat(8193))]), + ] { + assert!(replace(&db, tx, &schema(), &invalid).is_err()); + assert_eq!(snapshot(tx)?, initial); } - assert!(set(&db, tx, "EXTRA", "").is_err()); - set(&db, tx, "K0", "updated")?; - assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); - assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); - assert!(delete(&db, tx, "K1")?); - assert!(!delete(&db, tx, "MISSING")?); - set(&db, tx, "EXTRA", "")?; Ok(()) }) .unwrap(); + let failed_publish = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + replace( + &db, + tx, + &schema(), + &BTreeMap::from([("REQUIRED".into(), "updated".into())]), + )?; + // Simulate a later failure in the same publish transaction. + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(failed_publish.is_err()); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); } } diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index ff017b733c8..38a198599b6 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -25,7 +25,7 @@ use crate::worker_metrics::{ record_module_host_init_attempt, record_module_host_init_failure, record_module_host_unexpected_exit, ModuleHostInitFailureCause, WORKER_METRICS, }; -use anyhow::{anyhow, bail, Context}; +use anyhow::{bail, Context}; use async_trait::async_trait; use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; @@ -92,6 +92,14 @@ where pub type ProgramStorage = Arc; +/// Private complete configuration for a not-yet-initialized database generation. +/// Implementations must verify the exact persisted database identity, program and +/// bootstrap generation. This source is never consulted during ordinary reopen. +#[async_trait] +pub trait InitialEnvironmentSource: Send + Sync { + async fn load(&self, database: &Database) -> anyhow::Result>; +} + /// A launched module host plus any pending controldb program-bootstrap completion work. pub struct ModuleHostWithBootstrap { pub module: ModuleHost, @@ -187,6 +195,7 @@ pub struct HostController { default_config: db::Config, /// The [`ProgramStorage`] to query when instantiating a module. program_storage: ProgramStorage, + initial_environment_source: Option>, /// The [`EnergyMonitor`] used by this controller. energy_monitor: Arc, /// The [`MemoryObserver`] used by this controller. @@ -357,6 +366,7 @@ impl HostController { hosts: <_>::default(), default_config, program_storage, + initial_environment_source: None, energy_monitor, memory_observer, persistence, @@ -368,6 +378,12 @@ impl HostController { } } + /// Install the private bootstrap input source before this controller is shared. + pub fn with_initial_environment_source(mut self, source: Arc) -> Self { + self.initial_environment_source = Some(source); + self + } + /// Replace the [`ProgramStorage`] used by this controller. pub fn set_program_storage(&mut self, ps: ProgramStorage) { self.program_storage = ps; @@ -514,6 +530,16 @@ impl HostController { /// This is not necessary during hotswap publishes, /// as the automigration planner and executor accomplish the same validity checks. pub async fn check_module_validity(&self, database: Database, program: Program) -> anyhow::Result> { + self.check_module_validity_with_environment(database, program, Default::default()) + .await + } + + pub async fn check_module_validity_with_environment( + &self, + database: Database, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result> { let (program, launched) = Host::try_init_in_memory_to_check( &self.runtimes, self.page_pool.clone(), @@ -529,12 +555,20 @@ impl HostController { ) .await?; - let InitDatabaseResult { reducer, .. } = launched.module_host.init_database(program).await?; + let result = launched + .module_host + .init_database_with_environment(program, environment) + .await; + let info = launched.module_host.info.clone(); + // Validation never starts scheduled work. Release its receiver before + // waiting for scheduler closure, including when initialization failed. + drop(launched.scheduler_starter); + launched.module_host.exit().await; + let InitDatabaseResult { reducer, .. } = result?; if let Some(call_result) = reducer { Result::from(call_result)?; } - - Ok(launched.module_host.info) + Ok(info) } /// Update the [`ModuleHost`] identified by `replica_id` to the given @@ -553,6 +587,27 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_environment( + database, + host_type, + replica_id, + program_bytes, + policy, + Default::default(), + ) + .await + } + + #[tracing::instrument(level = "trace", skip_all, err)] + pub async fn update_module_host_with_environment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -605,12 +660,14 @@ impl HostController { this.energy_monitor.clone(), this.unregister_fn(replica_id, database_identity), this.db_cores.take(), + environment, ) - .await?; + .await; + // Rejected publication leaves the existing host usable. Restore it + // before propagating validation or migration failure to the caller. *guard = Some(host); - - Ok::<_, anyhow::Error>(update_result) + update_result }) .await??; @@ -657,6 +714,25 @@ impl HostController { /// and deregister it from the controller. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { + let start = Instant::now(); + if tokio::time::timeout(timeout, self.exit_module_host_and_join(replica_id)) + .await + .is_err() + { + warn!( + "replica={replica_id} shutdown timed out after {}s", + start.elapsed().as_secs_f32() + ); + } + Ok(()) + } + + /// Wait for actual module and database closure, without treating an elapsed + /// request deadline as completion. The caller must retain this future and + /// exclude new launch admission until it returns, including if its own + /// request waiter is cancelled. + #[tracing::instrument(level = "trace", skip_all)] + pub async fn exit_module_host_and_join(&self, replica_id: u64) -> Result<(), anyhow::Error> { let Some(lock) = self.hosts.lock().remove(&replica_id) else { return Ok(()); }; @@ -677,35 +753,24 @@ impl HostController { }); defer!(warn_blocked.abort()); - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); + let mut guard = lock.write_owned().await; + let Some(host) = guard.take() else { + return Ok(()); + }; + let module = host.module.borrow().clone(); + let info = module.info(); - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); + let database_identity = info.database_identity; + let table_names = info.module_def.tables().map(|t| t.name.deref()); - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); - } + // Ensure we clear the metrics even if the future is cancelled. + defer!(remove_database_gauges(&database_identity, table_names)); + info!("replica={replica_id} database={database_identity} exiting module"); + module.exit().await; + info!("replica={replica_id} database={database_identity} exiting database"); + module.relational_db().shutdown().await; + info!("replica={replica_id} database={database_identity} module host exited"); Ok(()) } @@ -1030,32 +1095,25 @@ fn repair_stale_view_backing_tables_on_launch(launched: &LaunchedModule) -> anyh /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Otherwise publish the complete environment with the module, including when +/// its program hash is unchanged. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let addr = db.database_identity(); - match stored_program_hash(db)? { - None => Err(anyhow!("database `{addr}` not yet initialized")), - Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) - } - } + let Some(stored) = stored_program_hash(db)? else { + bail!("database `{addr}` not yet initialized"); + }; + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + // Even an unchanged program publishes a complete replacement environment. + module + .update_database_with_environment(program, old_module_info, policy, environment) + .await } /// Encapsulates a database, associated module, and auxiliary state. @@ -1194,6 +1252,14 @@ impl Host { } }; let bootstrap_generation = database.bootstrap_generation; + let initial_environment = if program_needs_init { + match &host_controller.initial_environment_source { + Some(source) => source.load(&database).await?, + None => Default::default(), + } + } else { + Default::default() + }; let mut bootstrap_completion = Some(BootstrapCompletion::durable(bootstrap_generation)); let relational_db = Arc::new(db); @@ -1284,7 +1350,10 @@ impl Host { }; if program_needs_init { - let InitDatabaseResult { reducer, tx_offset } = launched.module_host.init_database(program).await?; + let InitDatabaseResult { reducer, tx_offset } = launched + .module_host + .init_database_with_environment(program, initial_environment) + .await?; if let Some(call_result) = reducer { validate_init_reducer_call_result(call_result)?; } @@ -1414,6 +1483,7 @@ impl Host { energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, core: AllocatedJobCore, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1432,8 +1502,25 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = match update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + environment, + ) + .await + { + Ok(result) => result, + Err(error) => { + // This candidate was never installed or scheduled. Close its + // receiver first so cleanup cannot wait on an unstarted actor. + drop(scheduler_starter); + module.exit().await; + return Err(error); + } + }; // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. @@ -1470,7 +1557,10 @@ impl Host { let old_module = old_watcher.borrow().clone(); old_module.exit().await; } - _ => {} + _ => { + drop(scheduler_starter); + module.exit().await; + } } Ok(update_result) @@ -1678,6 +1768,42 @@ where mod tests { use super::*; + #[tokio::test] + async fn positive_close_stays_pending_past_a_waiter_deadline_until_host_ownership_is_released() { + use crate::db::persistence::LocalPersistenceProvider; + use spacetimedb_paths::FromPathUnchecked; + let temp = tempfile::tempdir().unwrap(); + let directory = Arc::new(ServerDataDir::from_path_unchecked(temp.path().to_owned())); + let controller = HostController::new( + directory.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + HostRuntimeConfig::new(WasmConfig::default(), V8Config::default(), ModuleHttpConfig::default()), + Arc::new(|_| std::future::ready(anyhow::Ok(None))), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(directory)), + JobCores::without_pinned_cores(), + ); + let cell = Arc::new(AsyncRwLock::new(None)); + controller.hosts.lock().insert(17, cell.clone()); + let accepted_reader = cell.clone().read_owned().await; + let mut close = tokio::spawn(async move { controller.exit_module_host_and_join(17).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut close) + .await + .is_err()); + // A deadline on the observer did not complete or discard the owned close. + assert!(!close.is_finished()); + drop(accepted_reader); + tokio::time::timeout(Duration::from_secs(5), close) + .await + .unwrap() + .unwrap() + .unwrap(); + } + fn reducer_call_result(outcome: ReducerOutcome) -> ReducerCallResult { ReducerCallResult { outcome, diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1bfb4f7f7c7..ac9885b83a1 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -50,6 +50,9 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Bound by the host after validating this instance's module metadata. + environment_module: Option<(spacetimedb_lib::Hash, Arc)>, + environment_call_active: bool, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -236,6 +239,8 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + environment_module: None, + environment_call_active: false, in_anon_tx: false, procedure_last_tx_offset: None, } @@ -252,6 +257,7 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.environment_call_active = true; } /// Returns the name of the most recent reducer to be run in this environment, @@ -260,9 +266,29 @@ impl InstanceEnv { self.func_name.as_deref() } - /// Swap in a temporary function type, returning the previous one. - pub fn swap_func_type(&mut self, func_type: FuncCallType) -> FuncCallType { - mem::replace(&mut self.func_type, func_type) + pub(crate) fn bind_environment_module( + &mut self, + hash: spacetimedb_lib::Hash, + def: Arc, + ) { + self.environment_module = Some((hash, def)); + } + + pub(crate) fn finish_funcall(&mut self) { + self.environment_call_active = false; + } + + /// Nested host-dispatched view refreshes must use the view's namespace, + /// then restore the enclosing procedure's namespace and dependency tracking. + pub(crate) fn swap_func_context( + &mut self, + name: Option, + func_type: FuncCallType, + ) -> (Option, FuncCallType) { + ( + mem::replace(&mut self.func_name, name), + mem::replace(&mut self.func_type, func_type), + ) } fn get_tx(&self) -> Result + '_, GetTxError> { @@ -282,23 +308,58 @@ impl InstanceEnv { self.replica_ctx.relational_db() } - /// Dedicated read-only environment access. Missing reads also register a - /// dependency so a later insert refreshes a view that observed absence. + /// Read configuration using the schema of this exact module instance. + /// Missing optional reads also register a view dependency on `st_env`. pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { - use crate::db::environment; use spacetimedb_datastore::system_tables::ST_ENV_ID; spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; - let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if !self.environment_call_active || self.func_name.as_ref().is_none_or(|name| name.is_namespaced()) { + return Err(DBError::Other(anyhow::anyhow!( + "environment access requires a host-dispatched root module function" + )) + .into()); + } if let Ok(mut tx) = self.get_tx() { tx.record_table_scan(&self.func_type, ST_ENV_ID); - return read(&*tx); + return self.read_declared_environment(&*tx, key); } if !matches!(self.func_type, FuncCallType::Procedure) { return Err(NodesError::NotInTransaction); } - self.relational_db().with_read_only(Workload::Internal, |tx| { - environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) - }) + self.relational_db() + .with_read_only(Workload::Internal, |tx| self.read_declared_environment(tx, key)) + } + + fn read_declared_environment(&self, state: &impl StateView, key: &str) -> Result, NodesError> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let fail = |message| NodesError::from(DBError::Other(anyhow::anyhow!("{message}"))); + let (hash, module) = self + .environment_module + .as_ref() + .ok_or_else(|| fail("environment schema is not available"))?; + let declaration = module + .environment() + .get(key) + .ok_or_else(|| fail("environment key is not declared"))?; + // Check inside this same snapshot. A suspended old procedure must never + // combine its declarations with values installed for a different module. + let row = state + .iter(ST_MODULE_ID) + .map_err(DBError::from)? + .next() + .ok_or_else(|| fail("database program is not initialized"))?; + let current_hash = spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash) + .map_err(DBError::from)?; + if current_hash != *hash { + return Err(fail("module was replaced while this function was running")); + } + let value = crate::db::environment::get(state, key).map_err(|error| DBError::Other(error.into()))?; + if value.is_none() && !declaration.optional { + return Err(fail( + "required environment value is missing from the published configuration", + )); + } + Ok(value) } pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { @@ -1487,36 +1548,148 @@ mod test { Ok(db) } + fn bind_test_environment(env: &mut InstanceEnv) -> Result { + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_environment( + [("A", false), ("MISSING", true)] + .into_iter() + .map(|(name, optional)| EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional, + }) + .collect(), + ); + let module: spacetimedb_schema::def::ModuleDef = builder.finish().try_into()?; + let program = spacetimedb_datastore::traits::Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"environment-unit-test".as_slice(), + ); + env.bind_environment_module(program.hash, Arc::new(module)); + Ok(program) + } + #[test] fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { use crate::db::environment; use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; use spacetimedb_primitives::ViewId; + use std::collections::BTreeMap; let db = relational_db()?; let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("root".into())?), + Timestamp::now(), + FuncCallType::Reducer, + ); assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); - env.func_type = FuncCallType::Procedure; - assert_eq!(env.env_get("A")?, None); let mut tx = begin_mut_tx(&db); - environment::set(&db, &mut tx, "A", "uncommitted")?; + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into())]), + )?; env.tx.set_raw(tx); assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + assert!(env.env_get("UNDECLARED").is_err()); let view = ViewCallInfo::anonymous(ViewId(88)); - env.func_type = FuncCallType::View(view.clone()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("view".into())?), + Timestamp::now(), + FuncCallType::View(view.clone()), + ); assert_eq!(env.env_get("MISSING")?, None); let tx = env.tx.take()?; db.commit_tx(tx)?; let mut tx = begin_mut_tx(&db); - environment::set(&db, &mut tx, "MISSING", "")?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into()), ("MISSING".into(), "".into())]), + )?; assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); let (_, metrics, reducer) = db.rollback_mut_tx(tx); db.report_mut_tx_metrics(reducer, metrics, None); - env.func_type = FuncCallType::Procedure; + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); assert_eq!(env.env_get("MISSING")?, None); assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); Ok(()) } + #[test] + fn environment_rejects_submodules_finished_calls_and_replaced_programs() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::traits::Program; + use std::collections::BTreeMap; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + let mut tx = begin_mut_tx(&db); + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "old-value".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + let previous = env.swap_func_context( + Some(NamespacedIdentifier::from_segments(vec![ + spacetimedb_schema::identifier::Identifier::new("child".into())?, + spacetimedb_schema::identifier::Identifier::new("view".into())?, + ])), + FuncCallType::Procedure, + ); + assert!(env.env_get("A").is_err()); + env.swap_func_context(previous.0, previous.1); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + env.finish_funcall(); + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + let mut tx = begin_mut_tx(&db); + let newer = Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"new-code".as_slice(), + ); + db.update_program(&mut tx, newer)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "new-secret".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_mutable_tx()?; + assert!(env.env_get("A").is_err()); + let tx = env.take_mutable_tx_for_commit()?; + env.rollback_procedure_tx(tx); + Ok(()) + } + #[test] fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { use spacetimedb_datastore::system_tables::ST_ENV_ID; diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index f28a515c910..d0a1f23c560 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -26,8 +26,8 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, - HostController, HostRuntimeConfig, MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, - ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, + HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, + ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 25eb09e6382..685a6046e13 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -608,15 +608,23 @@ pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> (anyhow::Result, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + program, + environment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> anyhow::Result<(InitDatabaseResult, bool)> { log::debug!("init database"); @@ -674,6 +682,7 @@ fn init_database_inner( .with_context(|| format!("failed to create row-level security for table `{table_id}`: `{sql}`",))?; } + crate::db::environment::replace(stdb, tx, module_def.environment(), &environment)?; stdb.set_initialized(tx, program)?; anyhow::Ok(()) @@ -1621,6 +1630,18 @@ impl fmt::Debug for ViewCallResult { } impl ViewCallResult { + fn into_materialized_tx(self, db: &RelationalDB, trapped: bool) -> Result { + let error = match self.outcome { + ViewOutcome::Success if !trapped => return Ok(self.tx), + ViewOutcome::Success => "View instance trapped during materialization".to_owned(), + ViewOutcome::Failed(error) => error, + ViewOutcome::BudgetExceeded => "View terminated due to insufficient budget".to_owned(), + }; + let (_, metrics, reducer) = db.rollback_mut_tx(self.tx); + db.report_mut_tx_metrics(reducer, metrics, None); + Err(ViewCallError::InternalError(error)) + } + pub fn default(tx: MutTxId) -> Self { Self { outcome: ViewOutcome::Success, @@ -1700,6 +1721,9 @@ pub enum ClientConnectedError { pub struct RefInstance<'a, I: WasmInstance> { pub common: &'a mut InstanceCommon, pub instance: &'a mut I, + // Invocation-local disposal state survives errors propagated through SQL or + // subscription helpers, whose Result error does not carry a success tuple. + pub(crate) trapped: bool, } macro_rules! call_view_command_method { @@ -2937,17 +2961,26 @@ impl ModuleHost { /// Passing [`Workload::Sql`] will update the instance's last-used timestamp. /// Passing [`Workload::Subscribe`] will also increment the subscriber's refcount. pub fn materialize_views( - mut tx: MutTxId, + tx: MutTxId, instance: &mut RefInstance<'_, I>, view_collector: &impl CollectViews, caller: Identity, workload: Workload, ) -> Result<(MutTxId, bool), ViewCallError> { use FunctionArgs::*; + let db = instance.instance.replica_ctx().relational_db().clone(); + // Keep all earlier view materializations and subscription refcounts in + // the same rollback boundary if any later view fails. + let mut tx = scopeguard::guard(Some(tx), |tx| { + if let Some(tx) = tx { + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + } + }); let mut view_ids = HashSet::new(); view_collector.collect_views(&mut view_ids); for view_id in view_ids { - let st_view_row = tx.lookup_st_view(view_id)?; + let st_view_row = tx.as_ref().unwrap().lookup_st_view(view_id)?; let view_name: NamespacedIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; @@ -2959,25 +2992,30 @@ impl ModuleHost { }; let view_call = ViewCallInfo::from_args(view_id, args); let sender = args.sender(); - let is_materialized = tx.is_view_materialized(&view_call)?; + let is_materialized = tx.as_ref().unwrap().is_view_materialized(&view_call)?; if !is_materialized { - let (res, trapped) = - Self::call_view(instance, tx, &view_name, view_id, table_id, Nullary, caller, sender)?; - tx = res.tx; - if trapped { - return Ok((tx, true)); - } + let (res, trapped) = Self::call_view( + instance, + tx.take().unwrap(), + &view_name, + view_id, + table_id, + Nullary, + caller, + sender, + )?; + *tx = Some(res.into_materialized_tx(&db, trapped)?); } - // If this is a sql call, we only update this view's "last called" timestamp + let tx = tx.as_mut().unwrap(); + // These changes commit only after every requested view succeeds. if let Workload::Sql = workload { tx.update_view_timestamp(view_call.clone(), args)?; } - // If this is a subscribe call, we also increment this view's subscriber count if let Workload::Subscribe = workload { tx.subscribe_view(view_call, args, caller)?; } } - Ok((tx, false)) + Ok((ScopeGuard::into_inner(tx).unwrap(), false)) } /// Refreshes every view made stale by `tx`. @@ -3124,6 +3162,11 @@ impl ModuleHost { sender: Option, timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { + let db = instance.instance.replica_ctx().relational_db().clone(); + let tx = scopeguard::guard(tx, |tx| { + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + }); let module_def = &instance.common.info().module_def; let (global_fn_ptr, view_def, owning_def) = module_def .view_by_name_with_global_fn_ptr(view_name) @@ -3135,7 +3178,7 @@ impl ModuleHost { Ok(Self::call_view_inner( instance, - tx, + ScopeGuard::into_inner(tx), view_name, view_id, table_id, @@ -3177,16 +3220,26 @@ impl ModuleHost { view_typespace, }; - instance.common.call_view_with_tx(tx, params, instance.instance) + let (result, trapped) = instance.common.call_view_with_tx(tx, params, instance.instance); + instance.trapped |= trapped; + (result, trapped) } pub async fn init_database(&self, program: Program) -> Result { + self.init_database_with_environment(program, Default::default()).await + } + + pub async fn init_database_with_environment( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> Result { call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, environment), + |(program, environment), inst| inst.init_database(program, environment), + |(program, environment), inst| inst.init_database(program, environment).await, )? .map_err(InitDatabaseError::Other) } @@ -3196,13 +3249,24 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_environment(program, old_module_info, policy, Default::default()) + .await + } + + pub async fn update_database_with_environment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, environment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } @@ -3660,6 +3724,69 @@ mod tests { use spacetimedb_sats::product; use std::sync::Arc; + #[test] + fn failed_view_materialization_rolls_back_prior_rows_and_subscriber_counts() -> anyhow::Result<()> { + use super::{ViewCallResult, ViewOutcome}; + use crate::db::relational_db::tests_utils::begin_mut_tx; + use spacetimedb_datastore::locking_tx_datastore::{ViewCallInfo, ViewInstanceArgs}; + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; + use spacetimedb_lib::ProductType; + use spacetimedb_schema::def::ModuleDef; + + let db = TestDB::in_memory()?; + let mut builder = RawModuleDefV10Builder::new(); + let row = builder.add_algebraic_type( + [], + "Row", + AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::U8)])), + true, + ); + builder.add_view( + "earlier", + 0, + true, + true, + ProductType::unit(), + AlgebraicType::array(row.into()), + ); + let module: ModuleDef = builder.finish().try_into()?; + let mut tx = begin_mut_tx(&db); + let (view_id, table_id) = db.create_view(&mut tx, &module, module.view("earlier").unwrap())?; + db.commit_tx(tx)?; + let call = ViewCallInfo::anonymous(view_id); + + for (outcome, trapped) in [ + (ViewOutcome::Failed("denied".into()), false), + (ViewOutcome::Failed("trap".into()), true), + (ViewOutcome::BudgetExceeded, true), + (ViewOutcome::Success, true), + ] { + let mut tx = begin_mut_tx(&db); + // A preceding view succeeded in this same multi-view request. + db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; + tx.subscribe_view(call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; + assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); + let mut result = ViewCallResult::default(tx); + result.outcome = outcome; + assert!(result.into_materialized_tx(&db, trapped).is_err()); + + let tx = begin_mut_tx(&db); + assert!(tx.active_subscribers_for_view(view_id).is_empty()); + assert!(!tx.is_view_materialized(&call)?); + assert_eq!(db.iter_mut(&tx, table_id)?.count(), 0); + let _ = db.rollback_mut_tx(tx); + } + // Success retains the owned transaction for the normal commit path. + let mut tx = begin_mut_tx(&db); + db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; + tx.subscribe_view(call, ViewInstanceArgs::Anonymous, Identity::ZERO)?; + db.commit_tx(ViewCallResult::default(tx).into_materialized_tx(&db, false)?)?; + let tx = begin_mut_tx(&db); + assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + fn v2_client_config() -> ClientConfig { ClientConfig { protocol: Protocol::Binary, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 6f357962bbb..90a83691fd8 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -400,6 +400,7 @@ impl JsInstanceEnv { /// This resets all of the state associated to a single function call, /// and returns instrumentation records. fn finish_funcall(&mut self) -> ExecutionTimings { + self.instance_env.finish_funcall(); let total_duration = self.reducer_start().elapsed(); let func_name = self.log_record_function().unwrap_or("").to_owned(); @@ -424,7 +425,9 @@ impl JsInstanceEnv { } } - fn set_module_def(&mut self, module_def: Arc) { + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def); } @@ -475,11 +478,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + environment, }) .await } @@ -535,8 +540,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.request(InitDatabaseRequest { program, environment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -620,6 +629,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, } => "update_database", anyhow::Result, UpdateDatabase } @@ -662,6 +672,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + environment: std::collections::BTreeMap, } => "init_database", anyhow::Result, InitDatabase } @@ -804,6 +815,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -864,6 +876,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>, program: Program, + environment: std::collections::BTreeMap, }, } @@ -1398,8 +1411,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + environment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, environment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1510,16 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + environment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, bool) = + init_database(replica_ctx, &info.module_def, program, environment, call_reducer); + (res, trapped) + }), } } @@ -1668,7 +1684,10 @@ where return; } Ok(Ok((crf, module_common))) => { - env_on_isolate_unwrap(scope).set_module_def(module_common.info().module_def.clone()); + env_on_isolate_unwrap(scope).set_module_def( + module_common.info().module_def.clone(), + module_common.info().module_hash, + ); if let Some(result_tx) = startup_result_tx.take() && result_tx.send(Ok(module_common.clone())).is_err() @@ -1872,8 +1891,8 @@ impl WasmInstance for V8Instance<'_, '_, '_> { self.scope.get_slot::().unwrap().instance_env.tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - env_on_isolate_unwrap(self.scope).set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + env_on_isolate_unwrap(self.scope).set_module_def(module_def, module_hash); } fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult { diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 52b78463def..e00cf40d1f0 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -865,9 +865,9 @@ fn call_view( fn_ptr: ViewFnPtr, sender: Option, ) -> SysCallResult { - let prev_func_type = get_env(scope)? + let (prev_func_name, prev_func_type) = get_env(scope)? .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let result = { let args = crate::host::ArgsTuple::nullary(); @@ -900,7 +900,9 @@ fn call_view( } }; - get_env(scope)?.instance_env.swap_func_type(prev_func_type); + get_env(scope)? + .instance_env + .swap_func_context(prev_func_name, prev_func_type); result.map_err(|err| match err { ErrorOrException::Err(err) => TypeError(format!( diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index aee984fe3a5..2733d4cefe3 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,6 +37,7 @@ use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo, ViewInstanceArgs}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::ExecutionParams; @@ -85,7 +86,7 @@ pub trait WasmInstance { fn tx_slot(&self) -> TxSlot; - fn set_module_def(&mut self, module_def: Arc); + fn set_module_def(&mut self, module_def: Arc, module_hash: Hash); fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult; @@ -190,6 +191,15 @@ pub(crate) fn run_query_for_view( // Validate shape and disallow views-on-views. for plan in &plans { + // This SQL originates in module code, not an authenticated external + // query. Check every source, including non-returned join inputs, before + // any plan executes. The checked env accessor is the only module path. + ensure!( + !plan + .table_ids() + .any(spacetimedb_datastore::system_tables::is_module_restricted_table), + "module SQL views cannot read a module-restricted table" + ); let Some(source_schema) = plan.return_table() else { bail!("query does not return plain table rows"); }; @@ -431,7 +441,7 @@ impl WasmModuleHostActor { impl WasmModuleHostActor { fn make_from_instance(&self, mut instance: T::Instance) -> WasmModuleInstance { let common = InstanceCommon::new(&self.common); - instance.set_module_def(common.info().module_def.clone()); + instance.set_module_def(common.info().module_def.clone(), common.info().module_hash); WasmModuleInstance { instance, common, @@ -489,9 +499,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, environment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -545,11 +556,15 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result { + pub fn init_database( + &mut self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { let module_def = &self.common.info.clone().module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx_offset(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database(replica_ctx, module_def, program, environment, call_reducer); self.trapped = trapped; res } @@ -621,6 +636,42 @@ impl WasmModuleInstance { } } +/// Client disconnection and materialized-view refresh are independent effects. +/// A breaking migration does not remove surviving cached view instances. +struct UpdateEffects { + refresh_views: bool, + disconnect_clients: bool, +} + +impl UpdateEffects { + fn after_migration(result: crate::db::update::UpdateResult, tx: &MutTxId) -> Self { + use crate::db::update::UpdateResult; + Self { + refresh_views: matches!(result, UpdateResult::EvaluateSubscribedViews) + || tx.views_for_refresh().next().is_some(), + disconnect_clients: matches!(result, UpdateResult::RequiresClientDisconnect), + } + } + + fn committed( + &self, + tx_offset: TransactionOffset, + durable_offset: Option, + ) -> UpdateDatabaseResult { + if self.disconnect_clients { + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + } else { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + } + } +} + pub struct InstanceCommon { info: Arc, energy_monitor: Arc, @@ -649,6 +700,7 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); @@ -674,7 +726,22 @@ impl InstanceCommon { let program_hash = program.hash; let host_type = HostType::from(program.kind); let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let row = tx + .iter(ST_MODULE_ID)? + .next() + .context("database program is not initialized")?; + let current_hash = + spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash)?; + anyhow::ensure!( + current_hash == old_module_info.module_hash, + "database program changed before publication" + ); + crate::db::environment::replace(stdb, tx, self.info.module_def.environment(), &environment)?; + stdb.update_program(tx, program)?; + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -717,45 +784,31 @@ impl InstanceCommon { }; let durable_offset = stdb.durable_tx_offset(); - let res: UpdateDatabaseResult = match res { - crate::db::update::UpdateResult::Success => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - crate::db::update::UpdateResult::EvaluateSubscribedViews => { - let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; - tx = out.tx; - if trapped || out.outcome != ViewOutcome::Success { - let msg = match trapped { - true => "Trapped while evaluating views during database update".to_string(), - false => format!( - "Views evaluation did not complete successfully during database update: {:?}", - out.outcome - ), - }; - - let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); - stdb.report_mut_tx_metrics(reducer, tx_metrics, None); - UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) - } else { - let tx_offset = - succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - } - crate::db::update::UpdateResult::RequiresClientDisconnect => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { - tx_offset, - durable_offset, - } + let effects = UpdateEffects::after_migration(res, &tx); + let res = if effects.refresh_views { + // Resolve surviving materializations through the new module, + // even when this migration also requires client disconnection. + let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; + tx = out.tx; + if trapped || out.outcome != ViewOutcome::Success { + let msg = match trapped { + true => "Trapped while evaluating views during database update".to_string(), + false => format!( + "Views evaluation did not complete successfully during database update: {:?}", + out.outcome + ), + }; + + let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, tx_metrics, None); + UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) + } else { + let tx_offset = succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); + effects.committed(tx_offset, durable_offset) } + } else { + let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); + effects.committed(tx_offset, durable_offset) }; Ok(res) @@ -778,32 +831,22 @@ impl InstanceCommon { params: CallProcedureParams, inst: &mut I, ) -> (CallProcedureReturn, bool) { + // Resolve authority and type refs from the same canonical flattened ID. + // A ProcedureDef's local name alone does not identify its host scope. + let (op, procedure_def, owning_def) = + ProcedureOp::for_module(&self.info.module_def, ¶ms).expect("validated procedure id should resolve"); + let procedure_name = op.name.clone(); let CallProcedureParams { timestamp, caller_identity, - caller_connection_id, timer, - procedure_id, - args, + .. } = params; - // We've already validated by this point that the procedure exists, - // so it's fine to use the panicking `procedure_by_id`. - let procedure_def = self.info.module_def.procedure_by_id(procedure_id); - let procedure_name = &procedure_def.name; - // TODO(observability): Add tracing spans, energy, metrics? // These will require further thinking once we implement procedure suspend/resume, // and so are not worth doing yet. - let op = ProcedureOp { - id: procedure_id, - name: procedure_name.clone().into(), - caller_identity, - caller_connection_id, - timestamp, - arg_bytes: args.get_bsatn().clone(), - }; let energy_fingerprint = FunctionFingerprint { module_hash: self.info.module_hash, module_identity: self.info.owner_identity, @@ -833,7 +876,7 @@ impl InstanceCommon { WORKER_METRICS .wasm_instance_errors - .with_label_values(&self.info.database_identity, &self.info.module_hash, procedure_name) + .with_label_values(&self.info.database_identity, &self.info.module_hash, &procedure_name) .inc(); // TODO(procedure-energy): @@ -846,7 +889,7 @@ impl InstanceCommon { } Ok(return_val) => { let return_type = &procedure_def.return_type; - let seed = spacetimedb_sats::WithTypespace::new(self.info.module_def.typespace(), return_type); + let seed = spacetimedb_sats::WithTypespace::new(owning_def.typespace(), return_type); seed.deserialize(bsatn::Deserializer::new(&mut &return_val[..])) .map_err(|err| ProcedureCallError::InternalError(format!("{err}"))) .map(|return_val| ProcedureCallResult { @@ -1167,6 +1210,7 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, + trapped: false, }; let (res, trapped) = match cmds { ViewCommand::AddSingleSubscription { @@ -1253,7 +1297,7 @@ impl InstanceCommon { if let Err(err) = &res { error_target.send(&info.subscriptions, err); } - (res, trapped) + (res, trapped || inst.trapped) } pub(in crate::host) fn handle_sql_cmd( @@ -1264,6 +1308,7 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, + trapped: false, }; let SqlCommand { db, @@ -1280,9 +1325,9 @@ impl InstanceCommon { result: Ok(result), head, }, - trapped, + trapped || inst.trapped, ), - Err(err) => (SqlCommandResult { result: Err(err), head }, false), + Err(err) => (SqlCommandResult { result: Err(err), head }, inst.trapped), } } @@ -1449,6 +1494,7 @@ impl InstanceCommon { let mut instance = RefInstance { common: self, instance: inst, + trapped: false, }; ModuleHost::call_views_with_tx_at(tx, &mut instance, caller, timestamp) } @@ -1962,6 +2008,27 @@ pub struct ProcedureOp { pub arg_bytes: Bytes, } +impl ProcedureOp { + fn for_module<'a>( + module: &'a ModuleDef, + params: &CallProcedureParams, + ) -> Option<(Self, &'a spacetimedb_schema::def::ProcedureDef, &'a ModuleDef)> { + let (name, def, owning) = module.get_procedure_by_id_with_module(params.procedure_id)?; + Some(( + Self { + id: params.procedure_id, + name, + caller_identity: params.caller_identity, + caller_connection_id: params.caller_connection_id, + timestamp: params.timestamp, + arg_bytes: params.args.get_bsatn().clone(), + }, + def, + owning, + )) + } +} + impl InstanceOp for ProcedureOp { fn name(&self) -> &NamespacedIdentifier { &self.name @@ -2006,6 +2073,242 @@ mod tests { use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; + #[test] + fn breaking_migration_preserves_disconnect_and_refreshes_surviving_environment_view() -> anyhow::Result<()> { + use super::UpdateEffects; + use crate::db::{environment, update}; + use crate::host::UpdateDatabaseResult; + use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_datastore::locking_tx_datastore::FuncCallType; + use spacetimedb_datastore::system_tables::{ModuleKind, ST_ENV_ID}; + use spacetimedb_datastore::traits::Program; + use spacetimedb_lib::db::raw_def::{ + v10::{RawModuleDefV10Builder, RawModuleDefV10Section}, + v9::TableAccess, + }; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + use spacetimedb_lib::identity::AuthCtx; + use spacetimedb_schema::auto_migrate::ponder_migrate; + use std::collections::BTreeMap; + + struct TestLogger; + impl update::UpdateLogger for TestLogger { + fn info(&self, _: &str) {} + } + fn module(include_obsolete_table: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let row = builder.add_algebraic_type( + [], + "EnvironmentViewRow", + AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::String)])), + true, + ); + builder.add_view( + "environment_view", + 0, + true, + true, + ProductType::unit(), + AlgebraicType::array(AlgebraicType::Ref(row)), + ); + if include_obsolete_table { + builder + .build_table_with_new_type("obsolete", ProductType::from_iter([("id", AlgebraicType::U64)]), true) + .with_access(TableAccess::Public) + .finish(); + } + let mut raw = builder.finish(); + raw.sections + .push(RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])); + raw.try_into().expect("valid ENV view module") + } + + let db = TestDB::in_memory()?; + let old = module(true); + let new = module(false); + let before = BTreeMap::from([("TOKEN".into(), "before".into())]); + let after = BTreeMap::from([("TOKEN".into(), "after".into())]); + let mut tx = begin_mut_tx(&db); + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"old-program".as_slice()), + )?; + for table in old.tables() { + update::create_table_from_def(&db, &mut tx, &old, table)?; + } + let (view_id, _) = db.create_view(&mut tx, &old, old.view("environment_view").unwrap())?; + let call = ViewCallInfo::anonymous(view_id); + // Ordinary SQL materialization has no live subscriber to disconnect. + tx.update_view_timestamp(call.clone(), ViewInstanceArgs::Anonymous)?; + tx.record_table_scan(&FuncCallType::View(call.clone()), ST_ENV_ID); + environment::replace(&db, &mut tx, old.environment(), &before)?; + db.commit_tx(tx)?; + + let mut tx = begin_mut_tx(&db); + environment::replace(&db, &mut tx, new.environment(), &after)?; + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"new-program".as_slice()), + )?; + let plan = ponder_migrate(&old, &new)?; + assert!( + plan.breaks_client(), + "removing the unrelated table must require disconnection" + ); + let result = update::update_database(&db, &mut tx, AuthCtx::for_testing(), plan, &TestLogger)?; + assert!(matches!(result, update::UpdateResult::RequiresClientDisconnect)); + assert!(tx.views_for_refresh().any(|dirty| *dirty == call)); + let effects = UpdateEffects::after_migration(result, &tx); + assert!(effects.refresh_views); + assert!(effects.disconnect_clients); + let calls = collect_subscribed_view_calls(&tx, &new, Identity::ZERO)?; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].view_id, view_id); + assert_eq!(&*calls[0].view_name, "environment_view"); + assert!(tx.active_subscribers_for_view(view_id).is_empty()); + let (_send, receive) = tokio::sync::oneshot::channel(); + assert!(matches!( + effects.committed(receive, None), + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + )); + + // The failed-view branch can roll back the same complete migration tx. + let _ = db.rollback_mut_tx(tx); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!(environment::snapshot(tx).unwrap(), before); + }); + let tx = begin_mut_tx(&db); + assert!(db.table_id_from_name_mut(&tx, "obsolete")?.is_some()); + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn module_sql_views_cannot_read_environment_directly_or_through_a_join() -> anyhow::Result<()> { + use super::run_query_for_view; + use crate::db::environment; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use spacetimedb_primitives::ViewId; + use spacetimedb_sats::product; + use std::collections::BTreeMap; + + let db = TestDB::in_memory()?; + let visible = db.create_table_for_test( + "visible", + &[("key", AlgebraicType::String), ("value", AlgebraicType::String)], + &[0.into()], + )?; + let mut tx = begin_mut_tx(&db); + tx.insert_via_serialize_bsatn(visible, &product!("TOKEN", "ordinary-value"))?; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("TOKEN".into(), "private-value".into())]), + )?; + let row_type = ProductType::from_iter([("key", AlgebraicType::String), ("value", AlgebraicType::String)]); + let call = ViewCallInfo::anonymous(ViewId(99)); + let run = |tx: &mut _, query| run_query_for_view(tx, query, &row_type, &call, db.database_identity()); + assert_eq!( + run(&mut tx, "SELECT * FROM visible")?, + vec![product!("TOKEN", "ordinary-value")] + ); + for query in [ + "SELECT * FROM st_env", + "SELECT e.* FROM st_env AS e", + "SELECT v.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + "SELECT e.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + ] { + let error = run(&mut tx, query).expect_err("module SQL must use the checked environment accessor"); + assert!( + error.to_string().contains("module-restricted table"), + "unexpected query failure for {query}: {error:#}" + ); + assert!(!error.to_string().contains("private-value")); + } + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn procedure_operations_resolve_root_and_nested_host_scope_and_typespace() { + use super::{CallProcedureParams, InstanceOp, ProcedureOp}; + use crate::host::ArgsTuple; + use spacetimedb_lib::db::raw_def::v10::{ + RawModuleDefV10, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10, + }; + use spacetimedb_lib::de::DeserializeSeed; + use spacetimedb_lib::Timestamp; + use spacetimedb_primitives::ProcedureId; + use spacetimedb_sats::{bsatn, AlgebraicValue, ProductValue, WithTypespace}; + + fn module(value_type: AlgebraicType) -> RawModuleDefV10 { + let mut builder = RawModuleDefV10Builder::new(); + let result_type = builder.add_algebraic_type( + [], + "ResultRow", + AlgebraicType::Product(ProductType::from_iter([("value", value_type)])), + true, + ); + builder.add_procedure("read_env", ProductType::unit(), AlgebraicType::Ref(result_type)); + builder.finish() + } + + let mut child = module(AlgebraicType::String); + child + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "nested".into(), + module: module(AlgebraicType::Bool), + }])); + let mut root = module(AlgebraicType::U64); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".into(), + module: child, + }])); + let module: ModuleDef = root.try_into().expect("valid nested module"); + for (index, expected_name) in ["read_env", "lib.read_env", "lib.nested.read_env"] + .into_iter() + .enumerate() + { + let params = CallProcedureParams::from_system( + Timestamp::UNIX_EPOCH, + Identity::ZERO, + ProcedureId::from(index), + ArgsTuple::nullary(), + ); + let (op, def, owning) = ProcedureOp::for_module(&module, ¶ms).unwrap(); + assert_eq!(&**op.name(), expected_name); + assert_eq!(op.name().is_namespaced(), index != 0); + assert_eq!(op.id, params.procedure_id); + assert_eq!(&*def.name, "read_env", "declaration names remain local"); + if index == 2 { + let expected = AlgebraicValue::Product(ProductValue::from_iter([AlgebraicValue::Bool(true)])); + let bytes = bsatn::to_vec(&expected).unwrap(); + let decoded = WithTypespace::new(owning.typespace(), &def.return_type) + .deserialize(bsatn::Deserializer::new(&mut &bytes[..])) + .unwrap(); + assert_eq!( + decoded, expected, + "nested type refs must not use the root U64 typespace" + ); + } + } + assert!(module + .get_procedure_by_id_with_module(ProcedureId::from(3usize)) + .is_none()); + } + fn module_def_for_view(name: &str, is_anonymous: bool) -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); let name = RawIdentifier::new(name); diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 23d33983440..0c22ee54694 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -298,7 +298,9 @@ impl WasmInstanceEnv { self.mem = Some(mem); } - pub fn set_module_def(&mut self, module_def: Arc) { + pub fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def) } @@ -379,6 +381,7 @@ impl WasmInstanceEnv { /// /// This resets the call times and clears the arguments source and error sink. pub fn finish_funcall(&mut self, result_sink: u32) -> (ExecutionTimings, Vec) { + self.instance_env.finish_funcall(); // For the moment, // we only explicitly clear the source/sink buffers and the "syscall" times. // TODO: should we be clearing `iters` and/or `timing_spans`? @@ -1889,10 +1892,10 @@ impl WasmInstanceEnv { fn_ptr: ViewFnPtr, sender: Option, ) -> anyhow::Result { - let prev_func_type = caller + let (prev_func_name, prev_func_type) = caller .data_mut() .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let mut nested_result_sink = None; let call_result = (|| -> anyhow::Result { @@ -1924,7 +1927,10 @@ impl WasmInstanceEnv { Ok(code) })(); - caller.data_mut().instance_env.swap_func_type(prev_func_type); + caller + .data_mut() + .instance_env + .swap_func_context(prev_func_name, prev_func_type); let result_bytes = { let env = caller.data_mut(); diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index a5316ceb95c..aa589df3b5a 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -625,8 +625,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { self.store.data().instance_env().tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - self.store.data_mut().set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.store.data_mut().set_module_def(module_def, module_hash); } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index e927921c886..1f2c587b762 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -91,7 +91,7 @@ fn run_inner( let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; // Check mutation authority while the automatic rollback guard owns // the transaction, including rejected administrative statements. - if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + if matches!(&stmt, Statement::DML(_)) && !auth.has_write_access() { return Err(anyhow!( "Caller {} is not authorized to run SQL mutations", auth.caller() @@ -101,7 +101,7 @@ fn run_inner( && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID { return Err(anyhow!( - "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + "Database environment variables can only be changed by publishing" )); } Ok(stmt) @@ -157,19 +157,9 @@ fn run_inner( trapped, )) } - stmt @ (Statement::DML(_) | Statement::Environment(_)) => { - // Evaluate the mutation + Statement::DML(stmt) => { let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { - match stmt { - Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, - Statement::Environment(environment) => match environment.value { - Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, - None => { - crate::db::environment::delete(&db, tx, &environment.key)?; - } - }, - Statement::Select(_) => unreachable!(), - } + execute_dml_stmt(&auth, stmt, tx, &mut metrics)?; Ok(()) })?; @@ -267,8 +257,10 @@ pub(crate) mod tests { use spacetimedb_schema::table_name::TableName; #[test] - fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + fn environment_sql_is_read_only_including_for_owner() { + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; use spacetimedb_lib::identity::SqlPermission; + use std::collections::BTreeMap; let db = TestDB::in_memory().unwrap(); let runtime = tokio::runtime::Runtime::new().unwrap(); let owner = AuthCtx::for_current(Identity::ZERO); @@ -277,63 +269,48 @@ pub(crate) mod tests { Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), ); let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let value = "secret-marker"; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + crate::db::environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), value.into())])) + }) + .unwrap(); let execute = |statement: &str, auth: AuthCtx| { runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) }; - let value = "it's \\quoted;\nUTF-8 é\0tail"; - execute( - &format!( - "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", - value.replace('\'', "''") - ), - owner.clone(), - ) - .unwrap(); - execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); - let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) - .unwrap() - .rows; - assert_eq!(rows, vec![product![value]]); - assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); - for auth in [viewer, outsider] { - assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); - assert!(execute("DELETE env.Mixed_Key", auth).is_err()); - } - for statement in [ - "SET env.EMPTY = 5".to_string(), - "SET env.\"BAD-KEY\" = 'value'".to_string(), - format!("SET env.EMPTY = '{}'", "x".repeat(8193)), - "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), - "DELETE env.Mixed_Key WHERE true".to_string(), - "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), - "UPDATE st_env SET value = 'bypass'".to_string(), - "DELETE FROM st_env".to_string(), - ] { - assert!( - execute(&statement, owner.clone()).is_err(), - "unexpectedly accepted {statement}" - ); - } - // Every rejected path released its transaction and preserved old data. assert_eq!( - execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", viewer.clone()) .unwrap() .rows, - vec![product![""]] + vec![product![value]] ); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [owner.clone(), viewer, outsider] { + for statement in [ + "SET env.TOKEN = 'forbidden'", + "DELETE env.TOKEN", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!( + execute(statement, auth.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + } + // Rejected writes must release their transactions and preserve data. assert_eq!( - execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", owner) .unwrap() .rows, vec![product![value]] ); - execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); - execute("DELETE env.EMPTY", owner.clone()).unwrap(); - execute("DELETE env.EMPTY", owner.clone()).unwrap(); - assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) - .unwrap() - .rows - .is_empty()); } /// Short-cut for simplify test execution diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index d61ade64f54..1e7315cb4eb 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,10 +132,6 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { - #[error(transparent)] - Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), - #[error("environment values must be SQL string literals")] - EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index 3293553f1b6..9fbdb869a20 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,12 +31,6 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), - Environment(EnvironmentWrite), -} - -pub struct EnvironmentWrite { - pub key: Box, - pub value: Option>, } pub enum DML { @@ -460,21 +454,6 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, _auth: &AuthCtx) -> T SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), - SqlAst::Environment(environment) => { - // Resolve through the normal private-table visibility check. - tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; - let key = &*environment.key.0; - spacetimedb_lib::environment::validate_key(key)?; - let value = match environment.value { - None => None, - Some(SqlLiteral::Str(value)) => { - spacetimedb_lib::environment::validate_value(&value)?; - Some(value) - } - Some(_) => return Err(TypingError::EnvironmentValueType), - }; - Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) - } } } diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index b21d8470b84..9aec59ffaa8 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -101,6 +101,9 @@ pub enum RawModuleDefV10Section { /// Submodules, keyed by the namespace they are registered under. Submodules(Vec), + + /// Declared publish-only configuration. Even an empty section requires ENV support. + Environment(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -715,6 +718,27 @@ impl RawModuleDefV10Builder { Default::default() } + /// Declare a complete environment schema, including an explicit empty schema. + /// Repeated calls remain repeated sections so host validation rejects ambiguity. + pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { + self.module + .sections + .push(RawModuleDefV10Section::Environment(declarations)); + self + } + + /// New ENV-aware bindings declare an empty schema when no declaration is registered. + pub fn ensure_environment(&mut self) { + if !self + .module + .sections + .iter() + .any(|section| matches!(section, RawModuleDefV10Section::Environment(_))) + { + self.add_environment(Vec::new()); + } + } + /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { let idx = self diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs index 17b5c3f3293..8b6f6db38d8 100644 --- a/crates/lib/src/environment.rs +++ b/crates/lib/src/environment.rs @@ -3,6 +3,9 @@ pub const MAX_ENV_KEY_BYTES: usize = 256; pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; pub const MAX_ENV_VARS: usize = 256; +/// Total key and literal bytes in a declaration schema, independent of runtime values. +pub const MAX_ENV_SCHEMA_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_ENV_UNION_ENTRIES: usize = 256; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EnvironmentValidationError { @@ -60,3 +63,321 @@ mod tests { assert!(validate_value(&"é".repeat(4097)).is_err()); } } + +/// Host-validated string constraints. Values remain strings in every module SDK. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub enum EnvironmentConstraint { + AnyString, + Literal(String), + OneOf(Vec), +} + +/// Declaration metadata, never an environment value supplied during publishing. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub struct EnvironmentDeclaration { + pub name: String, + pub constraint: EnvironmentConstraint, + pub optional: bool, +} + +/// An environment schema whose declarations have passed host validation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct EnvironmentSchema { + declarations: std::collections::BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EnvironmentSchemaErrorKind { + InvalidName, + TooManyDeclarations, + DuplicateDeclaration, + EmptyUnion, + TooManyUnionEntries, + SchemaTooLarge, + LiteralTooLarge, + Undeclared, + MissingRequired, + ValueTooLarge, + ConstraintMismatch, +} + +/// Errors identify a key and rule, and never contain a supplied or allowed value. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct EnvironmentSchemaError { + pub key: Option, + pub kind: EnvironmentSchemaErrorKind, +} + +impl std::fmt::Display for EnvironmentSchemaError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(key) = &self.key { + write!(f, "environment key {key:?}: ")?; + } + f.write_str(match self.kind { + EnvironmentSchemaErrorKind::InvalidName => "invalid name", + EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", + EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", + EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", + EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", + EnvironmentSchemaErrorKind::SchemaTooLarge => "declaration schema exceeds size limit", + EnvironmentSchemaErrorKind::LiteralTooLarge => "declared literal exceeds value size limit", + EnvironmentSchemaErrorKind::Undeclared => "key is not declared", + EnvironmentSchemaErrorKind::MissingRequired => "required value is missing", + EnvironmentSchemaErrorKind::ValueTooLarge => "value exceeds size limit", + EnvironmentSchemaErrorKind::ConstraintMismatch => "value does not satisfy its declared string constraint", + }) + } +} + +impl std::error::Error for EnvironmentSchemaError {} + +impl EnvironmentSchema { + fn validate_metadata(declarations: &[EnvironmentDeclaration]) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + if declarations.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::TooManyDeclarations, + }); + } + let mut bytes = 0usize; + for declaration in declarations { + // Never retain or format unvalidated key bytes in diagnostics. + validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind, + }; + bytes += declaration.name.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + let literals = match &declaration.constraint { + EnvironmentConstraint::AnyString => &[][..], + EnvironmentConstraint::Literal(value) => std::slice::from_ref(value), + EnvironmentConstraint::OneOf(values) => { + if values.is_empty() { + return Err(error(Kind::EmptyUnion)); + } + if values.len() > MAX_ENV_UNION_ENTRIES { + return Err(error(Kind::TooManyUnionEntries)); + } + values.as_slice() + } + }; + for value in literals { + validate_value(value).map_err(|_| error(Kind::LiteralTooLarge))?; + bytes += value.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + } + } + Ok(()) + } + + pub fn new(declarations: Vec) -> Result { + Self::validate_metadata(&declarations)?; + let mut schema = Self::default(); + for mut declaration in declarations { + if let EnvironmentConstraint::OneOf(values) = &mut declaration.constraint { + values.sort_unstable(); + values.dedup(); + } + if schema.declarations.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name), + kind: EnvironmentSchemaErrorKind::DuplicateDeclaration, + }); + } + schema.declarations.insert(declaration.name.clone(), declaration); + } + Ok(schema) + } + + /// Check bounds before cloning raw untrusted metadata into the validated schema. + pub fn from_declarations(declarations: &[EnvironmentDeclaration]) -> Result { + Self::validate_metadata(declarations)?; + Self::new(declarations.to_vec()) + } + + pub fn get(&self, name: &str) -> Option<&EnvironmentDeclaration> { + self.declarations.get(name) + } + + pub fn declarations(&self) -> impl ExactSizeIterator { + self.declarations.values() + } + + pub fn into_declarations(self) -> Vec { + self.declarations.into_values().collect() + } + + pub fn is_empty(&self) -> bool { + self.declarations.is_empty() + } + + /// Validate a complete publish input. Existing stored values are not inputs. + pub fn validate_values( + &self, + values: &std::collections::BTreeMap, + ) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + for (name, value) in values { + validate_key(name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(name.clone()), + kind, + }; + let declaration = self.get(name).ok_or_else(|| error(Kind::Undeclared))?; + validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + let matches = match &declaration.constraint { + EnvironmentConstraint::AnyString => true, + EnvironmentConstraint::Literal(expected) => value == expected, + EnvironmentConstraint::OneOf(allowed) => allowed.binary_search(value).is_ok(), + }; + if !matches { + return Err(error(Kind::ConstraintMismatch)); + } + } + for declaration in self.declarations() { + if !declaration.optional && !values.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind: Kind::MissingRequired, + }); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod schema_tests { + use super::*; + use std::collections::BTreeMap; + + fn declaration(name: &str, constraint: EnvironmentConstraint, optional: bool) -> EnvironmentDeclaration { + EnvironmentDeclaration { + name: name.into(), + constraint, + optional, + } + } + + #[test] + fn complete_inputs_preserve_optional_empty_and_exact_string_constraints() { + let schema = EnvironmentSchema::new(vec![ + declaration("REQUIRED", EnvironmentConstraint::AnyString, false), + declaration( + "MODE", + EnvironmentConstraint::OneOf(vec!["false".into(), "true".into(), "false".into()]), + false, + ), + declaration("OPTIONAL", EnvironmentConstraint::Literal("".into()), true), + ]) + .unwrap(); + let mut values = BTreeMap::from([("REQUIRED".into(), "\0雪".into()), ("MODE".into(), "false".into())]); + schema.validate_values(&values).unwrap(); + values.insert("OPTIONAL".into(), "".into()); + schema.validate_values(&values).unwrap(); + values.insert("MODE".into(), "False".into()); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::ConstraintMismatch + ); + values.remove("MODE"); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::MissingRequired + ); + values.insert("UNDECLARED".into(), "secret-marker".into()); + let error = schema.validate_values(&values).unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::Undeclared); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + + #[test] + fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { + let optional = declaration("A", EnvironmentConstraint::AnyString, true); + assert_eq!( + EnvironmentSchema::new(vec![optional.clone(), optional]) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::DuplicateDeclaration + ); + assert_eq!( + EnvironmentSchema::new( + (0..=MAX_ENV_VARS) + .map(|i| declaration(&format!("K{i}"), EnvironmentConstraint::AnyString, true)) + .collect() + ) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::TooManyDeclarations + ); + for (name, constraint, expected) in [ + ( + "A-B", + EnvironmentConstraint::AnyString, + EnvironmentSchemaErrorKind::InvalidName, + ), + ( + "A", + EnvironmentConstraint::OneOf(vec![]), + EnvironmentSchemaErrorKind::EmptyUnion, + ), + ( + "A", + EnvironmentConstraint::Literal("x".repeat(MAX_ENV_VALUE_BYTES + 1)), + EnvironmentSchemaErrorKind::LiteralTooLarge, + ), + ] { + assert_eq!( + EnvironmentSchema::new(vec![declaration(name, constraint, true)]) + .unwrap_err() + .kind, + expected + ); + } + assert!(EnvironmentSchema::default().validate_values(&BTreeMap::new()).is_ok()); + assert_eq!( + EnvironmentSchema::default() + .validate_values(&BTreeMap::from([("A".into(), "".into())])) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::Undeclared + ); + } + + #[test] + fn raw_metadata_is_bounded_before_copying_or_formatting_untrusted_keys() { + let invalid = format!("private-marker\n{}", "x".repeat(100_000)); + let error = + EnvironmentSchema::new(vec![declaration(&invalid, EnvironmentConstraint::AnyString, true)]).unwrap_err(); + assert_eq!(error.key, None); + assert!(!format!("{error:?}: {error}").contains("private-marker")); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["".into(); MAX_ENV_UNION_ENTRIES + 1]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::TooManyUnionEntries); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["x".repeat(MAX_ENV_VALUE_BYTES); MAX_ENV_UNION_ENTRIES]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::SchemaTooLarge); + } +} diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 69168032a0a..717f9d38092 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -63,7 +63,7 @@ pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Resu } match parse_and_type_sql(sql, tx, auth)? { - stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), + stmt @ Statement::DML(_) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 521bcb35aa9..2516894ceea 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -179,6 +179,9 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, + + environment: spacetimedb_lib::environment::EnvironmentSchema, + environment_declared: bool, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -190,6 +193,16 @@ pub enum RawModuleDefVersion { } impl ModuleDef { + /// The validated root environment schema. Legacy modules have an empty schema. + pub fn environment(&self) -> &spacetimedb_lib::environment::EnvironmentSchema { + &self.environment + } + + /// Whether the raw module explicitly required environment support. + pub fn environment_declared(&self) -> bool { + self.environment_declared + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -823,15 +836,27 @@ impl ModuleDef { /// Look up a procuedure by its id, returning `None` if it doesn't exist. pub fn get_procedure_by_id(&self, id: ProcedureId) -> Option<&ProcedureDef> { + self.get_procedure_by_id_with_module(id).map(|(_, def, _)| def) + } + + /// Resolve a flattened wire ID to its host-qualified name, definition and + /// owning module. Procedure definitions store local names and type refs. + pub fn get_procedure_by_id_with_module( + &self, + id: ProcedureId, + ) -> Option<(NamespacedIdentifier, &ProcedureDef, &ModuleDef)> { let idx = id.idx(); if idx < self.procedures.len() { - return self.procedures.get_index(idx).map(|(_, def)| def); + return self + .procedures + .get_index(idx) + .map(|(_, def)| (self.path.join(def.name.clone()), def, self)); } let mut offset = self.procedures.len(); for submodule in self.submodules.values() { let count = submodule.procedure_count(); if idx < offset + count { - return submodule.get_procedure_by_id(ProcedureId::from(idx - offset)); + return submodule.get_procedure_by_id_with_module(ProcedureId::from(idx - offset)); } offset += count; } @@ -986,6 +1011,8 @@ impl From for RawModuleDefV9 { http_routes: _, raw_module_def_version: _, submodules: _, + environment: _, + environment_declared: _, } = val; // Extract column defaults from tables before consuming tables @@ -1046,9 +1073,14 @@ impl From for RawModuleDefV10 { http_routes, raw_module_def_version: _, submodules, + environment, + environment_declared, } = val; let mut sections = Vec::new(); + if environment_declared { + sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); + } let mut explicit_names = ExplicitNames::default(); sections.push(RawModuleDefV10Section::Typespace(typespace)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index d6303f3a81c..f164642a615 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -78,6 +78,7 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let environment = validate_environment(&def); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -297,10 +298,13 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules) = - (tables_types_reducers_procedures_views, submodules) - .combine_errors() - .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; + let ( + (tables, types, reducers, procedures, views, (http_handlers, http_routes)), + submodules, + (environment, environment_declared), + ) = (tables_types_reducers_procedures_views, submodules, environment) + .combine_errors() + .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); @@ -322,6 +326,8 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_routes, raw_module_def_version: RawModuleDefVersion::V10, submodules, + environment, + environment_declared, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -334,6 +340,22 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } +fn validate_environment(def: &RawModuleDefV10) -> Result<(spacetimedb_lib::environment::EnvironmentSchema, bool)> { + let mut sections = def.sections.iter().filter_map(|section| match section { + RawModuleDefV10Section::Environment(declarations) => Some(declarations), + _ => None, + }); + let Some(declarations) = sections.next() else { + return Ok((Default::default(), false)); + }; + if sections.next().is_some() { + return Err(ValidationError::RepeatedEnvironmentSection.into()); + } + let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) + .map_err(|error| ValidationError::Environment { error })?; + Ok((schema, true)) +} + /// Validate that each submodule's namespace is a valid identifier of at most 63 characters, /// that no two submodules share the same namespace, and that no submodule declares lifecycle /// reducers (lifecycle reducers are only permitted in the root module). @@ -365,6 +387,11 @@ fn validate_submodules(submodules: Vec) -> Result { + if !def.environment().is_empty() { + errors.push(ValidationError::EnvironmentInSubmodule { + namespace: submodule.namespace.clone(), + }); + } for (lifecycle, opt_id) in def.lifecycle_reducers_map() { if opt_id.is_some() { errors.push(ValidationError::LifecycleInSubmodule { @@ -2796,3 +2823,78 @@ mod tests { }); } } + +#[cfg(test)] +mod environment_tests { + use super::*; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn declared(name: &str) -> RawModuleDefV10 { + RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }])], + } + } + + #[test] + fn environment_schema_round_trip_preserves_explicit_empty_and_exact_keys() { + let legacy = validate(RawModuleDefV10::default()).unwrap(); + assert!(legacy.environment().is_empty()); + assert!(!legacy.environment_declared()); + let explicit = validate(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }) + .unwrap(); + assert!(explicit.environment().is_empty()); + assert!(explicit.environment_declared()); + let raw: RawModuleDefV10 = explicit.into(); + assert!(validate(raw).unwrap().environment_declared()); + let module = validate(declared("Mixed_CASE")).unwrap(); + assert!(module.environment().get("Mixed_CASE").is_some()); + assert!(module.environment().get("mixed_case").is_none()); + let raw: RawModuleDefV10 = module.into(); + assert!(validate(raw).unwrap().environment().get("Mixed_CASE").is_some()); + assert_eq!( + spacetimedb_lib::bsatn::to_vec(&RawModuleDefV10Section::Environment(vec![])).unwrap(), + vec![15, 0, 0, 0, 0] + ); + } + + #[test] + fn environment_rejects_ambiguous_sections_and_nested_declarations() { + let mut duplicate = declared("A"); + duplicate.sections.push(RawModuleDefV10Section::Environment(vec![])); + assert!(validate(duplicate) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::RepeatedEnvironmentSection))); + let nested = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "outer".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "inner".into(), + module: declared("SECRET"), + }])], + }, + }])], + }; + assert!(validate(nested) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::EnvironmentInSubmodule { .. }))); + let empty = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "allowed".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }, + }])], + }; + assert!(validate(empty).is_ok()); + assert!(validate(declared("INVALID-NAME")).is_err()); + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 8de4927afa2..1ba7c03c7b9 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -171,6 +171,8 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), + environment: Default::default(), + environment_declared: false, }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index e9408a35482..770c44c7a3e 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("module has repeated environment sections")] + RepeatedEnvironmentSection, + #[error("invalid environment declaration: {error}")] + Environment { + error: spacetimedb_lib::environment::EnvironmentSchemaError, + }, + #[error("submodule {namespace:?} cannot declare environment variables")] + EnvironmentInSubmodule { namespace: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/smoketests/modules/Cargo.lock b/crates/smoketests/modules/Cargo.lock index 69d26e1f356..3bfe68ba8c2 100644 --- a/crates/smoketests/modules/Cargo.lock +++ b/crates/smoketests/modules/Cargo.lock @@ -756,6 +756,13 @@ dependencies = [ "spacetimedb", ] +[[package]] +name = "smoketest-module-environment-publish" +version = "0.1.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "smoketest-module-fail-initial-publish-broken" version = "0.1.0" diff --git a/crates/smoketests/modules/Cargo.toml b/crates/smoketests/modules/Cargo.toml index 63dc67687eb..5184afb1947 100644 --- a/crates/smoketests/modules/Cargo.toml +++ b/crates/smoketests/modules/Cargo.toml @@ -119,6 +119,7 @@ members = [ "new-user-flow", "module-nested-op", "noop", + "environment-publish", "fail-initial-publish-broken", "fail-initial-publish-fixed", diff --git a/crates/smoketests/modules/environment-publish/Cargo.toml b/crates/smoketests/modules/environment-publish/Cargo.toml new file mode 100644 index 00000000000..8ed5366b336 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "smoketest-module-environment-publish" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +spacetimedb.workspace = true diff --git a/crates/smoketests/modules/environment-publish/src/lib.rs b/crates/smoketests/modules/environment-publish/src/lib.rs new file mode 100644 index 00000000000..57d82e8b7d2 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/src/lib.rs @@ -0,0 +1,46 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::env] +pub struct Env { + pub SMOKE_REQUIRED: String, + #[env(values("ready", "other"))] + pub SMOKE_MODE: String, + pub SMOKE_OPTIONAL: Option, + pub SMOKE_EMPTY: Option, + pub SMOKE_NUMBER: Option, + pub SMOKE_FLAG: Option, +} + +#[spacetimedb::table(accessor = initial_environment, public)] +pub struct InitialEnvironment { + required: String, + mode: String, + optional: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.initial_environment().insert(InitialEnvironment { + required: ctx.env.SMOKE_REQUIRED(), + mode: ctx.env.SMOKE_MODE(), + optional: ctx.env.SMOKE_OPTIONAL(), + }); +} + +#[spacetimedb::reducer] +pub fn check_environment( + ctx: &ReducerContext, + required: String, + mode: String, + optional: Option, + empty: Option, + number: Option, + flag: Option, +) { + assert_eq!(ctx.env.SMOKE_REQUIRED(), required); + assert_eq!(ctx.env.SMOKE_MODE(), mode); + assert_eq!(ctx.env.SMOKE_OPTIONAL(), optional); + assert_eq!(ctx.env.SMOKE_EMPTY(), empty); + assert_eq!(ctx.env.SMOKE_NUMBER(), number); + assert_eq!(ctx.env.SMOKE_FLAG(), flag); +} diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs new file mode 100644 index 00000000000..aed684811ce --- /dev/null +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -0,0 +1,439 @@ +//! Publish-only environment configuration through the real CLI and local server. +use serde_json::{json, Value}; +use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_smoketests::{modules, random_string, Smoketest}; +use std::{ + fs, + io::{Read as _, Seek as _}, + path::PathBuf, + process::{Child, Command, Output, Stdio}, + time::{Duration, Instant}, +}; + +const KEYS: &[&str] = &[ + "SMOKE_REQUIRED", + "SMOKE_MODE", + "SMOKE_OPTIONAL", + "SMOKE_EMPTY", + "SMOKE_NUMBER", + "SMOKE_FLAG", +]; + +struct Fixture { + test: Smoketest, + database: String, + wasm: PathBuf, +} + +impl Fixture { + fn new() -> Self { + // Check before the harness can connect or copy a prior login. These are + // standalone-only tests, including when accidentally run in a remote job. + for key in [ + "SPACETIME_REMOTE_SERVER", + "SPACETIME_USE_AUTH_HOST", + "SPACETIME_SMOKETEST_BASE_CONFIG_PATH", + ] { + assert!( + std::env::var_os(key).is_none(), + "ENV smoke test requires isolated local settings ({key})" + ); + } + let test = Smoketest::builder() + .precompiled_module("environment-publish") + .autopublish(false) + .build(); + assert!(test.guard.is_some()); + let address = test + .server_url + .strip_prefix("http://") + .unwrap() + .parse::() + .unwrap(); + assert_eq!(address.ip(), std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); + assert_ne!(address.port(), 0); + let wasm = test.project_dir.path().join("published.wasm"); + fs::copy(modules::precompiled_module("environment-publish"), &wasm).unwrap(); + // A misleading local source proves --bin-path reads declarations from + // these exact bytes, rather than trusting nearby source/config metadata. + fs::create_dir(test.project_dir.path().join("src")).unwrap(); + fs::write( + test.project_dir.path().join("src/lib.rs"), + "#[spacetimedb::env] pub struct Env { pub WRONG_SOURCE_DECLARATION: String }", + ) + .unwrap(); + let fixture = Self { + test, + database: format!("environment-{}", random_string()), + wasm, + }; + fixture.success(&["login", "--server-issued-login", &fixture.test.server_url], &[]); + fixture + } + + fn command(&self, args: &[&str], shell: &[(&str, &str)]) -> Output { + let mut command = Command::new(ensure_binaries_built()); + command.env_clear(); + // Runtime executables are already built. No user credentials, remote + // settings, or ambient module variables enter these child processes. + for key in ["PATH", "SystemRoot", "WINDIR", "TMP", "TEMP", "TMPDIR"] { + if let Some(value) = std::env::var_os(key) { + command.env(key, value); + } + } + command + .env("HOME", self.test.project_dir.path()) + .env("USERPROFILE", self.test.project_dir.path()) + .env("XDG_CONFIG_HOME", self.test.project_dir.path()) + .env("NO_PROXY", "*") + .env("no_proxy", "*") + .envs(shell.iter().copied()) + .arg("--config-path") + .arg(&self.test.config_path) + .args(args) + .current_dir(self.test.project_dir.path()) + .stdin(Stdio::null()); + bounded_output(command) + } + + fn success(&self, args: &[&str], shell: &[(&str, &str)]) -> String { + let output = self.command(args, shell); + assert!( + output.status.success(), + "local ENV CLI command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + fn config(&self, environment: Option) { + for file in [ + "spacetime.local.json", + "spacetime.prod.json", + "spacetime.prod.local.json", + ] { + let path = self.test.project_dir.path().join(file); + if path.exists() { + fs::remove_file(path).unwrap(); + } + } + let mut config = json!({"database": self.database}); + if let Some(environment) = environment { + config["env"] = environment; + } + self.write("spacetime.json", config); + } + + fn write(&self, file: &str, value: Value) { + fs::write( + self.test.project_dir.path().join(file), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + + fn publish(&self, shell: &[(&str, &str)], extra: &[&str]) -> Output { + let mut args = vec![ + "publish", + &self.database, + "--bin-path", + self.wasm.to_str().unwrap(), + "--server", + &self.test.server_url, + "--yes", + ]; + args.extend_from_slice(extra); + self.command(&args, shell) + } + + fn published(&self, shell: &[(&str, &str)], extra: &[&str]) -> String { + let before = fs::read(&self.wasm).unwrap(); + let output = self.publish(shell, extra); + assert!( + output.status.success(), + "publish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fs::read(&self.wasm).unwrap(), before); + String::from_utf8(output.stdout).unwrap() + } + + fn get(&self, key: &str) -> String { + self.success( + &[ + "env", + "get", + &self.database, + key, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } + + fn list(&self) -> String { + self.success( + &[ + "env", + "list", + &self.database, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } + + fn typed(&self, required: &str, mode: &str, rest: [Option<&str>; 4]) { + let option = |value: Option<&str>| match value { + Some(value) => json!({"some": value}), + None => json!({"none": []}), + }; + let arguments = [ + json!(required), + json!(mode), + option(rest[0]), + option(rest[1]), + option(rest[2]), + option(rest[3]), + ] + .map(|value| value.to_string()); + let mut args = vec![ + "call", + &self.database, + "check_environment", + "--no-config", + "--server", + &self.test.server_url, + ]; + args.extend(arguments.iter().map(String::as_str)); + self.success(&args, &[]); + } + + fn sql(&self, statement: &str) -> Output { + self.command( + &[ + "sql", + &self.database, + statement, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } +} + +// Keep ownership through failure/timeout and avoid pipe backpressure. Output is +// generated fixture data; the cap also prevents accidental unbounded diagnostics. +fn bounded_output(mut command: Command) -> Output { + struct OwnedChild(Option); + impl Drop for OwnedChild { + fn drop(&mut self) { + if let Some(child) = self.0.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + let mut stdout = tempfile::tempfile().unwrap(); + let mut stderr = tempfile::tempfile().unwrap(); + let mut child = OwnedChild(Some( + command + .stdout(Stdio::from(stdout.try_clone().unwrap())) + .stderr(Stdio::from(stderr.try_clone().unwrap())) + .spawn() + .unwrap(), + )); + let deadline = Instant::now() + Duration::from_secs(90); + let status = loop { + assert!( + stdout.metadata().unwrap().len() <= 1024 * 1024 && stderr.metadata().unwrap().len() <= 1024 * 1024, + "local ENV CLI output exceeds bound" + ); + if let Some(status) = child.0.as_mut().unwrap().try_wait().unwrap() { + child.0.take(); // Already reaped: never signal this process identifier again. + break status; + } + assert!(Instant::now() < deadline, "local ENV CLI command timed out"); + std::thread::sleep(Duration::from_millis(10)); + }; + stdout.rewind().unwrap(); + stderr.rewind().unwrap(); + let mut out = Vec::new(); + let mut err = Vec::new(); + stdout.read_to_end(&mut out).unwrap(); + stderr.read_to_end(&mut err).unwrap(); + Output { + status, + stdout: out, + stderr: err, + } +} + +#[test] +fn cli_environment_layers_shell_and_exact_precompiled_declarations() { + let f = Fixture::new(); + f.write( + "spacetime.json", + json!({"database":"unused-parent", "env":{ + "SMOKE_REQUIRED":"base-required", "SMOKE_MODE":"ready", "SMOKE_OPTIONAL":"base-optional", + "SMOKE_EMPTY":"base-empty", "SMOKE_FLAG": false + }, "children":[{"database":f.database,"env":{"SMOKE_OPTIONAL":"child-optional"}}]}), + ); + // Use a raw JSON number to verify there is no f64 round trip. + fs::write(f.test.project_dir.path().join("spacetime.local.json"), + r#"{"env":{"SMOKE_REQUIRED":"local-required","SMOKE_NUMBER":9007199254740993123456789},"children":[{"env":{"SMOKE_OPTIONAL":"child-local"}}]}"#).unwrap(); + f.write( + "spacetime.prod.json", + json!({"env":{"SMOKE_REQUIRED":"prod-required","SMOKE_MODE":"other"}, + "children":[{"env":{"SMOKE_EMPTY":"child-prod"}}]}), + ); + f.write( + "spacetime.prod.local.json", + json!({"env":{"SMOKE_REQUIRED":"prod-local-required"}, + "children":[{"env":{"SMOKE_OPTIONAL":"child-final"}}]}), + ); + let output = f.published( + &[ + ("SMOKE_REQUIRED", "shell-required"), + ("SMOKE_EMPTY", ""), + ("SMOKE_UNDECLARED", "ambient-not-published"), + ], + &["--env", "prod"], + ); + for key in KEYS { + assert!(output.contains(key)); + } + for value in [ + "shell-required", + "child-final", + "9007199254740993123456789", + "ambient-not-published", + ] { + assert!(!output.contains(value), "publish display leaked a fixture value"); + } + assert!(output.contains("SMOKE_REQUIRED (shell)")); + assert!(output.contains("SMOKE_NUMBER (config)")); + assert_eq!(f.get("SMOKE_EMPTY"), "\n"); + assert_eq!(f.get("SMOKE_NUMBER"), "9007199254740993123456789\n"); + assert_eq!(f.get("SMOKE_FLAG"), "false\n"); + f.typed( + "shell-required", + "other", + [ + Some("child-final"), + Some(""), + Some("9007199254740993123456789"), + Some("false"), + ], + ); + let mut keys = KEYS.to_vec(); + keys.sort_unstable(); + assert_eq!(f.list(), format!("{}\n", keys.join("\n"))); + let initial = f.sql("SELECT required, mode FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("shell-required") && initial.contains("other")); +} + +#[test] +fn cli_environment_replacement_rejection_and_read_only_commands() { + let f = Fixture::new(); + f.config(Some( + json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), + )); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"replacement-sentinel","SMOKE_MODE":"other"}), + )); + f.published(&[], &[]); + f.typed("replacement-sentinel", "other", [None; 4]); + assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert!(!f + .command( + &[ + "env", + "get", + &f.database, + "SMOKE_OPTIONAL", + "--server", + &f.test.server_url, + "--no-config" + ], + &[] + ) + .status + .success()); + for input in [ + json!({"SMOKE_MODE":"ready"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"invalid-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","UNKNOWN":"unknown-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":{}}), + ] { + f.config(Some(input)); + let output = f.publish(&[], &[]); + assert!(!output.status.success()); + for value in ["rejected-sentinel", "invalid-sentinel", "unknown-sentinel"] { + assert!(!String::from_utf8_lossy(&output.stdout).contains(value)); + assert!(!String::from_utf8_lossy(&output.stderr).contains(value)); + } + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + f.typed("replacement-sentinel", "other", [None; 4]); + } + // Invalid local configuration must not prevent an explicit read-only target. + assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + for statement in [ + "SET env.SMOKE_REQUIRED = 'bypass'", + "DELETE env.SMOKE_REQUIRED", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!(!f.sql(statement).status.success()); + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + } + for operation in ["set", "delete", "unset"] { + assert!(!f + .command( + &[ + "env", + operation, + &f.database, + "SMOKE_REQUIRED", + "--server", + &f.test.server_url + ], + &[] + ) + .status + .success()); + } +} + +#[test] +fn cli_environment_initial_rejection_clear_and_omitted_payload() { + let mut f = Fixture::new(); + f.config(None); + assert!(!f.publish(&[], &[]).status.success()); + f.config(Some(json!({"SMOKE_REQUIRED":"clear-initial","SMOKE_MODE":"ready"}))); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"clear-replaced","SMOKE_MODE":"other","SMOKE_EMPTY":""}), + )); + f.published(&[], &["--delete-data"]); + f.typed("clear-replaced", "other", [None, Some(""), None, None]); + let initial = f.sql("SELECT required FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("clear-replaced") && !initial.contains("clear-initial")); + // A legacy module with no ENV declaration receives an empty complete input. + f.config(None); + f.wasm = modules::precompiled_module("noop"); + f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); + assert_eq!(f.list(), ""); +} diff --git a/crates/smoketests/tests/standalone/cli/mod.rs b/crates/smoketests/tests/standalone/cli/mod.rs index 8c2b6481318..ee04d64547e 100644 --- a/crates/smoketests/tests/standalone/cli/mod.rs +++ b/crates/smoketests/tests/standalone/cli/mod.rs @@ -1,5 +1,6 @@ mod auth; mod dev; +mod environment; mod generate; mod list; mod server; diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index a7593cc4582..be7b753f395 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,15 +19,6 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), - /// Administrative environment mutation, distinct from generic table DML. - Environment(SqlEnvironment), -} - -#[derive(Debug)] -pub struct SqlEnvironment { - pub key: SqlIdent, - /// None is DELETE; a string literal, including empty, is SET. - pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index a70ea4d5b39..0508baae5b2 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,13 +133,11 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, - keywords::Keyword, parser::Parser, - tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -150,36 +148,7 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - // DELETE env.KEY is a SpacetimeDB administrative statement, not the - // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression - // parser, including comments and SQL string escaping, for this extension. - let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; - let verb = parser.peek_token().token; - let environment_prefix = matches!(parser.peek_nth_token(1).token, - Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) - && parser.peek_nth_token(2).token == Token::Period; - if environment_prefix - && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) - { - parser.next_token(); - parser.next_token(); - parser.next_token(); - let key = SqlIdent(parser.parse_identifier()?.value.into()); - let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { - if !parser.parse_keyword(Keyword::TO) { - parser.expect_token(&Token::Eq)?; - } - Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) - } else { - None - }; - let _ = parser.consume_token(&Token::SemiColon); - if parser.peek_token().token != Token::EOF { - return Err(SqlUnsupported::MultiStatement.into()); - } - return Ok(SqlAst::Environment(SqlEnvironment { key, value })); - } - let mut stmts = parser.parse_statements()?; + let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index a637d38afcc..9432b6ee097 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use sled::transaction::{ self, ConflictableTransactionError, ConflictableTransactionResult, TransactionError, TransactionResult, @@ -339,19 +341,18 @@ impl ControlDb { let scan_key: &[u8] = b""; for result in tree.range(scan_key..) { let (_key, value) = result?; - let database = compat::Database::from_slice(&value)?.into(); + let database = self.decode_database(&value)?; databases.push(database); } Ok(databases) } pub fn get_database_by_id(&self, id: u64) -> Result> { - for database in self.get_databases()? { - if database.id == id { - return Ok(Some(database)); - } - } - Ok(None) + self.db + .open_tree("database")? + .get(id.to_be_bytes())? + .map(|bytes| self.decode_database(&bytes)) + .transpose() } pub fn get_database_by_identity(&self, identity: &Identity) -> Result> { @@ -359,12 +360,13 @@ impl ControlDb { let key = identity.to_be_byte_array(); let value = tree.get(&key[..])?; if let Some(value) = value { - let database = compat::Database::from_slice(&value[..])?.into(); + let database = self.decode_database(&value)?; return Ok(Some(database)); } Ok(None) } + #[cfg(test)] pub fn insert_database(&self, mut database: Database) -> Result { let id = self.db.generate_id()?; let tree = self.db.open_tree("database_by_identity")?; @@ -388,23 +390,6 @@ impl ControlDb { Ok(id) } - pub(crate) fn update_database(&self, database: Database) -> Result<()> { - let Some(stored_database) = self.get_database_by_identity(&database.database_identity)? else { - return Err(Error::DatabaseNotFound(database.database_identity)); - }; - - let tree = self.db.open_tree("database_by_identity")?; - let buf = sled::IVec::from(compat::Database::from(database).to_vec()?); - tree.insert(stored_database.database_identity.to_be_byte_array(), buf.clone())?; - tree.flush()?; - - let tree = self.db.open_tree("database")?; - tree.insert(stored_database.id.to_be_bytes(), buf)?; - tree.flush()?; - - Ok(()) - } - pub fn is_database_locked(&self, database_identity: &Identity) -> Result { let tree = self.db.open_tree("database_locks")?; let key = database_identity.to_be_byte_array(); @@ -419,20 +404,7 @@ impl ControlDb { } pub fn delete_database(&self, id: u64) -> Result> { - let tree = self.db.open_tree("database")?; - let tree_by_identity = self.db.open_tree("database_by_identity")?; - - if let Some(old_value) = tree.get(id.to_be_bytes())? { - let database = compat::Database::from_slice(&old_value[..])?; - let key = database.database_identity().to_be_byte_array(); - - tree_by_identity.remove(&key[..])?; - tree.remove(id.to_be_bytes())?; - tree.flush()?; - return Ok(Some(id)); - } - - Ok(None) + self.delete_database_and_environment(id) } pub fn get_replicas(&self) -> Result> { diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs new file mode 100644 index 00000000000..ede423c3935 --- /dev/null +++ b/crates/standalone/src/control_db/environment.rs @@ -0,0 +1,316 @@ +//! Private bootstrap inputs, separate from the historical public Database encoding. +use super::*; +use spacetimedb_client_api_messages::publish::PublishRequest; +use spacetimedb_lib::Hash; +use std::collections::BTreeMap; + +const METADATA_TREE: &str = "database_bootstrap"; +const VALUES_TREE: &str = "initial_environment"; + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Bootstrap { + version: u8, + database_id: u64, + identity: Identity, + program: Hash, + generation: u64, +} + +fn invalid() -> Error { + Error::Other(anyhow::anyhow!("invalid private bootstrap record")) +} + +impl Bootstrap { + fn decode(bytes: &[u8], database: &Database) -> Result { + if bytes.len() > 1024 { + return Err(invalid()); + } + let value: Self = serde_json::from_slice(bytes).map_err(|_| invalid())?; + if value.version != 1 + || value.database_id != database.id + || value.identity != database.database_identity + || value.program != database.initial_program + || value.generation == 0 + { + return Err(invalid()); + } + Ok(value) + } +} + +impl ControlDb { + pub(super) fn decode_database(&self, bytes: &[u8]) -> Result { + let mut database: Database = compat::Database::from_slice(bytes)?.into(); + if let Some(bytes) = self.db.open_tree(METADATA_TREE)?.get(database.id.to_be_bytes())? { + database.bootstrap_generation = Bootstrap::decode(&bytes, &database)?.generation; + } + Ok(database) + } + + /// Recheck the exact persisted generation and program before releasing any values. + pub(crate) fn initial_environment(&self, database: &Database) -> Result> { + let databases = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &metadata, &values).transaction(|(databases, metadata, values)| { + let persisted = databases + .get(database.database_identity.to_be_byte_array())? + .ok_or_else(|| { + ConflictableTransactionError::Abort(Error::DatabaseNotFound(database.database_identity)) + })?; + let stored: Database = compat::Database::from_slice(&persisted) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != database.id + || stored.initial_program != database.initial_program + || stored.owner_identity != database.owner_identity + { + return transaction::abort(invalid()); + } + match metadata.get(database.id.to_be_bytes())? { + Some(bytes) => { + let binding = + Bootstrap::decode(&bytes, &stored).map_err(ConflictableTransactionError::Abort)?; + if binding.generation != database.bootstrap_generation { + return transaction::abort(invalid()); + } + let bytes = values + .get(database.id.to_be_bytes())? + .ok_or_else(|| ConflictableTransactionError::Abort(invalid()))?; + Ok(Some(bytes)) + } + None if database.bootstrap_generation == 0 => { + if values.get(database.id.to_be_bytes())?.is_some() { + return transaction::abort(invalid()); + } + Ok(None) + } + None => transaction::abort(invalid()), + } + }); + let bytes = result.map_err(transaction_error)?; + match bytes { + None => Ok(BTreeMap::new()), + Some(bytes) => { + let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; + if !request.module.is_empty() { + return Err(invalid()); + } + Ok(request.environment) + } + } + } + + /// Install the desired initial database, its complete private input and a + /// durable leader nomination together. Cancellation before host launch can + /// therefore recover through the ordinary leader lookup. + pub(crate) fn install_database_with_environment( + &self, + mut database: Database, + expected: Option<&Database>, + environment: BTreeMap, + previous_replicas: &[Replica], + ) -> Result<(Database, Replica)> { + if expected.is_none() { + database.id = self.db.generate_id()?; + } + database.bootstrap_generation = expected + .map_or(Some(1), |old| old.bootstrap_generation.checked_add(1)) + .ok_or_else(invalid)?; + let replica = Replica { + id: self.db.generate_id()?, + database_id: database.id, + node_id: 0, + leader: true, + }; + let input = PublishRequest { + module: Vec::new(), + environment, + } + .encode() + .map_err(|_| invalid())?; + let binding = Bootstrap { + version: 1, + database_id: database.id, + identity: database.database_identity, + program: database.initial_program, + generation: database.bootstrap_generation, + }; + let binding = serde_json::to_vec(&binding).map_err(|_| invalid())?; + let encoded = compat::Database::from(database.clone()).to_vec()?; + let encoded_replica = bsatn::to_vec(&replica)?; + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let replicas = self.db.open_tree("replica")?; + let result: TransactionResult<(), Error> = (&databases, &identities, &metadata, &values, &replicas) + .transaction(|(databases, identities, metadata, values, replicas)| { + let identity_key = database.database_identity.to_be_byte_array(); + match (expected, identities.get(identity_key)?) { + (None, None) => {} + (Some(expected), Some(bytes)) => { + let stored: Database = compat::Database::from_slice(&bytes) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != expected.id + || stored.initial_program != expected.initial_program + || stored.owner_identity != expected.owner_identity + { + return transaction::abort(invalid()); + } + let generation = match metadata.get(stored.id.to_be_bytes())? { + Some(bytes) => { + Bootstrap::decode(&bytes, &stored) + .map_err(ConflictableTransactionError::Abort)? + .generation + } + None => 0, + }; + if generation != expected.bootstrap_generation { + return transaction::abort(invalid()); + } + } + (None, Some(_)) => { + return transaction::abort(Error::DatabaseAlreadyExists(database.database_identity)) + } + (Some(_), None) => return transaction::abort(Error::DatabaseNotFound(database.database_identity)), + } + databases.insert(&database.id.to_be_bytes(), encoded.clone())?; + identities.insert(&identity_key, encoded.clone())?; + metadata.insert(&database.id.to_be_bytes(), binding.clone())?; + values.insert(&database.id.to_be_bytes(), input.clone())?; + for previous in previous_replicas { + if previous.database_id != database.id { + return transaction::abort(invalid()); + } + replicas.remove(&previous.id.to_be_bytes())?; + } + replicas.insert(&replica.id.to_be_bytes(), encoded_replica.clone())?; + databases.flush(); + Ok(()) + }); + result.map_err(transaction_error)?; + self.db.flush()?; + Ok((database, replica)) + } + + pub(super) fn delete_database_and_environment(&self, id: u64) -> Result> { + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &identities, &metadata, &values).transaction(|(databases, identities, metadata, values)| { + let Some(bytes) = databases.get(id.to_be_bytes())? else { + return Ok(None); + }; + let database = + compat::Database::from_slice(&bytes).map_err(|_| ConflictableTransactionError::Abort(invalid()))?; + identities.remove(&database.database_identity().to_be_byte_array())?; + databases.remove(&id.to_be_bytes())?; + metadata.remove(&id.to_be_bytes())?; + values.remove(&id.to_be_bytes())?; + databases.flush(); + Ok(Some(id)) + }); + let result = result.map_err(transaction_error)?; + self.db.flush()?; + Ok(result) + } +} + +fn transaction_error(error: TransactionError) -> Error { + match error { + TransactionError::Abort(error) => error, + TransactionError::Storage(error) => error.into(), + } +} + +#[async_trait::async_trait] +impl spacetimedb::host::InitialEnvironmentSource for ControlDb { + async fn load(&self, database: &Database) -> anyhow::Result> { + let source = self.clone(); + let database = database.clone(); + spacetimedb::util::asyncify(move || source.initial_environment(&database)) + .await + .map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb::messages::control_db::HostType; + + fn database() -> Database { + Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: Identity::ZERO, + host_type: HostType::Wasm, + initial_program: spacetimedb_lib::hash_bytes(b"module"), + bootstrap_generation: 0, + } + } + + #[test] + fn bootstrap_is_durable_atomic_generation_bound_and_deleted_with_database() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let (original, original_replica) = { + let control = ControlDb::at(temp.path())?; + control.install_database_with_environment( + database(), + None, + BTreeMap::from([("VALUE".into(), "first".into())]), + &[], + )? + }; + let control = ControlDb::at(temp.path())?; + let loaded = control.get_database_by_id(original.id)?.unwrap(); + assert_eq!(loaded.bootstrap_generation, 1); + assert_eq!(control.initial_environment(&loaded)?["VALUE"], "first"); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + original_replica.id + ); + let (replaced, new_replica) = control.install_database_with_environment( + loaded.clone(), + Some(&loaded), + BTreeMap::new(), + &[original_replica], + )?; + assert_eq!(replaced.bootstrap_generation, 2); + assert!(control.initial_environment(&original).is_err()); + assert!(control.initial_environment(&replaced)?.is_empty()); + assert_eq!(control.get_replicas_by_database(loaded.id)?.len(), 1); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + new_replica.id + ); + assert!(control + .install_database_with_environment(loaded.clone(), Some(&loaded), BTreeMap::new(), &[]) + .is_err()); + assert_eq!(control.get_database_by_id(loaded.id)?.unwrap().bootstrap_generation, 2); + control.delete_database(loaded.id)?; + assert!(control.initial_environment(&replaced).is_err()); + assert!(control.db.open_tree(VALUES_TREE)?.is_empty()); + Ok(()) + } + + #[test] + fn legacy_absence_is_empty_but_missing_new_values_fails_closed() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let control = ControlDb::at(temp.path())?; + let id = control.insert_database(database())?; + let legacy = control.get_database_by_id(id)?.unwrap(); + assert!(control.initial_environment(&legacy)?.is_empty()); + let (new, _) = + control.install_database_with_environment(legacy.clone(), Some(&legacy), BTreeMap::new(), &[])?; + control.db.open_tree(VALUES_TREE)?.remove(new.id.to_be_bytes())?; + assert!(control.initial_environment(&new).is_err()); + Ok(()) + } +} diff --git a/crates/standalone/src/environment_tests.rs b/crates/standalone/src/environment_tests.rs new file mode 100644 index 00000000000..abeb5a84193 --- /dev/null +++ b/crates/standalone/src/environment_tests.rs @@ -0,0 +1,190 @@ +//! Actual module persistence with explicit local input, without a server or CLI configuration. +use super::*; +use spacetimedb::host::FunctionArgs; +use spacetimedb_client_api::{ControlStateWriteAccess as _, DatabaseDef}; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue}; +use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; +use spacetimedb_paths::FromPathUnchecked; +use std::collections::BTreeMap; + +type Values = BTreeMap; + +async fn read(env: &StandaloneEnv, database: u64, key: &str) -> anyhow::Result { + let module = env.leader(database).await?.module().await?; + Ok(module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key])?.into()), + ) + .await + .result? + .return_val) +} + +#[tokio::test] +#[ignore = "requires an explicitly configured local environment-test Wasm artifact"] +async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_semantics() -> anyhow::Result<()> { + let module_path = std::path::PathBuf::from( + std::env::var_os("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE") + .context("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE must name the owned local fixture")?, + ); + anyhow::ensure!(module_path.is_absolute(), "fixture path must be absolute"); + let bytes = axum::body::Bytes::from(std::fs::read(module_path)?); + anyhow::ensure!(bytes.starts_with(b"\0asm"), "fixture must be a Wasm module"); + let temp = tempfile::tempdir()?; + let keys = temp.path().join("keys"); + std::fs::create_dir(&keys)?; + let ca = CertificateAuthority { + jwt_pub_key_path: PubKeyPath(keys.join("public")), + jwt_priv_key_path: PrivKeyPath(keys.join("private")), + }; + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(temp.path().join("data"))); + data_dir.create()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + let env = StandaloneEnv::init(config, &ca, data_dir, JobCores::without_pinned_cores()).await?; + let test_env = env.clone(); + let run = tokio::spawn(async move { + let env = test_env; + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + let spec = |environment| DatabaseDef { + database_identity: Identity::ZERO, + program_bytes: bytes.clone(), + environment, + num_replicas: None, + host_type: HostType::Wasm, + parent: None, + organization: None, + }; + log::info!("ENV standalone fixture: initial publication"); + assert!(env + .publish_database(&Identity::ZERO, spec(initial.clone()), MigrationPolicy::Compatible) + .await? + .is_none()); + let database = env.control_db.get_database_by_identity(&Identity::ZERO)?.unwrap(); + let replica = env.control_db.get_leader_replica_by_database(database.id).unwrap(); + log::info!("ENV standalone fixture: rejected publication preserves live host"); + let rejected = env + .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .await; + assert!( + rejected.as_ref().is_err() + || rejected + .as_ref() + .unwrap() + .as_ref() + .is_some_and(|result| !result.was_successful()) + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("initial-required".to_owned())) + ); + log::info!("ENV standalone fixture: same-program update"); + let mut updated = initial.clone(); + updated.insert("REQUIRED".into(), "republished".into()); + assert!(env + .publish_database(&Identity::ZERO, spec(updated), MigrationPolicy::Compatible) + .await? + .unwrap() + .was_successful()); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // The old private bootstrap input intentionally remains distinct. Ordinary + // reopen must consult persisted st_module/st_env and skip that input. + assert_eq!( + env.control_db.initial_environment(&database)?["REQUIRED"], + "initial-required" + ); + log::info!("ENV standalone fixture: positive close and normal reopen"); + let replica_id = replica.id; + env.own_publication( + move |owner| async move { owner.host_controller.exit_module_host_and_join(replica_id).await }, + ) + .await?; + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // Init asserts initial-required. A successful reopen with republished proves + // that init was not run again and old bootstrap input was not restored. + assert!(env + .reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: Values::new(), + num_replicas: None, + host_type: None, + } + ) + .await + .is_err()); + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 1 + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + log::info!("ENV standalone fixture: complete no-artifact reset"); + let mut reset = initial; + reset.insert("EMPTY".into(), "reset-input".into()); + env.reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: reset, + num_replicas: None, + host_type: None, + }, + ) + .await?; + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 2 + ); + assert_ne!( + env.control_db.get_leader_replica_by_database(database.id).unwrap().id, + replica.id + ); + assert_eq!( + read(&env, database.id, "EMPTY").await?, + AlgebraicValue::from(Some("reset-input".to_owned())) + ); + anyhow::Ok(()) + }) + .await; + // Join physical cleanup even if an assertion in the accepted fixture task + // panicked. Never let a failed test detach its database host. + log::info!("ENV standalone fixture: positive cleanup"); + env.delete_database(&Identity::ZERO, &Identity::ZERO).await?; + run??; + assert!(env.control_db.get_database_by_identity(&Identity::ZERO)?.is_none()); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index a79f9814f5d..efe11d860fa 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -1,4 +1,6 @@ mod control_db; +#[cfg(test)] +mod environment_tests; pub mod subcommands; pub mod util; pub mod version; @@ -16,7 +18,7 @@ use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; -use spacetimedb::messages::control_db::{Database, Node, Replica}; +use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::metrics::ENGINE_METRICS; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use spacetimedb::util::jobs::JobCores; @@ -34,7 +36,8 @@ use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; -use std::sync::Arc; +use std::sync::{Arc, Weak}; +#[cfg(test)] use std::time::Duration; pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; @@ -51,6 +54,8 @@ pub struct StandaloneOptions { pub struct StandaloneEnv { control_db: ControlDb, + publication_lock: Arc>, + weak_self: Weak, program_store: Arc, host_controller: HostController, client_actor_index: ClientActorIndex, @@ -90,7 +95,8 @@ impl StandaloneEnv { Arc::new(()), persistence_provider, db_cores, - ); + ) + .with_initial_environment_source(Arc::new(control_db.clone())); let client_actor_index = ClientActorIndex::new(); let jwt_keys = certs.get_or_create_keys()?; @@ -102,8 +108,10 @@ impl StandaloneEnv { metrics_registry.register(Box::new(&*DB_METRICS)).unwrap(); metrics_registry.register(Box::new(&*DATA_SIZE_METRICS)).unwrap(); - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|weak_self| Self { control_db, + publication_lock: Arc::new(tokio::sync::RwLock::new(())), + weak_self: weak_self.clone(), program_store, host_controller, client_actor_index, @@ -176,20 +184,14 @@ impl NodeDelegate for StandaloneEnv { } async fn leader(&self, database_id: u64) -> Result { - let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { - return Err(GetLeaderHostError::NoSuchReplica); - }; - - let Some(database) = self.control_db.get_database_by_id(database_id)? else { - return Err(GetLeaderHostError::NoSuchDatabase); - }; - - self.host_controller - .get_or_launch_module_host(database, leader.id) - .await - .map_err(|source| GetLeaderHostError::LaunchError { source })?; - - Ok(Host::new(leader.id, self.host_controller.clone())) + let guard = self.publication_lock.clone().read_owned().await; + let owner = self.weak_self.upgrade().expect("standalone owner exists during lookup"); + tokio::spawn(async move { + let _guard = guard; + owner.leader_under_publication(database_id).await + }) + .await + .map_err(|error| GetLeaderHostError::LaunchError { source: error.into() })? } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { @@ -276,6 +278,153 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { publisher: &Identity, spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, + ) -> anyhow::Result> { + let publisher = *publisher; + self.own_publication(move |owner| async move { owner.publish_database_owned(&publisher, spec, policy).await }) + .await + } + + async fn migrate_plan( + &self, + spec: spacetimedb_client_api::DatabaseDef, + style: PrettyPrintStyle, + ) -> anyhow::Result { + let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; + + match existing_db { + Some(db) => { + let host = self.leader(db.id).await?; + self.host_controller + .migrate_plan( + db, + spec.host_type, + host.replica_id, + spec.program_bytes.to_vec().into(), + style, + ) + .await + } + None => anyhow::bail!( + "Database `{}` does not exist", + spec.database_identity.to_abbreviated_hex() + ), + } + } + + async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + let database_identity = *database_identity; + self.own_publication(move |owner| async move { + owner.delete_database_owned(&caller_identity, &database_identity).await + }) + .await + } + + async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + self.own_publication(move |owner| async move { owner.reset_database_owned(&caller_identity, spec).await }) + .await + } + + async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { + let balance = self + .control_db + .get_energy_balance(identity)? + .unwrap_or(EnergyBalance::ZERO); + + let balance = balance.saturating_add_energy(amount); + + self.control_db.set_energy_balance(*identity, balance)?; + Ok(()) + } + async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { + // The energy balance code is obsolete. + Ok(()) + } + + async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { + Ok(self.control_db.spacetime_register_tld(tld, *identity)?) + } + + async fn create_dns_record( + &self, + owner_identity: &Identity, + domain: &DomainName, + database_identity: &Identity, + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) + } + + async fn replace_dns_records( + &self, + database_identity: &Identity, + owner_identity: &Identity, + domain_names: &[DomainName], + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) + } + + async fn set_database_lock( + &self, + _caller_identity: &Identity, + database_identity: &Identity, + locked: bool, + ) -> anyhow::Result<()> { + let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { + anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); + }; + self.control_db.set_database_lock(database_identity, locked)?; + Ok(()) + } +} + +impl StandaloneEnv { + /// Admission and completion are owned together. Losing an HTTP waiter cannot + /// release the reset fence while HostController still owns accepted work. + async fn own_publication(&self, operation: F) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let guard = self.publication_lock.clone().write_owned().await; + let owner = self + .weak_self + .upgrade() + .expect("standalone owner exists during publication"); + tokio::spawn(async move { + let _guard = guard; + operation(owner).await + }) + .await? + } + + async fn leader_under_publication(&self, database_id: u64) -> Result { + let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { + return Err(GetLeaderHostError::NoSuchReplica); + }; + + let Some(database) = self.control_db.get_database_by_id(database_id)? else { + return Err(GetLeaderHostError::NoSuchDatabase); + }; + + self.host_controller + .get_or_launch_module_host(database, leader.id) + .await + .map_err(|source| GetLeaderHostError::LaunchError { source })?; + + Ok(Host::new(leader.id, self.host_controller.clone())) + } + + async fn publish_database_owned( + &self, + publisher: &Identity, + spec: spacetimedb_client_api::DatabaseDef, + policy: MigrationPolicy, ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; @@ -301,28 +450,42 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { // Instantiate a temporary database in order to check that the module is valid. // This will e.g. typecheck RLS filters. self.host_controller - .check_module_validity(database.clone(), program) + .check_module_validity_with_environment(database.clone(), program, spec.environment.clone()) .await?; let program_hash = self.program_store.put(&spec.program_bytes).await?; debug_assert_eq!(_hash_for_assert, program_hash); - let database_id = self.control_db.insert_database(database)?; - - self.schedule_replicas(database_id, num_replicas).await?; + let (database, replica) = + self.control_db + .install_database_with_environment(database, None, spec.environment, &[])?; + // The leader nomination and input are durable already. If this + // waiter is cancelled, ordinary lookup resumes the same input. + self.on_insert_replica(&replica).await?; + debug_assert_eq!(database.id, replica.database_id); Ok(None) } // The database already exists, so we'll try to update it. // If that fails, we'll keep the old one. Some(database) => { + anyhow::ensure!( + database.owner_identity == *publisher, + "database ownership changed before publication" + ); let database_id = database.id; let database_identity = database.database_identity; - let leader = self.leader(database_id).await?; + let leader = self.leader_under_publication(database_id).await?; let update_result = leader - .update(database, spec.host_type, spec.program_bytes.to_vec().into(), policy) + .update_with_environment( + database, + spec.host_type, + spec.program_bytes.to_vec().into(), + policy, + spec.environment, + ) .await?; if update_result.was_successful() { let replicas = self.control_db.get_replicas_by_database(database_id)?; @@ -372,37 +535,18 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } } - async fn migrate_plan( + async fn delete_database_owned( &self, - spec: spacetimedb_client_api::DatabaseDef, - style: PrettyPrintStyle, - ) -> anyhow::Result { - let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; - - match existing_db { - Some(db) => { - let host = self.leader(db.id).await?; - self.host_controller - .migrate_plan( - db, - spec.host_type, - host.replica_id, - spec.program_bytes.to_vec().into(), - style, - ) - .await - } - None => anyhow::bail!( - "Database `{}` does not exist", - spec.database_identity.to_abbreviated_hex() - ), - } - } - - async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + caller_identity: &Identity, + database_identity: &Identity, + ) -> anyhow::Result<()> { let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { return Ok(()); }; + anyhow::ensure!( + database.owner_identity == *caller_identity, + "database ownership changed before deletion" + ); self.control_db.delete_database(database.id)?; for instance in self.control_db.get_replicas_by_database(database.id)? { @@ -412,93 +556,51 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { Ok(()) } - async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { - let mut database = self + async fn reset_database_owned(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let previous = self .control_db .get_database_by_identity(&spec.database_identity)? .with_context(|| format!("Database `{}` does not exist", spec.database_identity))?; - let database_id = database.id; - - if let Some(program) = spec.program_bytes { - if let Some(host_type) = spec.host_type { - database.host_type = host_type; + anyhow::ensure!( + previous.owner_identity == *caller_identity, + "database ownership changed before reset" + ); + let mut database = previous.clone(); + let program = match spec.program_bytes { + Some(bytes) => { + let host_type = spec.host_type.unwrap_or(database.host_type); + Program::from_bytes(host_type.into(), &bytes[..]) + } + None => { + // A reset without an artifact retains the currently committed + // module, not the original bootstrap program or its old values. + let module = self.leader_under_publication(database.id).await?.module().await?; + module + .relational_db() + .program()? + .context("database is not initialized")? } - let program_bytes = &program[..]; - let program = Program::from_bytes(database.host_type.into(), program_bytes); - let _hash_for_assert = program.hash; - - database.initial_program = program.hash; - - self.host_controller - .check_module_validity(database.clone(), program) - .await?; - let _stored_hash_for_assert = self.program_store.put(program_bytes).await?; - debug_assert_eq!(_hash_for_assert, _stored_hash_for_assert); - } - self.control_db.update_database(database)?; - - for instance in self.control_db.get_replicas_by_database(database_id)? { - self.delete_replica(instance.id).await?; - } - // Standalone only support a single replica. - let num_replicas = 1; - self.schedule_replicas(database_id, num_replicas).await?; - - Ok(()) - } - - async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { - let balance = self - .control_db - .get_energy_balance(identity)? - .unwrap_or(EnergyBalance::ZERO); - - let balance = balance.saturating_add_energy(amount); - - self.control_db.set_energy_balance(*identity, balance)?; - Ok(()) - } - async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { - // The energy balance code is obsolete. - Ok(()) - } - - async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { - Ok(self.control_db.spacetime_register_tld(tld, *identity)?) - } - - async fn create_dns_record( - &self, - owner_identity: &Identity, - domain: &DomainName, - database_identity: &Identity, - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) - } - - async fn replace_dns_records( - &self, - database_identity: &Identity, - owner_identity: &Identity, - domain_names: &[DomainName], - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) - } - - async fn set_database_lock( - &self, - _caller_identity: &Identity, - database_identity: &Identity, - locked: bool, - ) -> anyhow::Result<()> { - let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { - anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); }; - self.control_db.set_database_lock(database_identity, locked)?; + database.host_type = HostType::from(program.kind); + database.initial_program = program.hash; + self.host_controller + .check_module_validity_with_environment(database.clone(), program.clone(), spec.environment.clone()) + .await?; + let stored = self.program_store.put(&program.bytes).await?; + anyhow::ensure!(stored == program.hash, "stored reset program changed"); + let previous_replicas = self.control_db.get_replicas_by_database(database.id)?; + // Keep old nominations until all requested closes succeed. The owned + // publication guard excludes leader admission across close and commit. + for replica in &previous_replicas { + self.on_delete_replica(replica.id).await?; + } + let (_, replica) = self.control_db.install_database_with_environment( + database, + Some(&previous), + spec.environment, + &previous_replicas, + )?; + self.on_insert_replica(&replica).await?; Ok(()) } } @@ -567,21 +669,6 @@ impl StandaloneEnv { Ok(()) } - async fn schedule_replicas(&self, database_id: u64, num_replicas: u8) -> Result<(), anyhow::Error> { - // Just scheduling a bunch of replicas to the only machine - for i in 0..num_replicas { - let replica = Replica { - id: 0, - database_id, - node_id: 0, - leader: i == 0, - }; - self.insert_replica(replica).await?; - } - - Ok(()) - } - async fn on_insert_replica(&self, instance: &Replica) -> Result<(), anyhow::Error> { if instance.leader { let database = self @@ -593,7 +680,7 @@ impl StandaloneEnv { instance.database_id, instance.id ) })?; - self.leader(database.id).await?; + self.leader_under_publication(database.id).await?; } Ok(()) @@ -604,9 +691,7 @@ impl StandaloneEnv { // replicas which have been deleted. This will just drop // them from memory, but will not remove them from disk. We need // some kind of database lifecycle manager long term. - self.host_controller - .exit_module_host(replica_id, Duration::from_secs(30)) - .await?; + self.host_controller.exit_module_host_and_join(replica_id).await?; Ok(()) } @@ -690,4 +775,74 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cancelled_publication_waiter_keeps_mutations_and_leader_admission_fenced() -> Result<()> { + let tempdir = TempDir::new()?; + // Use one subdir for keys and another for the data dir. + let keys = tempdir.path().join("keys"); + let root = tempdir.path().join("data"); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(root)); + + fs::create_dir(&keys)?; + data_dir.create()?; + + let pub_key = PubKeyPath(keys.join("public")); + let priv_key = PrivKeyPath(keys.join("private")); + let ca = CertificateAuthority { + jwt_pub_key_path: pub_key, + jwt_priv_key_path: priv_key, + }; + + // Create the keys. + ca.get_or_create_keys()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + + let env = StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; + + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let owner = env.clone(); + let waiter = tokio::spawn(async move { + owner + .own_publication(move |_owner| async move { + let _ = started.send(()); + release_rx.await?; + Ok(()) + }) + .await + }); + started_rx.await?; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + + // Both a later mutation and ordinary leader lookup must stay behind the + // accepted operation even after its request future has disappeared. + let next_owner = env.clone(); + let mut next = tokio::spawn(async move { next_owner.own_publication(|_| async { Ok(()) }).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut next) + .await + .is_err()); + let read_owner = env.clone(); + let mut reader = tokio::spawn(async move { read_owner.leader(u64::MAX).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut reader) + .await + .is_err()); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(5), next).await???; + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), reader).await??, + Err(GetLeaderHostError::NoSuchReplica) + )); + Ok(()) + } } diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 465e22c3d6e..09d35280562 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::env; use std::future::Future; use std::panic::AssertUnwindSafe; @@ -67,6 +68,46 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Publish a complete configuration through the standalone control API. + pub async fn republish_environment( + &self, + environment: BTreeMap, + ) -> anyhow::Result { + let program = self + .client + .module() + .relational_db() + .program()? + .expect("published module"); + self.republish_program(program.bytes.into(), program.kind.into(), environment) + .await + } + + /// Publish exact replacement artifact bytes and their complete configuration. + pub async fn republish_program( + &self, + program_bytes: Bytes, + host_type: HostType, + environment: BTreeMap, + ) -> anyhow::Result { + self.env + .publish_database( + &Identity::ZERO, + DatabaseDef { + database_identity: self.db_identity, + program_bytes, + environment, + num_replicas: None, + host_type, + parent: None, + organization: None, + }, + MigrationPolicy::Compatible, + ) + .await? + .ok_or_else(|| anyhow::anyhow!("expected an update to the existing database")) + } + async fn call_reducer_result(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result { let result = self .client @@ -235,6 +276,18 @@ pub enum CompilationMode { } impl CompiledModule { + /// Use an artifact built by an external toolchain, such as Linux NativeAOT. + /// This changes only compilation; publication and execution still use the + /// same in-process standalone server as locally compiled fixtures. + pub fn from_artifact(name: &str, host_type: HostType, path: PathBuf) -> Self { + Self { + name: name.to_owned(), + path, + host_type, + program_bytes: OnceLock::new(), + } + } + pub fn compile(name: &str, mode: CompilationMode) -> Self { let (path, host_type) = spacetimedb_cli::build( &module_path(name), @@ -245,12 +298,7 @@ impl CompiledModule { None, ) .expect("Module compilation failed"); - Self { - name: name.to_owned(), - path, - host_type: host_type.parse().unwrap(), - program_bytes: OnceLock::new(), - } + Self::from_artifact(name, host_type.parse().unwrap(), path) } pub fn path(&self) -> &Path { @@ -279,10 +327,22 @@ impl CompiledModule { where R: FnOnce(ModuleHandle) -> F, F: Future, + { + self.with_module_async_with_environment(config, BTreeMap::new(), routine) + } + + pub fn with_module_async_with_environment( + &self, + config: Config, + environment: BTreeMap, + routine: R, + ) where + R: FnOnce(ModuleHandle) -> F, + F: Future, { with_runtime(move |runtime| { runtime.block_on(async { - let module = self.load_module(config, None).await; + let module = self.load_module_with_environment(config, None, environment).await; let env = module.env.clone(); let db_identity = module.db_identity; let routine_result = AssertUnwindSafe(routine(module)).catch_unwind().await.map(drop); @@ -311,6 +371,16 @@ impl CompiledModule { /// without resetting the database. /// This is used to speed up benchmarks running under callgrind (it allows them to reuse native-compiled wasm modules). pub async fn load_module(&self, config: Config, reuse_db_path: Option<&RootDir>) -> ModuleHandle { + self.load_module_with_environment(config, reuse_db_path, BTreeMap::new()) + .await + } + + pub async fn load_module_with_environment( + &self, + config: Config, + reuse_db_path: Option<&RootDir>, + environment: BTreeMap, + ) -> ModuleHandle { let paths = match reuse_db_path { Some(path) => SpacetimePaths::from_root_dir(path), None => { @@ -350,6 +420,7 @@ impl CompiledModule { DatabaseDef { database_identity: db_identity, program_bytes: self.program_bytes(), + environment, num_replicas: None, host_type: self.host_type, parent: None, diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 7cdf4fecf0e..96a29b29bf9 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -1,14 +1,20 @@ -//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +//! Actual publication and module calls exercise configuration atomicity and ABI enforcement. use serial_test::serial; +use spacetimedb::client::{messages::SerializableMessage, OutboundMessage}; use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_client_api_messages::websocket::v1 as ws_v1; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; -use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, ModuleHandle, DEFAULT_CONFIG}; +use std::collections::BTreeMap; +use std::time::Duration; -async fn sql(module: &ModuleHost, statement: String) -> Vec { +type Values = BTreeMap; + +async fn sql(module: &ModuleHost, statement: &str) -> Vec { spacetimedb::sql::execute::run( module.relational_db().clone(), - statement, + statement.to_string(), AuthCtx::for_current(Identity::ZERO), Some(module.info.subscriptions.clone()), Some(module.clone()), @@ -19,13 +25,190 @@ async fn sql(module: &ModuleHost, statement: String) -> Vec ModuleHost { + let result = handle.republish_environment(values.clone()).await.unwrap(); + assert!(result.was_successful(), "configuration publication failed"); + handle.client.module() +} + +async fn read(module: &ModuleHost, key: &str) -> AlgebraicValue { + module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()), + ) + .await + .result + .unwrap() + .return_val +} + +async fn next_message(handle: &mut ModuleHandle) -> OutboundMessage { + tokio::time::timeout(Duration::from_secs(10), handle.recv_message()) + .await + .expect("timed out waiting for environment subscription update") + .expect("environment subscription disconnected") +} + +async fn expect_view_update(handle: &mut ModuleHandle) { + let message = next_message(handle).await; + assert!(matches!(message, OutboundMessage::V1(SerializableMessage::TxUpdate(_)))); + // Replacing the one-row view must send both the old row's deletion and + // the new row's insertion to an already connected subscriber. + assert_eq!(message.num_rows(), Some(2)); +} + +async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { + values.insert("EMPTY".into(), "root-visible".into()); + let module = publish(handle, values).await; + assert!(module.info.module_def.reducer_by_name("lib.env_read_reducer").is_some()); + let child = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "lib.env_read_reducer", + FunctionArgs::Nullary, + ) + .await; + assert!(child.is_err() || child.unwrap().outcome.into_result().is_err()); + for procedure in ["lib.env_read_procedure", "lib.env_read_in_tx"] { + assert!(module.info.module_def.procedure_by_name(procedure).is_some()); + let result = module + .call_procedure(Identity::ZERO, None, None, procedure, FunctionArgs::Nullary) + .await; + assert!(result.result.is_err(), "submodule procedure read the root environment"); + } + for view in ["lib.env_read_view", "lib.env_read_sql_view", "env_read_root_sql_view"] { + assert!(module.info.module_def.view_by_name_with_module(view).is_some()); + let result = spacetimedb::sql::execute::run( + module.relational_db().clone(), + format!("SELECT * FROM {view}"), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await; + let error = format!( + "{:#}", + result.expect_err("view bypassed the environment read interface") + ); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + // A valid view and a forbidden view share one actual subscription + // request. It must fail as a whole, without an initial-success message. + let request_id = 880; + let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { + query_strings: ["SELECT * FROM my_player".into(), format!("SELECT * FROM {view}").into()].into(), + request_id, + }); + let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; + let message = next_message(handle).await; + let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { + panic!("failed view subscription returned a success or unexpected message: {message:?}"); + }; + assert_eq!(message.request_id, Some(request_id)); + let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { + panic!("forbidden view subscription returned rows"); + }; + assert!(!error.message.is_empty()); + assert!(!error.message.contains("not found")); + } + // HTTP routes are root entries today. Calling an exported child callback + // as an ordinary helper retains that root entry's authority. + assert_eq!( + &handle.call_http_route_get("/env-child").await.unwrap()[..], + b"root-visible" + ); +} + +// SQL, subscription materialization, and ordinary reducers all use the same +// main Wasmtime instance. The fixture view sets a guest-global marker and traps; +// the next reducer checks the marker is absent in the replacement instance. +async fn check_wasm_trap_disposal(handle: &mut ModuleHandle) { + let module = handle.client.module(); + let view = "SELECT * FROM environment_trap"; + let result = spacetimedb::sql::execute::run( + module.relational_db().clone(), + view.to_string(), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await; + let error = format!("{:#}", result.expect_err("trapped view returned SQL success")); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + expect_clean_wasm_instance(&module).await; + + // Keep the existing successful ENV-view subscription while adding this + // separate query. The failing request must report an error, not rows. + let request_id = 890; + let subscribe = ws_v1::ClientMessage::::SubscribeSingle(ws_v1::SubscribeSingle { + query: view.into(), + request_id, + query_id: ws_v1::QueryId::new(890), + }); + let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; + let message = next_message(handle).await; + let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { + panic!("trapped view subscription returned success or an unexpected message: {message:?}"); + }; + assert_eq!(message.request_id, Some(request_id)); + let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { + panic!("trapped view subscription returned rows"); + }; + assert!(!error.message.is_empty()); + assert!(!error.message.contains("not found")); + expect_clean_wasm_instance(&module).await; +} + +async fn expect_clean_wasm_instance(module: &ModuleHost) { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product!["MISSING", None::]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); } fn exercise_fixture(name: &str) { - CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { - let module = handle.client.module(); + let initial = if name == "environment-test" { + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]) + } else { + Values::new() + }; + // NativeAOT's WebAssembly compiler requires a supported compiler host. + // Allow this one fixture to consume the exact artifact built there while + // exercising all normal publication and runtime paths below. + let artifact = (name == "module-test-cs") + .then(|| std::env::var_os("SPACETIMEDB_ENV_CSHARP_MODULE")) + .flatten(); + let compiled = match artifact { + Some(path) => { + CompiledModule::from_artifact(name, spacetimedb::messages::control_db::HostType::Wasm, path.into()) + } + None => CompiledModule::compile(name, CompilationMode::Debug), + }; + compiled.with_module_async_with_environment(DEFAULT_CONFIG, initial.clone(), |mut handle| async move { + let mut values = initial; for (key, expected) in [ ("MISSING", None), ("EMPTY", Some("".to_string())), @@ -34,9 +217,9 @@ fn exercise_fixture(name: &str) { ("MAXIMUM", Some("é".repeat(4096))), ] { if let Some(value) = &expected { - set_environment(&module, key, value).await; + values.insert(key.into(), value.clone()); } - let args = product![key, expected.clone()]; + let module = publish(&handle, &values).await; let result = module .call_reducer( Identity::ZERO, @@ -45,46 +228,46 @@ fn exercise_fixture(name: &str) { None, None, "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, expected.clone()]).unwrap().into()), ) - .await; - let result = result - .map_err(anyhow::Error::from) - .and_then(|r| r.outcome.into_result()); - assert!( - result.is_ok(), - "{name} {key}: {result:?}; module log: {}", - handle.read_log(None).await - ); - let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(expected.clone())); + .unwrap(); + result.outcome.into_result().unwrap(); + assert_eq!(read(&module, key).await, AlgebraicValue::from(expected.clone())); if expected.is_some() { - set_environment(&module, key, "updated").await; - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) - .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); - sql(&module, format!("DELETE env.{key}")).await; - let result = module - .call_procedure(Identity::ZERO, None, None, "read_environment", read()) - .await - .result - .unwrap() - .return_val; - assert_eq!(result, AlgebraicValue::from(None::)); + values.insert(key.into(), "updated".into()); + let module = publish(&handle, &values).await; + assert_eq!( + read(&module, key).await, + AlgebraicValue::from(Some("updated".to_string())) + ); + values.remove(key); + let module = publish(&handle, &values).await; + assert_eq!(read(&module, key).await, AlgebraicValue::from(None::)); } } if name == "environment-test" { - set_environment(&module, "HANDLER", "handler snapshot").await; + // Required values cannot be inherited from the previous publish, + // and an invalid literal cannot replace the previous configuration. + for invalid in [ + Values::new(), + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "invalid-secret-marker".into()), + ]), + ] { + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("initial-required".to_string())) + ); + } + // A same-module publish does not run init again. Its new required + // value deliberately differs from the value init asserted. + values.insert("REQUIRED".into(), "republished".into()); + values.insert("HANDLER".into(), "handler snapshot".into()); + let module = publish(&handle, &values).await; let (_, body) = module .call_http_handler( module.info.module_def.http_handler_ids_and_defs().next().unwrap().0, @@ -100,23 +283,39 @@ fn exercise_fixture(name: &str) { .await .unwrap(); assert_eq!(&body[..], b"handler snapshot"); - // This view first reads a missing key. Its dependency must survive - // absence, and normal SQL mutations must invalidate its cached row. - let read_view = || "SELECT * FROM environment_value".to_string(); - assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); - set_environment(&module, "WATCHED", "first").await; - assert_eq!( - sql(&module, read_view()).await, - vec![product![Some("first".to_string())]] - ); - set_environment(&module, "WATCHED", "second").await; + let view = "SELECT * FROM environment_value"; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { + query_strings: [view.into()].into(), + request_id: 71, + }); + handle.send(bsatn::to_vec(&subscribe).unwrap()).await.unwrap(); + let initial_update = next_message(&mut handle).await; + assert!(matches!( + initial_update, + OutboundMessage::V1(SerializableMessage::Subscribe(_)) + )); + assert_eq!(initial_update.num_rows(), Some(1)); + for value in ["first", "second"] { + values.insert("WATCHED".into(), value.into()); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![Some(value.to_string())]]); + expect_view_update(&mut handle).await; + } + let mut invalid = values.clone(); + invalid.insert("WATCHED".into(), "fail-view".into()); + let failed = handle.republish_environment(invalid).await; + assert!(failed.as_ref().is_err() || !failed.as_ref().unwrap().was_successful()); assert_eq!( - sql(&module, read_view()).await, + sql(&handle.client.module(), view).await, vec![product![Some("second".to_string())]] ); - sql(&module, "DELETE env.WATCHED".into()).await; - assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); - set_environment(&module, "LIMIT", &"x".repeat(8192)).await; + values.remove("WATCHED"); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + expect_view_update(&mut handle).await; + values.insert("LIMIT".into(), "x".repeat(8192)); + let module = publish(&handle, &values).await; for _ in 0..2 { module .call_reducer( @@ -134,44 +333,153 @@ fn exercise_fixture(name: &str) { .into_result() .unwrap(); } + check_wasm_trap_disposal(&mut handle).await; + } + if name == "module-test-ts" { + check_submodule_scope(&mut handle, &mut values).await; + } + for key in ["UNDECLARED", "A=B"] { + let result = handle + .client + .module() + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, None::]).unwrap().into()), + ) + .await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); } - let args = product!["A=B", None::]; - let result = module - .call_reducer( - Identity::ZERO, - None, - None, - None, - None, - "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), - ) - .await; - module.exit().await; - assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); }); } #[test] #[serial] -fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn rust_environment_publish_is_atomic_and_reads_follow_declared_configuration() { exercise_fixture("environment-test"); } #[test] #[serial] -fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn typescript_environment_publish_and_checked_reads() { exercise_fixture("module-test-ts"); } #[test] #[serial] -fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn cpp_environment_publish_and_checked_reads() { exercise_fixture("module-test-cpp"); } #[test] #[serial] -fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { +fn csharp_environment_publish_and_checked_reads() { exercise_fixture("module-test-cs"); } + +#[cfg(feature = "allow_loopback_http_for_tests")] +#[test] +#[serial] +fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { + use anyhow::Context as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + for explicit_tx in [false, true] { + let old = handle.client.module(); + let program = old.relational_db().program().unwrap().unwrap(); + let old_hash = program.hash; + let mut replacement = program.bytes.to_vec(); + // A valid custom section changes the exact program hash without + // changing the schema or behavior of this real Wasm module. + replacement.extend_from_slice(&[0, 3, 1, b'e', u8::from(explicit_tx)]); + let values = Values::from([ + ("REQUIRED".into(), "new-program-value".into()), + ("MODE".into(), "ready".into()), + ]); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/hold", listener.local_addr().unwrap()); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mut server = tokio::spawn(async move { + let (mut stream, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept()).await??; + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + anyhow::ensure!(request.len() < 4096, "test request exceeded its header bound"); + request.push(tokio::time::timeout(Duration::from_secs(10), stream.read_u8()).await??); + } + entered_tx + .send(()) + .map_err(|_| anyhow::anyhow!("test coordinator closed"))?; + tokio::time::timeout(Duration::from_secs(20), release_rx).await??; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + anyhow::Ok(()) + }); + let call = old.call_procedure( + Identity::ZERO, + None, + None, + "read_environment_after_http", + FunctionArgs::Bsatn(bsatn::to_vec(&product![url, explicit_tx]).unwrap().into()), + ); + let publish_while_suspended = async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .context("procedure never reached owned HTTP barrier")??; + let publish = handle.republish_program(replacement.into(), program.kind.into(), values); + let release_after_commit = async { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + if old.relational_db().program()?.unwrap().hash != old_hash { + return anyhow::Ok(()); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("replacement program did not commit while procedure was suspended")??; + release_tx.send(()).map_err(|_| anyhow::anyhow!("HTTP barrier closed")) + }; + // Drive publication even when it waits for the old procedure + // to finish; release only after the new program is committed. + let (published, released) = tokio::join!(publish, release_after_commit); + released?; + anyhow::ensure!(published?.was_successful(), "replacement publication failed"); + anyhow::Ok(()) + }; + let (result, coordinated) = tokio::join!(call, publish_while_suspended); + let server_result = match tokio::time::timeout(Duration::from_secs(35), &mut server).await { + Ok(result) => result.unwrap(), + Err(_) => { + server.abort(); + let _ = server.await; + Err(anyhow::anyhow!("owned HTTP test server did not finish")) + } + }; + // Join the owned listener and publication before reporting any + // assertion failure, so failure cannot leave a live test host. + coordinated.unwrap(); + server_result.unwrap(); + assert!(result.result.is_err(), "old procedure read the replacement environment"); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("new-program-value".to_string())) + ); + } + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md index f8aaf847553..2f91291133c 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md +++ b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md @@ -104,6 +104,12 @@ spacetime publish --delete-data For all available publishing options and flags, see the [`spacetime publish` CLI reference](../../00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md#spacetime-publish). +### Environment Variables + +Modules can declare environment variables for configuration and secrets. Each publish supplies the complete set of values from the target's `env` configuration and declared shell variables. Required values must be supplied on every publish; omitted optional values are removed. The module and its environment update atomically. + +See [Environment Variables](./00700-environment-variables.md) for declarations, SDK accessors, publishing examples, and private tables for secrets that need to change without republishing. + ## Next Steps After publishing: diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md new file mode 100644 index 00000000000..25eb6069208 --- /dev/null +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -0,0 +1,235 @@ +--- +title: Environment Variables +slug: /databases/environment-variables +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Environment Variables + +Environment variables store configuration and secrets for a database, such as API keys and deployment settings. A module declares the names it accepts and any allowed values. Each publish supplies the complete set of values for that module. Module code reads them through `ctx.env`, or `ctx.Env` in C#. + +Use environment variables for configuration that changes when a module is published. For secrets or configuration that must change without publishing, use a [private table](#dynamically-editable-or-untyped-secrets). + +This guide assumes a module set up using a quickstart, such as the [Rust quickstart](../../00100-intro/00200-quickstarts/00500-rust.md), and familiarity with [publishing](./00300-spacetime-publish.md). + +## Declare and read variables + +Declare every environment key in the module. All values are strings. A declaration can accept any string, one exact string, or a set of allowed strings. Optional declarations permit an absent value. + +These examples declare a required `API_KEY`, a required `MODE` restricted to `development` or `production`, and an optional `LOG_LEVEL` restricted to `info` or `debug`. + + + + +Pass the declaration as the `env` option to `schema`. Include the module's existing tables in the first argument if it has any. + +```typescript +import { schema, t } from 'spacetimedb/server'; + +const spacetimedb = schema( + {}, + { + env: { + API_KEY: t.string(), + MODE: t.enum('Mode', ['development', 'production']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, + } +); + +export default spacetimedb; +``` + +Inside a reducer, procedure, or view, read values from its context: + +```typescript +const apiKey: string = ctx.env.API_KEY; +const mode: 'development' | 'production' = ctx.env.MODE; +const logLevel: 'info' | 'debug' | undefined = ctx.env.LOG_LEVEL; +const checked: string | null = ctx.env.get('LOG_LEVEL'); +``` + +Within an environment declaration, a simple enum specifies allowed strings. Enums used elsewhere in the module retain their usual tagged representation. An enum with one case restricts the value to that string. Enums with payloads cannot be used as environment constraints. + + + + +Add one environment declaration to the module: + +```rust +#[spacetimedb::env] +pub struct Env { + pub API_KEY: String, + #[env(values("development", "production"))] + pub MODE: String, + #[env(values("info", "debug"))] + pub LOG_LEVEL: Option, +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```rust +let api_key: String = ctx.env.API_KEY(); +let mode: String = ctx.env.MODE(); +let log_level: Option = ctx.env.LOG_LEVEL(); +let checked: Option = ctx.env.get("LOG_LEVEL"); +``` + +`String` requires a value, and `Option` permits absence. `#[env(values(...))]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +Add one environment declaration to the module: + +```csharp +#nullable enable + +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_KEY; + [SpacetimeDB.EnvValues("development", "production")] + public string MODE; + [SpacetimeDB.EnvValues("info", "debug")] + public string? LOG_LEVEL; +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```csharp +string apiKey = ctx.Env.API_KEY; +string mode = ctx.Env.MODE; +string? logLevel = ctx.Env.LOG_LEVEL; +string? checkedValue = ctx.Env.Get("LOG_LEVEL"); +``` + +`string` requires a value, and `string?` permits absence. `[SpacetimeDB.EnvValues(...)]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +Declare the environment in a dedicated header named `environment.h`: + +```cpp +#pragma once +#include + +SPACETIMEDB_ENV( + (API_KEY, std::string), + (MODE, std::string, ("development", "production")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB SDK directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +``` + +The SDK's CMake target includes this declaration consistently in the SDK and module source files that use the context type. Including it manually in just one source file is insufficient. + +Inside a reducer or procedure, read values from its context: + +```cpp +std::string api_key = ctx.env.API_KEY(); +std::string mode = ctx.env.MODE(); +std::optional log_level = ctx.env.LOG_LEVEL(); +std::optional checked = ctx.env.get("LOG_LEVEL"); +``` + +`std::string` requires a value, and `std::optional` permits absence. The optional third element restricts the allowed strings. Supplying one string makes it an exact-value constraint. + + + + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails; it does not return an absent value. Named optional accessors return `None`, `undefined`, `null`, or `std::nullopt`, depending on the language. The TypeScript string-key getter uses `null` for absence. + +Key names are exact and case-sensitive. The getter name `get`, or `Get` in C#, is reserved: a key with that name remains accessible through the string-key getter. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx` in Rust or C++, `withTx` in TypeScript, or `WithTx` in C#. + +## Supply values when publishing + +Add non-secret defaults to the selected database target in `spacetime.json`: + +```json +{ + "database": "env-example", + "server": "http://127.0.0.1:3000", + "module-path": "./spacetimedb", + "env": { + "MODE": "development", + "LOG_LEVEL": "info" + } +} +``` + +With a local server running on port 3000, publish from the directory containing that configuration. This example uses a disposable development value: + +```bash +API_KEY='development-only-key' spacetime publish +``` + +For real credentials, supply the value through the publishing process's environment or an appropriately ignored local configuration file. Keep secrets out of checked-in configuration and module source. + +The CLI resolves each declared key in this order: + +1. A value in the publishing process's environment. +2. A value in the resolved configuration's `env` map. +3. Absence, which is accepted only for an optional declaration. + +A shell value overrides JSON even when it is an empty string or the key is absent from JSON. Already-exported variables behave the same as inline assignments. Unrelated shell variables are ignored unless the module declares their names. The CLI displays supplied key names and their sources, without printing their values. + +The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. + +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. + +### Every publish replaces the complete environment + +Publishing validates the supplied values and installs them atomically with the module, before initialization or migration. Invalid environment configuration leaves the previous module and values unchanged. Changing only values still requires publishing, and does not rerun `init` on an existing database. + +Previously stored values are **not** defaults for the next publish. Every publish must supply each required value again. An optional value omitted from all effective inputs is removed. To remove `LOG_LEVEL` in the example, remove it from every applicable configuration layer and unset any exported `LOG_LEVEL` before publishing. An empty string is a value, not a deletion instruction. + +The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. + +## Inspect published values + +The database owner and collaborators with private-table read access can inspect the environment. For the local database above: + +```bash +spacetime env list env-example --server http://127.0.0.1:3000 +spacetime env get env-example MODE --server http://127.0.0.1:3000 +``` + +`env list` prints keys only. `env get` prints the requested value and fails if it is absent. Reading a secret with `env get` therefore exposes that secret in the command's output. + +Values are stored in the private system table `st_env`. Authorized SQL reads are also supported: + +```sql +SELECT key FROM st_env; +SELECT value FROM st_env WHERE key = 'MODE'; +``` + +SQL writes to `st_env`, module-side writes, and separate CLI setters are not supported. Changes go through a publish with the complete desired environment. + +## Access and limits + +Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. + +A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. + +Keys must match `[A-Za-z_][A-Za-z0-9_]*`. The limits are 256 bytes per key, 8 KiB per value, and 256 declarations per database. Length limits count UTF-8 bytes. Values may be empty if their declaration accepts an empty string. + +## Dynamically editable or untyped secrets + +Use an ordinary [private table](../00300-tables/00400-access-permissions.md) when a secret must change without republishing, or when its keys and allowed values should not be declared in the environment schema. For example, a table with a string primary-key column and a string value column can store arbitrary secret names and values. The table itself still has typed columns; its individual keys need no environment declarations or constraints. + +Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. + +Unlike environment variables, these values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 6a065748f69..7c9ee00f3f7 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -11,6 +11,9 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime`↴](#spacetime) * [`spacetime publish`↴](#spacetime-publish) +* [`spacetime env`↴](#spacetime-env) +* [`spacetime env get`↴](#spacetime-env-get) +* [`spacetime env list`↴](#spacetime-env-list) * [`spacetime delete`↴](#spacetime-delete) * [`spacetime logs`↴](#spacetime-logs) * [`spacetime call`↴](#spacetime-call) @@ -48,6 +51,7 @@ This document contains the help content for the `spacetime` command-line program ###### **Subcommands:** * `publish` — Create and update a SpacetimeDB database +* `env` — Inspect published database environment variables * `delete` — Deletes a SpacetimeDB database * `logs` — Prints logs from a SpacetimeDB database * `call` — Invokes a function (reducer or procedure) in a database. WARNING: This command is UNSTABLE and subject to breaking changes. @@ -82,7 +86,7 @@ Create and update a SpacetimeDB database **Usage:** `spacetime publish [OPTIONS] [name|identity]` -Run `spacetime help publish` for more detailed information. +Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information. ###### **Arguments:** @@ -137,6 +141,66 @@ Run `spacetime help publish` for more detailed information. +## `spacetime env` + +Inspect published database environment variables + +**Usage:** `spacetime env ` + +###### **Subcommands:** + +* `get` — Read one published environment value +* `list` — List published environment keys (never values) + + + +## `spacetime env get` + +Read one published environment value + +**Usage:** `spacetime env get [OPTIONS] ` + +###### **Arguments:** + +* `` — The declared environment key to read +* `` — The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `--anonymous` — Perform this action with an anonymous identity +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` — Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` — Ignore project configuration when resolving the database target + + + +## `spacetime env list` + +List published environment keys (never values) + +**Usage:** `spacetime env list [OPTIONS] ` + +###### **Arguments:** + +* `` — The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `--anonymous` — Perform this action with an anonymous identity +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` — Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` — Ignore project configuration when resolving the database target + + + ## `spacetime delete` Deletes a SpacetimeDB database diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 3ff913ba0ff..accf916617a 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -3,6 +3,7 @@ import type * as Preset from '@docusaurus/preset-classic'; import rehypeShiki, { RehypeShikiOptions } from '@shikijs/rehype'; import bash from 'shiki/langs/bash.mjs'; import c from 'shiki/langs/c.mjs'; +import cmake from 'shiki/langs/cmake.mjs'; import csharp from 'shiki/langs/csharp.mjs'; import fsharp from 'shiki/langs/fsharp.mjs'; import json from 'shiki/langs/json.mjs'; @@ -158,6 +159,7 @@ const config: Config = { toml, python, c, + cmake, cpp, protobuf, fsharp, diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 07c8c6aedd8..8803534eee7 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -1,7 +1,35 @@ use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; +use std::sync::atomic::{AtomicBool, Ordering}; + +static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: String, + #[env(values("ready", "other"))] + pub MODE: String, + pub MISSING: Option, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, + pub HANDLER: Option, + pub WATCHED: Option, + pub LIMIT: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + assert_eq!(ctx.env.REQUIRED(), "initial-required"); + assert_eq!(ctx.env.MODE(), "ready"); +} #[spacetimedb::reducer] pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert!( + !VIEW_TRAP_ENTERED.load(Ordering::Relaxed), + "trapped Wasm view instance was reused" + ); assert_eq!(ctx.env.get(&key), expected); assert_eq!(ctx.as_read_only().env.get(&key), expected); assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); @@ -14,6 +42,24 @@ pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option Option { + let request = spacetimedb::http::Request::builder() + .uri(url) + .extension(spacetimedb::http::Timeout::from(spacetimedb::TimeDuration::from( + std::time::Duration::from_secs(30), + ))) + .body(()) + .unwrap(); + assert!(ctx.http.send(request).unwrap().status().is_success()); + if explicit_tx { + ctx.with_tx(|tx| tx.env.REQUIRED().into()) + } else { + ctx.env.REQUIRED().into() + } +} + #[derive(SpacetimeType)] pub struct EnvironmentValue { pub value: Option, @@ -21,9 +67,17 @@ pub struct EnvironmentValue { #[spacetimedb::view(accessor = environment_value, public)] pub fn environment_value(ctx: &AnonymousViewContext) -> Option { - Some(EnvironmentValue { - value: ctx.env.get("WATCHED"), - }) + let value = ctx.env.WATCHED(); + assert_ne!(value.as_deref(), Some("fail-view")); + Some(EnvironmentValue { value }) +} + +/// A Rust panic is a Wasm trap. The next main-instance reducer verifies that +/// the guest-global marker did not survive the failed SQL/subscription call. +#[spacetimedb::view(accessor = environment_trap, public)] +pub fn environment_trap(_ctx: &AnonymousViewContext) -> Option { + VIEW_TRAP_ENTERED.store(true, Ordering::Relaxed); + panic!("intentional environment view trap"); } /// Hand-written ABI callers cannot retain unbounded host allocations. diff --git a/modules/module-test-cpp/CMakeLists.txt b/modules/module-test-cpp/CMakeLists.txt index ae5c37ca153..eccf682be83 100644 --- a/modules/module-test-cpp/CMakeLists.txt +++ b/modules/module-test-cpp/CMakeLists.txt @@ -34,6 +34,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() +# Declare the same context type in module and SDK compilation units. +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") + # Link the SpacetimeDB library add_subdirectory(${SPACETIMEDB_CPP_LIBRARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/spacetimedb_cpp_library) target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) diff --git a/modules/module-test-cpp/environment.h b/modules/module-test-cpp/environment.h new file mode 100644 index 00000000000..e0e7e5d34f7 --- /dev/null +++ b/modules/module-test-cpp/environment.h @@ -0,0 +1,9 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (MISSING, std::optional), + (EMPTY, std::optional), + (UTF8, std::optional), + (NUL, std::optional), + (MAXIMUM, std::optional) +) diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index 1102ef8be5f..fe47c4ce2a1 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -722,6 +722,8 @@ SPACETIMEDB_HTTP_ROUTER(router) { } SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + // Parentheses avoid the existing enum helper macro named EMPTY(). + if ((ctx.env.EMPTY)() != ctx.env.get("EMPTY")) LOG_PANIC("named environment mismatch"); if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); return Ok(); } diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs index 1798d7bda44..b0630c30ec3 100644 --- a/modules/module-test-cs/EnvironmentTests.cs +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -3,11 +3,22 @@ namespace SpacetimeDB.Modules.ModuleTestCs; using SpacetimeDB; +[SpacetimeDB.Env] +public partial struct ModuleEnvironmentSchema +{ + public string? MISSING; + public string? EMPTY; + public string? UTF8; + public string? NUL; + public string? MAXIMUM; +} + public static partial class EnvironmentTests { [Reducer] public static void expect_environment(ReducerContext ctx, string key, string? expected) { + if (ctx.Env.EMPTY != ctx.Env.Get("EMPTY")) throw new Exception("named environment mismatch"); if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); } diff --git a/modules/module-test-ts/src/environment_sys.d.ts b/modules/module-test-ts/src/environment_sys.d.ts new file mode 100644 index 00000000000..8bf1660c106 --- /dev/null +++ b/modules/module-test-ts/src/environment_sys.d.ts @@ -0,0 +1,4 @@ +// Raw host ABI used to verify that SDK context changes cannot grant authority. +declare module 'spacetime:sys@2.3' { + export function env_get(key: string): string | null; +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 10520d2b72c..2d903356b00 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -249,7 +249,13 @@ const spacetimedb = schema({ ), tableToRemove: table({ name: 'table_to_remove' }, { id: t.u32() }), lib: libSubmodule, -}); +}, { env: { + MISSING: t.string().optional(), + EMPTY: t.string().optional(), + UTF8: t.string().optional(), + NUL: t.string().optional(), + MAXIMUM: t.string().optional(), +} }); export default spacetimedb; // ───────────────────────────────────────────────────────────────────────────── @@ -549,14 +555,30 @@ export const libHello = spacetimedb.httpHandler((ctx, req) => { return libSubmodule.libHello(ctx.as.lib, req); }); +// Ordinary JS delegation retains this root host entry, even with ctx.as.lib. +// Direct host dispatch to lib.envReadHandler is the separate denied case. +export const envReadChildHandler = spacetimedb.httpHandler((ctx, req) => + libSubmodule.envReadHandler(ctx.as.lib, req) +); + +// Root entries must use the checked accessor too; returning raw st_env SQL is +// forbidden even when the same entry could legitimately call ctx.env.get. +export const envReadRootSqlView = spacetimedb.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => libSubmodule.uncheckedEnvironmentQuery(ctx.from.player) +); + export const router = spacetimedb.httpRouter( - new Router().get('/get', getSimple).get('/lib-hello', libHello) + new Router().get('/get', getSimple).get('/lib-hello', libHello).get('/env-child', envReadChildHandler) ); // Dedicated environment ABI integration exercised by crates/testing. export const expect_environment = spacetimedb.reducer( { key: t.string(), expected: t.option(t.string()) }, (ctx, { key, expected }) => { + if (libSubmodule.readRootEnvironmentHelper() !== ctx.env.get('EMPTY')) throw new Error('helper environment scope mismatch'); + if (ctx.env.EMPTY !== (ctx.env.get('EMPTY') ?? undefined)) throw new Error('named environment mismatch'); if (ctx.env.get(key) !== (expected ?? null)) { throw new Error('environment value mismatch'); } diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts index 3c2832d602c..ff4c56b3be8 100644 --- a/modules/module-test-ts/src/lib_submodule.ts +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -1,4 +1,6 @@ +/// import { schema, table, t, SyncResponse } from 'spacetimedb/server'; +import { env_get } from 'spacetime:sys@2.3'; const libData = table( { name: 'libData', public: true }, @@ -26,3 +28,37 @@ export const libCount = libSubmoduleSchema.procedure(t.u64(), ctx => export const libHello = libSubmoduleSchema.httpHandler((_ctx, _req) => { return new SyncResponse('Hello from lib submodule!'); }); + +// Ordinary helpers retain their caller's host scope. Exported module callbacks +// below are entered through the lib namespace and must be rejected by the host. +export function readRootEnvironmentHelper(): string | null { + return env_get('EMPTY'); +} +export const envReadReducer = libSubmoduleSchema.reducer(() => { + env_get('EMPTY'); +}); +export const envReadProcedure = libSubmoduleSchema.procedure(t.string(), () => + env_get('EMPTY') ?? '' +); +export const envReadInTx = libSubmoduleSchema.procedure(t.string(), ctx => + ctx.withTx(() => env_get('EMPTY') ?? '') +); +export const envReadView = libSubmoduleSchema.view( + { public: true }, t.array(t.object('EnvReadRow', { value: t.string() })), () => [{ value: env_get('EMPTY') ?? '' }] +); +export const envReadHandler = libSubmoduleSchema.httpHandler(() => + new SyncResponse(env_get('EMPTY') ?? '') +); + +// Deliberately forge module-returned SQL through an ordinary query object's +// runtime brand. SDK types are not a security boundary for this host feature. +export function uncheckedEnvironmentQuery(source: object) { + return Object.assign(Object.create(source), { + toSql: () => 'SELECT * FROM st_env', + }) as { key: string; value: string }[]; +} +export const envReadSqlView = libSubmoduleSchema.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => uncheckedEnvironmentQuery(ctx.from.libData) +); From 05a805392953bd44b8674b4e37ca6ea0f89251f4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 19:25:01 -0400 Subject: [PATCH 03/19] Clarify module library terminology in environment docs --- .../00100-databases/00300-spacetime-publish.md | 2 +- .../00100-databases/00700-environment-variables.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md index 2f91291133c..dd805b69ecf 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md +++ b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md @@ -108,7 +108,7 @@ For all available publishing options and flags, see the [`spacetime publish` CLI Modules can declare environment variables for configuration and secrets. Each publish supplies the complete set of values from the target's `env` configuration and declared shell variables. Required values must be supplied on every publish; omitted optional values are removed. The module and its environment update atomically. -See [Environment Variables](./00700-environment-variables.md) for declarations, SDK accessors, publishing examples, and private tables for secrets that need to change without republishing. +See [Environment Variables](./00700-environment-variables.md) for declarations, reading values in module code, publishing examples, and private tables for secrets that need to change without republishing. ## Next Steps diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 25eb6069208..34161f9095a 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -126,13 +126,13 @@ SPACETIMEDB_ENV( ) ``` -In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB SDK directory: +In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB module library directory: ```cmake set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") ``` -The SDK's CMake target includes this declaration consistently in the SDK and module source files that use the context type. Including it manually in just one source file is insufficient. +The library's CMake target includes this declaration consistently in the library and module source files that use the context type. Including it manually in just one source file is insufficient. Inside a reducer or procedure, read values from its context: From 648ae253462403b342469bcddcddcd640d33d6ed Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 21:52:58 -0400 Subject: [PATCH 04/19] Address environment API and CLI review feedback --- .../include/spacetimedb/abi/abi.h | 7 +- crates/bindings-cpp/src/abi/wasi_shims.cpp | 6 +- .../bindings-csharp/Runtime/Internal/FFI.cs | 6 +- crates/bindings-csharp/Runtime/bindings.c | 2 +- .../Runtime/build/SpacetimeDB.Runtime.targets | 3 +- crates/bindings-macro/src/environment.rs | 81 ++------- crates/bindings-sys/src/lib.rs | 3 +- .../src/server/environment.ts | 13 +- .../bindings-typescript/src/server/sys.d.ts | 4 +- .../tests/environment.test.ts | 4 +- crates/bindings-typescript/vitest.config.ts | 2 +- crates/bindings/src/lib.rs | 3 +- crates/bindings/src/rt.rs | 36 ++++ crates/bindings/tests/environment.rs | 4 +- crates/bindings/tests/pass/environment.rs | 17 +- crates/bindings/tests/ui/environment_types.rs | 38 ++++ .../tests/ui/environment_types.stderr | 129 +++++++++++++ crates/cli/src/lib.rs | 1 + crates/cli/src/schema_extract.rs | 172 ++++++++++++++++++ crates/cli/src/schema_extract/tests.rs | 143 +++++++++++++++ crates/cli/src/subcommands/env.rs | 30 ++- crates/cli/src/subcommands/generate.rs | 18 +- crates/cli/src/subcommands/publish.rs | 36 +++- .../src/subcommands/publish/environment.rs | 101 +--------- .../subcommands/publish/environment/tests.rs | 131 +------------ .../cli/src/subcommands/publish/wire_tests.rs | 56 ++++++ crates/cli/src/subcommands/sql.rs | 2 +- crates/core/src/host/v8/syscall/mod.rs | 3 +- crates/core/src/host/v8/syscall/v2.rs | 4 +- crates/core/src/host/wasm_common.rs | 3 +- .../tests/standalone/cli/environment.rs | 20 +- modules/environment-test/src/lib.rs | 7 +- .../module-test-ts/src/environment_sys.d.ts | 2 +- modules/module-test-ts/src/lib_submodule.ts | 2 +- 34 files changed, 712 insertions(+), 377 deletions(-) create mode 100644 crates/bindings/tests/ui/environment_types.rs create mode 100644 crates/bindings/tests/ui/environment_types.stderr create mode 100644 crates/cli/src/schema_extract.rs create mode 100644 crates/cli/src/schema_extract/tests.rs create mode 100644 crates/cli/src/subcommands/publish/wire_tests.rs diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 285974cff3c..502cd408be3 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,9 +39,8 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern -// ABI10.6 is reserved for the separate invocation-authority extension. -#define STDB_IMPORT_10_7(name) \ - __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; @@ -63,7 +62,7 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { -STDB_IMPORT_10_7(env_get) +STDB_IMPORT_10_6(env_get) Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); // ===== Table and Index Management ===== diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index ec461c8d51d..a698b4177f7 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -9,9 +9,9 @@ // with these standalone shim definitions. Use a distinct C++ name for the raw // host logging import; its WebAssembly signature is the same eight i32 values. extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void wasi_console_log(uint8_t level, const uint8_t* target_ptr, size_t target_len, - const uint8_t* filename_ptr, size_t filename_len, uint32_t line_number, - const uint8_t* message_ptr, size_t message_len); +void wasi_console_log(uint8_t level, const uint8_t* target_ptr, uint32_t target_len, + const uint8_t* filename_ptr, uint32_t filename_len, uint32_t line_number, + const uint8_t* message_ptr, uint32_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index cc3e8cf624b..8957759d8eb 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,16 +109,16 @@ internal static partial class FFI #endif ; - const string StdbNamespace10_7 = + const string StdbNamespace10_6 = #if EXPERIMENTAL_WASM_AOT || NET10_0_OR_GREATER - "spacetime_10.7" + "spacetime_10.6" #else "bindings" #endif ; [WasmImportLinkage] - [LibraryImport(StdbNamespace10_7)] + [LibraryImport(StdbNamespace10_6)] public static unsafe partial CheckedStatus env_get( byte* key, uint keyLen, diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index f4d3635a5f2..b877876b37f 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,7 +135,7 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION -#define SPACETIME_MODULE_VERSION "spacetime_10.7" +#define SPACETIME_MODULE_VERSION "spacetime_10.6" IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); #undef SPACETIME_MODULE_VERSION diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index 95c2f1cfdd5..45d4b570d58 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -65,8 +65,7 @@ - - + diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index 8377037082e..e58899d96db 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -2,7 +2,7 @@ use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::ext::IdentExt as _; use syn::punctuated::Punctuated; -use syn::{Fields, GenericArgument, ItemStruct, LitStr, PathArguments, Token, Type}; +use syn::{Fields, ItemStruct, LitStr, Token}; pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result { if !args.is_empty() { @@ -32,7 +32,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result> = None; for attr in field.attrs.iter().filter(|attr| attr.path().is_ident("env")) { attr.parse_nested_meta(|meta| { @@ -71,28 +71,23 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::OPTIONAL, }), ); if name == "get" { continue; } - let (return_type, body) = if optional { - ( - quote!(::std::option::Option<::std::string::String>), - quote!(::spacetimedb::Environment::get(self, #name)), - ) - } else { - ( - quote!(::std::string::String), - quote!(::spacetimedb::Environment::get(self, #name).expect(concat!("required environment key is missing: ", #name))), - ) - }; signatures.push(quote! { #[doc = concat!("Read the declared environment key `", #name, "` through the checked host ABI.")] - fn #ident(&self) -> #return_type; + fn #ident(&self) -> #ty; + }); + methods.push(quote! { + fn #ident(&self) -> #ty { + <#ty as ::spacetimedb::rt::EnvironmentValue>::get(self, #name) + } }); - methods.push(quote!(fn #ident(&self) -> #return_type { #body })); } let vis = &item.vis; let access = format_ident!("{}Access", item.ident.unraw()); @@ -119,65 +114,13 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result syn::Result { - let Type::Path(path) = ty else { - return Err(syn::Error::new_spanned(ty, "expected String or Option")); - }; - if path.qself.is_none() { - let segments: Vec<_> = path - .path - .segments - .iter() - .map(|segment| segment.ident.to_string()) - .collect(); - let names: Vec<_> = segments.iter().map(String::as_str).collect(); - if matches!(names.as_slice(), ["String"] | ["std" | "alloc", "string", "String"]) - && path - .path - .segments - .iter() - .all(|segment| matches!(segment.arguments, PathArguments::None)) - { - return Ok(false); - } - if matches!(names.as_slice(), ["Option"] | ["std" | "core", "option", "Option"]) - && path - .path - .segments - .iter() - .rev() - .skip(1) - .all(|segment| matches!(segment.arguments, PathArguments::None)) - && let PathArguments::AngleBracketed(arguments) = &path.path.segments.last().unwrap().arguments - && let [GenericArgument::Type(inner)] = arguments.args.iter().collect::>().as_slice() - && !optional_string(inner)? - { - return Ok(true); - } - } - Err(syn::Error::new_spanned( - ty, - "expected String or Option; aliases and other types are not env constraints", - )) -} - #[cfg(test)] mod tests { use super::*; #[test] - fn rejects_non_string_nested_optional_empty_union_and_invalid_names() { + fn rejects_empty_union_invalid_names_and_unsupported_struct_shapes() { for item in [ - quote!( - struct Env { - VALUE: bool, - } - ), - quote!( - struct Env { - VALUE: Option>, - } - ), quote!( struct Env { #[env(values())] diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 2a3bd77b454..c683d4c8a59 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,8 +883,7 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } - // ABI10.6 is reserved for the separate invocation-authority extension. - #[link(wasm_import_module = "spacetime_10.7")] + #[link(wasm_import_module = "spacetime_10.6")] unsafe extern "C" { /// Read a UTF-8 environment value. Writes INVALID for a missing key; /// present empty strings have a valid BytesSource. Returns ordinary errno. diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts index 203c608e08c..05458efae8a 100644 --- a/crates/bindings-typescript/src/server/environment.ts +++ b/crates/bindings-typescript/src/server/environment.ts @@ -1,4 +1,4 @@ -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; import type { Environment, EnvironmentSchema } from '../lib/environment'; import type { EnvironmentDeclaration, @@ -7,6 +7,11 @@ import type { } from '../lib/autogen/types'; import { OptionBuilder, StringBuilder } from '../lib/type_builders'; +// These UTF-8 byte/count limits match spacetimedb_lib::environment. +const MAX_ENV_KEY_BYTES = 256; +const MAX_ENV_VALUE_BYTES = 8 * 1024; +const MAX_ENV_VARS = 256; + /** Values are not cached: transaction and procedure reads retain host semantics. */ export const environment: Environment = new Proxy( Object.freeze( @@ -33,13 +38,13 @@ export function environmentDeclarations( schema: EnvironmentSchema ): EnvironmentDeclaration[] { const entries = Object.entries(schema); - if (entries.length > 256) + if (entries.length > MAX_ENV_VARS) throw new TypeError('Too many environment declarations'); const bytes = new TextEncoder(); return entries.map(([name, definition]) => { if ( !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || - bytes.encode(name).length > 256 + bytes.encode(name).length > MAX_ENV_KEY_BYTES ) { throw new TypeError('Invalid environment declaration name'); } @@ -68,7 +73,7 @@ export function environmentDeclarations( throw new TypeError( `Environment '${name}' enum cases must have names` ); - if (bytes.encode(variant.name).length > 8192) + if (bytes.encode(variant.name).length > MAX_ENV_VALUE_BYTES) throw new TypeError(`Environment '${name}' literal is too long`); return variant.name; }); diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index 1f74debd2fc..77a9dacaf4e 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -124,9 +124,7 @@ declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } -// sys2.2 is reserved for the separate invocation-authority extension. - -declare module 'spacetime:sys@2.3' { +declare module 'spacetime:sys@2.2' { /** Null means missing; an empty string is a present value. */ export function env_get(key: string): string | null; } diff --git a/crates/bindings-typescript/tests/environment.test.ts b/crates/bindings-typescript/tests/environment.test.ts index b78713d49e9..34fc8ae65c9 100644 --- a/crates/bindings-typescript/tests/environment.test.ts +++ b/crates/bindings-typescript/tests/environment.test.ts @@ -6,9 +6,9 @@ import { environmentDeclarations, } from '../src/server/environment'; import type { EnvironmentSchema } from '../src/lib/environment'; -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; -vi.mock('spacetime:sys@2.3', async importOriginal => ({ +vi.mock('spacetime:sys@2.2', async importOriginal => ({ ...(await importOriginal()), env_get: vi.fn(), })); diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index e5503037be6..f532d437b74 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,7 +14,7 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, - { find: 'spacetime:sys@2.3', replacement: sysMock }, + { find: 'spacetime:sys@2.2', replacement: sysMock }, ], }, test: { diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 1589eb54ad5..04be557fdf2 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -921,7 +921,8 @@ pub use query_builder::{Query, RawQuery}; /// Declare the complete publish-time environment schema and generate named accessors. /// -/// Fields must be `String` or `Option`; `#[env(values("a", "b"))]` +/// Fields must resolve to `String` or `Option`, including type aliases; +/// `#[env(values("a", "b"))]` /// constrains exact strings. Values are supplied on every publish, never in metadata. /// The macro generates an `EnvAccess` extension trait for a struct named `Env`. /// Import that trait when the declaration lives in a different Rust module. diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index bc5b0d8fae1..486ba90ba62 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,6 +917,42 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } +mod environment_value_sealed { + pub trait Sealed {} + + impl Sealed for String {} + impl Sealed for Option {} +} + +/// The compiler resolves declaration types, including aliases, before selecting +/// their metadata and accessor. Sealing keeps the accepted types identical to +/// the host's string and optional-string environment model. +#[doc(hidden)] +#[diagnostic::on_unimplemented(message = "environment fields must be `String` or `Option`")] +pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { + const OPTIONAL: bool; + + fn get(environment: &crate::Environment, key: &str) -> Self; +} + +impl EnvironmentValue for String { + const OPTIONAL: bool = false; + + fn get(environment: &crate::Environment, key: &str) -> Self { + environment + .get(key) + .unwrap_or_else(|| panic!("required environment key is missing: {key}")) + } +} + +impl EnvironmentValue for Option { + const OPTIONAL: bool = true; + + fn get(environment: &crate::Environment, key: &str) -> Self { + environment.get(key) + } +} + /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] pub fn register_environment(declarations: fn() -> Vec) { diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs index 5bffb7924ff..6636b9160ef 100644 --- a/crates/bindings/tests/environment.rs +++ b/crates/bindings/tests/environment.rs @@ -1,4 +1,6 @@ #[test] fn environment_declaration_accessors_compile_with_exact_types() { - trybuild::TestCases::new().pass("tests/pass/environment.rs"); + let tests = trybuild::TestCases::new(); + tests.pass("tests/pass/environment.rs"); + tests.compile_fail("tests/ui/environment_types.rs"); } diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs index 87a53d23aea..c0f80526ea4 100644 --- a/crates/bindings/tests/pass/environment.rs +++ b/crates/bindings/tests/pass/environment.rs @@ -1,14 +1,23 @@ #![deny(warnings)] +use std::option::Option as Maybe; +use std::string::String as RenamedString; + +type RequiredAlias = RenamedString; +type OptionalAlias = Maybe; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + #[spacetimedb::env] pub struct Env { - pub REQUIRED: String, + pub REQUIRED: RequiredAlias, #[env(values("false", "true"))] pub FLAG: String, #[env(values(""))] - pub OPTIONAL: Option, - pub get: Option, - pub r#type: String, + pub OPTIONAL: OptionalAlias, + pub get: Maybe, + pub r#type: RenamedString, } fn reads(env: spacetimedb::Environment) { diff --git a/crates/bindings/tests/ui/environment_types.rs b/crates/bindings/tests/ui/environment_types.rs new file mode 100644 index 00000000000..4fa3e3aae0b --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.rs @@ -0,0 +1,38 @@ +mod shadowed_string { + pub struct String; + + #[spacetimedb::env] + pub struct ShadowedString { + pub VALUE: String, + } +} + +mod shadowed_option { + pub struct Option(T); + + #[spacetimedb::env] + pub struct ShadowedOption { + pub VALUE: Option, + } +} + +#[spacetimedb::env] +pub struct Unsupported { + pub BOOL: bool, + pub NESTED: Option>, + // Even without a generated named accessor, metadata must check the type. + pub get: u32, +} + +struct Custom; + +// External code cannot extend the set of supported environment types. +impl spacetimedb::rt::EnvironmentValue for Custom { + const OPTIONAL: bool = false; + + fn get(_: &spacetimedb::Environment, _: &str) -> Self { + Self + } +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_types.stderr b/crates/bindings/tests/ui/environment_types.stderr new file mode 100644 index 00000000000..3c7346dcb4e --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.stderr @@ -0,0 +1,129 @@ +error[E0277]: the trait bound `Custom: spacetimedb::rt::environment_value_sealed::Sealed` is not satisfied + --> tests/ui/environment_types.rs:30:44 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^ unsatisfied trait bound + | +help: the trait `spacetimedb::rt::environment_value_sealed::Sealed` is not implemented for `Custom` + --> tests/ui/environment_types.rs:27:1 + | +27 | struct Custom; + | ^^^^^^^^^^^^^ +help: the following other types implement trait `spacetimedb::rt::environment_value_sealed::Sealed` + --> src/rt.rs + | + | impl Sealed for String {} + | ^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + | impl Sealed for Option {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` +note: required by a bound in `EnvironmentValue` + --> src/rt.rs + | + | pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `EnvironmentValue` + = note: `EnvironmentValue` is a "sealed trait", because to implement it you also need to implement `spacetimedb::rt::environment_value_sealed::Sealed`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it + = help: the following types implement the trait: + std::string::String + std::option::Option + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:6:20 + | + 6 | pub VALUE: String, + | ^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_string::String` + --> tests/ui/environment_types.rs:2:5 + | + 2 | pub struct String; + | ^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:15:20 + | +15 | pub VALUE: Option, + | ^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_option::Option` + --> tests/ui/environment_types.rs:11:5 + | +11 | pub struct Option(T); + | ^^^^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:21:15 + | +21 | pub BOOL: bool, + | ^^^^ the trait `EnvironmentValue` is not implemented for `bool` + | +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:22:17 + | +22 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^^^ the trait `EnvironmentValue` is not implemented for `std::option::Option>` + | +help: the trait `EnvironmentValue` is implemented for `std::option::Option` + --> src/rt.rs + | + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: environment fields must be `String` or `Option` + --> tests/ui/environment_types.rs:24:14 + | +24 | pub get: u32, + | ^^^ the trait `EnvironmentValue` is not implemented for `u32` + | +help: the following other types implement trait `EnvironmentValue` + --> tests/ui/environment_types.rs:30:1 + | +30 | impl spacetimedb::rt::EnvironmentValue for Custom { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` + | + ::: src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 96bb1034bfe..5357e8d067b 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -4,6 +4,7 @@ mod config; pub(crate) mod detect; mod edit_distance; mod errors; +mod schema_extract; pub mod spacetime_config; mod subcommands; mod tasks; diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs new file mode 100644 index 00000000000..c00481ae40f --- /dev/null +++ b/crates/cli/src/schema_extract.rs @@ -0,0 +1,172 @@ +//! Local schema extraction shared by publish and generate. Each invocation +//! inspects a private copy of exact bounded input bytes, bounds its output and +//! lifetime, and owns the child through kill/wait on failure or cancellation. +use anyhow::{ensure, Context}; +use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; +use spacetimedb_schema::def::ModuleDef; +use std::{ + path::{Path, PathBuf}, + process::Stdio, + time::Duration, +}; +use tokio::io::AsyncReadExt; + +fn extractor_path() -> anyhow::Result { + std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone")) +} + +/// Keep generate's synchronous injectable function API. Its small dedicated +/// runtime works both inside and outside a caller's Tokio runtime; process +/// ownership and validation are identical to publish's async path. +pub(crate) fn from_path(path: &Path) -> anyhow::Result { + let bytes = read_program(path)?; + let host_type = match path.extension().and_then(|ext| ext.to_str()) { + Some("wasm") => "Wasm", + Some("js") => "Js", + _ => anyhow::bail!("Cannot determine module type from file extension"), + }; + inspect_blocking(extractor_path()?, bytes, host_type.into()) +} + +fn inspect_blocking(extractor: PathBuf, bytes: Vec, host_type: String) -> anyhow::Result { + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(inspect_with(extractor, bytes, host_type, INSPECT_TIMEOUT)) + }) + .join() + .map_err(|_| anyhow::anyhow!("Local module inspection thread failed"))? +} + +pub(crate) fn read_program(path: &std::path::Path) -> anyhow::Result> { + use std::io::Read; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, + "Module exceeds publish size limit" + ); + Ok(bytes) +} + +const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; +const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Inspect exactly the artifact bytes that will be uploaded. The private copy +/// prevents path replacement between inspection and upload, including --bin-path. +/// This invokes only local extraction, never a server or a saved CLI context. +pub(crate) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { + let extractor = extractor_path()?; + inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await +} + +pub(crate) async fn inspect_with( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, +) -> anyhow::Result { + inspect_observed(extractor, program, host_type, deadline, Observation::default()).await +} + +#[derive(Default)] +struct Observation { + #[cfg(test)] + started: Option>, + #[cfg(test)] + reaped: Option>, +} +impl Observation { + fn started(&mut self, _pid: Option) { + #[cfg(test)] + if let Some(send) = self.started.take() { + let _ = send.send(_pid.expect("new child has a PID")); + } + } + fn reaped(&mut self, _status: std::process::ExitStatus) { + #[cfg(test)] + if let Some(send) = self.reaped.take() { + let _ = send.send(_status); + } + } +} + +async fn inspect_observed( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, + mut observation: Observation, +) -> anyhow::Result { + let (mut send, mut recv) = tokio::sync::oneshot::channel(); + // This owner retains the child and private file until actual reaping, even + // when its caller drops while extraction or stdout reading is in progress. + tokio::spawn(async move { + let result = async { + let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; + let module = dir.path().join("module"); + tokio::fs::write(&module, program) + .await + .context("Cannot prepare module inspection input")?; + let mut child = tokio::process::Command::new(extractor) + .arg("extract-schema") + .arg(&module) + .arg("--host-type") + .arg(host_type.to_ascii_lowercase()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("Cannot start local module schema inspection")?; + observation.started(child.id()); + let mut output = Vec::new(); + let mut stdout = child + .stdout + .take() + .context("Module inspection stdout unavailable")? + .take(MAX_SCHEMA_BYTES + 1); + let result = tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), + result = tokio::time::timeout(deadline, async { + stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; + ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); + let status = child.wait().await.context("Cannot reap local module inspector")?; + observation.reaped(status); + ensure!(status.success(), "Local module schema inspection failed"); + Ok(()) + }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), + }; + if result.is_err() { + // Queue termination, then retain ownership through positive reaping. + let _ = child.start_kill(); + let status = child + .wait() + .await + .context("Cannot reap failed local module inspector")?; + observation.reaped(status); + } + result?; + // Neither parser nor validation diagnostics may echo schema literals. + let SerdeWrapper::(raw) = serde_json::from_slice(&output) + .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; + let schema = + ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; + Ok(schema) + } + .await; + let _ = send.send(result); + }); + (&mut recv).await.context("Local module inspection owner failed")? +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs new file mode 100644 index 00000000000..d8c6daea04a --- /dev/null +++ b/crates/cli/src/schema_extract/tests.rs @@ -0,0 +1,143 @@ +use super::*; +use spacetimedb_lib::environment::{ + EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration, EnvironmentSchema, +}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +// These fixtures invoke only a locally generated executable in an owned tempdir. +// No CLI config, server credentials or user environment are imported. +#[cfg(unix)] +fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("inspector"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + (dir, path) +} + +#[cfg(unix)] +#[tokio::test] +async fn local_inspection_passes_exact_bytes_host_and_requires_success() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let result = inspect_with( + extractor, + b"exact-artifact".to_vec(), + "Js".into(), + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(result.environment(), &schema()); + drop(dir); + let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); + assert!( + inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .is_err() + ); +} + +#[cfg(unix)] +#[tokio::test(flavor = "current_thread")] +async fn synchronous_generate_adapter_uses_same_exact_byte_protocol_inside_a_runtime() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (_dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = wasm ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = generate-exact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let module = inspect_blocking(extractor, b"generate-exact".to_vec(), "Wasm".into()).unwrap(); + assert_eq!(module.environment(), &schema()); +} + +#[cfg(unix)] +#[tokio::test] +async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { + for script in [ + "printf 'generated-inspector-secret'", + "exec /usr/bin/head -c 16777217 /dev/zero", + ] { + let (_dir, extractor) = inspector(script); + let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!format!("{error:#}").contains("generated-inspector-secret")); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn timeout_and_caller_drop_produce_positive_wait_receipts() { + for cancel in [false, true] { + let (_dir, extractor) = inspector("while :; do :; done"); + let (started, started_receipt) = tokio::sync::oneshot::channel(); + let (reaped, reaped_receipt) = tokio::sync::oneshot::channel(); + let operation = tokio::spawn(inspect_observed( + extractor, + b"input".to_vec(), + "Wasm".into(), + if cancel { + Duration::from_secs(30) + } else { + Duration::from_millis(300) + }, + Observation { + started: Some(started), + reaped: Some(reaped), + }, + )); + assert!( + tokio::time::timeout(Duration::from_secs(5), started_receipt) + .await + .unwrap() + .unwrap() + > 0 + ); + if cancel { + operation.abort(); + let _ = operation.await; + } else { + assert!(operation.await.unwrap().is_err()); + } + let status = tokio::time::timeout(Duration::from_secs(5), reaped_receipt) + .await + .unwrap() + .unwrap(); + assert!(!status.success()); + } +} diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index 58876664f48..20bc11a5b9b 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -149,10 +149,21 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { Query::List => { ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); values.sort_unstable(); + let rows = values + .into_iter() + .map(|key| Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![key])); + let table = sql::build_table( + spacetimedb_lib::sats::satn::PsqlClient::SpacetimeDB, + &result.schema, + rows, + )?; + Ok(format!("{table}\n")) + } + Query::Get(_) => { + ensure!(values.len() == 1, "Environment key is absent"); + Ok(format!("{}\n", values[0])) } - Query::Get(_) => ensure!(values.len() == 1, "Environment key is absent"), } - Ok(values.into_iter().map(|v| format!("{v}\n")).collect()) } #[cfg(test)] @@ -196,8 +207,12 @@ mod tests { #[test] fn list_projects_keys_and_rejects_unexpected_secret_columns() { assert_eq!( - render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List).unwrap(), - "A\nZ\n" + render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List) + .unwrap() + .lines() + .map(str::trim) + .collect::>(), + ["key", "-----", "\"A\"", "\"Z\""] ); let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); assert!(!format!("{err:#}").contains("generated-secret-sentinel")); @@ -211,7 +226,12 @@ mod tests { async fn actual_loopback_queries_and_error_redaction() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; for (query, status, response, expected) in [ - (Query::List, "200 OK", body("key", vec![vec!["KEY"]]), Some("KEY\n")), + ( + Query::List, + "200 OK", + body("key", vec![vec!["KEY"]]), + Some(" key \n-------\n \"KEY\" \n"), + ), ( Query::Get("KEY".into()), "200 OK", diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index e312bf65ea9..14b39212458 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -10,11 +10,10 @@ use spacetimedb_codegen::{ UnrealCpp, AUTO_GENERATED_PREFIX, }; use spacetimedb_lib::de::serde::DeserializeWrapper; -use spacetimedb_lib::{sats, RawModuleDef}; +use spacetimedb_lib::RawModuleDef; use spacetimedb_schema; use spacetimedb_schema::def::ModuleDef; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use crate::common_args::parse_optional_dotnet_version; use crate::spacetime_config::{ @@ -22,7 +21,7 @@ use crate::spacetime_config::{ }; use crate::tasks::csharp::dotnet_format; use crate::tasks::rust::rustfmt; -use crate::util::{resolve_sibling_binary, y_or_n}; +use crate::util::y_or_n; use crate::Config; use crate::{build, common_args}; use clap::builder::PossibleValue; @@ -750,18 +749,7 @@ impl Language { pub type ExtractDescriptions = fn(&Path) -> anyhow::Result; pub fn extract_descriptions(wasm_file: &Path) -> anyhow::Result { - let bin_path = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") - .map(PathBuf::from) - .map(Ok) - .unwrap_or_else(|| resolve_sibling_binary("spacetimedb-standalone"))?; - let child = Command::new(&bin_path) - .arg("extract-schema") - .arg(wasm_file) - .stdout(Stdio::piped()) - .spawn() - .with_context(|| format!("failed to spawn {}", bin_path.display()))?; - let sats::serde::SerdeWrapper::(module) = serde_json::from_reader(child.stdout.unwrap())?; - Ok(module.try_into()?) + crate::schema_extract::from_path(wasm_file) } #[cfg(test)] diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index b734b37cba8..5162dbeca2c 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -1,4 +1,6 @@ mod environment; +#[cfg(test)] +mod wire_tests; use anyhow::{ensure, Context}; use clap::Arg; @@ -322,7 +324,28 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), .help("Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only)") ) .arg(common_args::dotnet_version()) - .after_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.") + .after_help("Run `spacetime help publish` for more detailed information.") + .after_long_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.") +} + +fn publication_body( + module: &spacetimedb_schema::def::ModuleDef, + bytes: Vec, + environment: std::collections::BTreeMap, +) -> anyhow::Result<(&'static str, Vec)> { + if module.environment_declared() { + let body = spacetimedb_client_api_messages::publish::PublishRequest { + module: bytes, + environment, + } + .encode()?; + Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) + } else { + anyhow::ensure!(environment.is_empty(), "Module does not declare environment keys"); + // Preserve older servers for ordinary modules. Explicit empty ENV + // declarations still use the envelope, expressing replacement intent. + Ok(("application/octet-stream", bytes)) + } } fn confirm_and_clear( @@ -647,16 +670,9 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let payload = spacetimedb_client_api_messages::publish::PublishRequest { - module: program_bytes, - environment: environment.values, - } - .encode()?; + let (content_type, payload) = publication_body(&module_schema, program_bytes, environment.values)?; let res = builder - .header( - reqwest::header::CONTENT_TYPE, - spacetimedb_client_api_messages::publish::CONTENT_TYPE, - ) + .header(reqwest::header::CONTENT_TYPE, content_type) .body(payload) .send() .await?; diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs index 17fc67efb62..4427ae52a65 100644 --- a/crates/cli/src/subcommands/publish/environment.rs +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -1,16 +1,11 @@ //! Resolve a complete, declared environment without consulting stored values. use std::collections::BTreeMap; use std::ffi::OsString; -use std::path::PathBuf; -use std::process::Stdio; -use std::time::Duration; +pub(super) use crate::schema_extract::{inspect, read_program}; use anyhow::{ensure, Context}; use serde_json::Value; use spacetimedb_lib::environment::EnvironmentSchema; -use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; -use spacetimedb_schema::def::ModuleDef; -use tokio::io::AsyncReadExt; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Source { @@ -82,99 +77,5 @@ pub(super) fn resolve( Ok(resolved) } -pub(super) fn read_program(path: &std::path::Path) -> anyhow::Result> { - use std::io::Read; - let mut bytes = Vec::new(); - std::fs::File::open(path)? - .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) - .read_to_end(&mut bytes)?; - ensure!( - bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, - "Module exceeds publish size limit" - ); - Ok(bytes) -} - -const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; -const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); - -/// Inspect exactly the artifact bytes that will be uploaded. The private copy -/// prevents path replacement between inspection and upload, including --bin-path. -/// This invokes only local extraction, never a server or a saved CLI context. -pub(super) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { - let extractor = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") - .map(PathBuf::from) - .map(Ok) - .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone"))?; - inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await -} - -async fn inspect_with( - extractor: PathBuf, - program: Vec, - host_type: String, - deadline: Duration, -) -> anyhow::Result { - let (mut send, mut recv) = tokio::sync::oneshot::channel(); - // This owner retains the child and private file until actual reaping, even - // when its caller drops while extraction or stdout reading is in progress. - tokio::spawn(async move { - let result = async { - let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; - let module = dir.path().join("module"); - tokio::fs::write(&module, program) - .await - .context("Cannot prepare module inspection input")?; - let mut child = tokio::process::Command::new(extractor) - .arg("extract-schema") - .arg(&module) - .arg("--host-type") - .arg(host_type.to_ascii_lowercase()) - .env_clear() - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .kill_on_drop(true) - .spawn() - .context("Cannot start local module schema inspection")?; - let mut output = Vec::new(); - let mut stdout = child - .stdout - .take() - .context("Module inspection stdout unavailable")? - .take(MAX_SCHEMA_BYTES + 1); - let result = tokio::select! { - biased; - _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), - result = tokio::time::timeout(deadline, async { - stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; - ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); - let status = child.wait().await.context("Cannot reap local module inspector")?; - ensure!(status.success(), "Local module schema inspection failed"); - Ok(()) - }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), - }; - if result.is_err() { - // Queue termination, then retain ownership through positive reaping. - let _ = child.start_kill(); - child - .wait() - .await - .context("Cannot reap failed local module inspector")?; - } - result?; - // Neither parser nor validation diagnostics may echo schema literals. - let SerdeWrapper::(raw) = serde_json::from_slice(&output) - .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; - let schema = - ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; - Ok(schema) - } - .await; - let _ = send.send(result); - }); - (&mut recv).await.context("Local module inspection owner failed")? -} - #[cfg(test)] mod tests; diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs index 5af04aa6877..7ed2bca6f2e 100644 --- a/crates/cli/src/subcommands/publish/environment/tests.rs +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -1,5 +1,6 @@ use super::*; use spacetimedb_lib::environment::{EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration}; +use std::{path::PathBuf, time::Duration}; fn schema() -> EnvironmentSchema { EnvironmentSchema::new(vec![ @@ -153,134 +154,6 @@ fn non_utf8_declared_shell_value_is_rejected_without_bytes() { assert!(error.to_string().contains("UTF-8")); } -// These fixtures invoke only a locally generated executable in an owned tempdir. -// No CLI config, server credentials or user environment are imported. -#[cfg(unix)] -fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { - use std::os::unix::fs::PermissionsExt; - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("inspector"); - std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); - (dir, path) -} - -#[cfg(unix)] -#[tokio::test] -async fn local_inspection_passes_exact_bytes_host_and_requires_success() { - use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; - let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], - }); - let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); - let (dir, extractor) = inspector(&format!( - "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") - )); - let result = inspect_with( - extractor, - b"exact-artifact".to_vec(), - "Js".into(), - Duration::from_secs(5), - ) - .await - .unwrap(); - assert_eq!(result.environment(), &schema()); - drop(dir); - let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); - assert!( - inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) - .await - .is_err() - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { - for script in [ - "printf 'generated-inspector-secret'", - "exec /usr/bin/head -c 16777217 /dev/zero", - ] { - let (_dir, extractor) = inspector(script); - let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) - .await - .unwrap_err(); - assert!(!format!("{error:#}").contains("generated-inspector-secret")); - } -} - -#[cfg(unix)] -#[tokio::test] -async fn inspection_timeout_and_dropped_waiter_reap_exact_child() { - // A shell-only busy loop has no descendants and records the exact child PID. - // kill(0) via the owned process is not used for proof: wait for /bin/kill -0 - // to report ESRCH after our owner has called wait, including cancellation. - async fn wait_pid(path: &std::path::Path) -> String { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Ok(pid) = tokio::fs::read_to_string(path).await - && !pid.trim().is_empty() - { - break pid; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap() - } - async fn gone(pid: &str) { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let status = tokio::process::Command::new("/bin/kill") - .args(["-0", pid.trim()]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .await - .unwrap(); - if !status.success() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - } - for cancel in [false, true] { - let (dir, extractor) = inspector("placeholder"); - let pid_file = dir.path().join("pid"); - std::fs::write( - &extractor, - format!( - "#!/bin/sh\nprintf '%s' \"$$\" > '{}'\nwhile :; do :; done\n", - pid_file.display() - ), - ) - .unwrap(); - let operation = tokio::spawn(inspect_with( - extractor, - b"input".to_vec(), - "Wasm".into(), - if cancel { - Duration::from_secs(30) - } else { - Duration::from_millis(300) - }, - )); - let pid = wait_pid(&pid_file).await; - if cancel { - operation.abort(); - let _ = operation.await; - } else { - assert!(operation.await.unwrap().is_err()); - } - gone(&pid).await; - } -} - #[tokio::test] #[ignore = "requires explicit locally built ENV-aware standalone and declared Wasm fixture paths"] async fn actual_precompiled_declarations_are_inspected_without_server_or_values() { @@ -289,7 +162,7 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( let module = PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_MODULE").expect("explicit module path")); assert!(extractor.is_absolute() && module.is_absolute()); let program = read_program(&module).unwrap(); - let inspected = inspect_with(extractor, program, "Wasm".into(), INSPECT_TIMEOUT) + let inspected = crate::schema_extract::inspect_with(extractor, program, "Wasm".into(), Duration::from_secs(60)) .await .unwrap(); let schema = inspected.environment(); diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs new file mode 100644 index 00000000000..70781417800 --- /dev/null +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -0,0 +1,56 @@ +use super::*; +use spacetimedb_lib::{ + db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}, + RawModuleDef, +}; +use spacetimedb_schema::def::ModuleDef; +use std::collections::BTreeMap; + +fn schema(declared: bool) -> ModuleDef { + let mut sections = vec![RawModuleDefV10Section::Typespace(Default::default())]; + if declared { + sections.push(RawModuleDefV10Section::Environment(vec![])); + } + ModuleDef::try_from(RawModuleDef::V10(RawModuleDefV10 { sections })).unwrap() +} + +#[test] +fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { + let bytes = b"exact selected module bytes\0\xff".to_vec(); + let (kind, body) = publication_body(&schema(false), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, "application/octet-stream"); + assert_eq!(body, bytes); + let (kind, body) = publication_body(&schema(true), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); + let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); + assert_eq!(envelope.module, bytes); + assert!(envelope.environment.is_empty()); + let error = publication_body( + &schema(false), + bytes, + BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), + ) + .unwrap_err(); + assert!(!format!("{error:#}").contains("secret-sentinel")); +} + +#[test] +fn short_help_is_concise_and_long_help_explains_environment_replacement() { + let short = cli().render_help().to_string(); + let long = cli() + .render_long_help() + .to_string() + .split_whitespace() + .collect::>() + .join(" "); + assert!(!short.contains("Every publish replaces")); + assert!(short.contains("spacetime help publish")); + for text in [ + "Every publish replaces the complete declared environment", + "including empty strings", + "Optional values omitted", + "--env selects config file layers", + ] { + assert!(long.contains(text), "missing long-help guidance: {text}"); + } +} diff --git a/crates/cli/src/subcommands/sql.rs b/crates/cli/src/subcommands/sql.rs index d0a75e17d76..5b782d7106b 100644 --- a/crates/cli/src/subcommands/sql.rs +++ b/crates/cli/src/subcommands/sql.rs @@ -287,7 +287,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } /// Generates a [`tabled::Table`] from a schema and rows, using the style of a psql table. -fn build_table( +pub(super) fn build_table( client: PsqlClient, schema: &ProductType, rows: impl Iterator>, diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index 029d5836282..467e7f26562 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,8 +62,7 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), - // sys2.2 is reserved for invocation authority. - (2, 3) => Ok(v2::sys_v2_3(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 8fb376ec7af..3e25145fc74 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,8 +169,8 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } -pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { - create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.2", (with_sys_result, AbiCall::EnvGet, env_get),) } fn env_get<'s>( diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index dc8baa44227..f5ab623fec3 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -444,8 +444,7 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, - // ABI10.6 is reserved for invocation authority. - "spacetime_10.7"::env_get, + "spacetime_10.6"::env_get, } $link_async! { diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index aed684811ce..1f080bd1da4 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -173,8 +173,8 @@ impl Fixture { ) } - fn list(&self) -> String { - self.success( + fn list(&self) -> Vec { + let output = self.success( &[ "env", "list", @@ -184,7 +184,13 @@ impl Fixture { "--no-config", ], &[], - ) + ); + let mut lines = output.lines().map(str::trim); + assert_eq!(lines.next(), Some("key")); + lines + .filter(|line| !line.is_empty() && !line.chars().all(|c| c == '-')) + .map(|line| serde_json::from_str::(line).unwrap()) + .collect() } fn typed(&self, required: &str, mode: &str, rest: [Option<&str>; 4]) { @@ -334,7 +340,7 @@ fn cli_environment_layers_shell_and_exact_precompiled_declarations() { ); let mut keys = KEYS.to_vec(); keys.sort_unstable(); - assert_eq!(f.list(), format!("{}\n", keys.join("\n"))); + assert_eq!(f.list(), keys); let initial = f.sql("SELECT required, mode FROM initial_environment"); assert!(initial.status.success()); let initial = String::from_utf8(initial.stdout).unwrap(); @@ -353,7 +359,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { )); f.published(&[], &[]); f.typed("replacement-sentinel", "other", [None; 4]); - assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); assert!(!f .command( &[ @@ -386,7 +392,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { f.typed("replacement-sentinel", "other", [None; 4]); } // Invalid local configuration must not prevent an explicit read-only target. - assert_eq!(f.list(), "SMOKE_MODE\nSMOKE_REQUIRED\n"); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); for statement in [ "SET env.SMOKE_REQUIRED = 'bypass'", "DELETE env.SMOKE_REQUIRED", @@ -435,5 +441,5 @@ fn cli_environment_initial_rejection_clear_and_omitted_payload() { f.config(None); f.wasm = modules::precompiled_module("noop"); f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); - assert_eq!(f.list(), ""); + assert!(f.list().is_empty()); } diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 8803534eee7..8d6efa37a11 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -3,12 +3,15 @@ use std::sync::atomic::{AtomicBool, Ordering}; static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); +type RequiredString = String; +type OptionalString = Option; + #[spacetimedb::env] pub struct Env { - pub REQUIRED: String, + pub REQUIRED: RequiredString, #[env(values("ready", "other"))] pub MODE: String, - pub MISSING: Option, + pub MISSING: OptionalString, pub EMPTY: Option, pub UTF8: Option, pub NUL: Option, diff --git a/modules/module-test-ts/src/environment_sys.d.ts b/modules/module-test-ts/src/environment_sys.d.ts index 8bf1660c106..d2abea72f80 100644 --- a/modules/module-test-ts/src/environment_sys.d.ts +++ b/modules/module-test-ts/src/environment_sys.d.ts @@ -1,4 +1,4 @@ // Raw host ABI used to verify that SDK context changes cannot grant authority. -declare module 'spacetime:sys@2.3' { +declare module 'spacetime:sys@2.2' { export function env_get(key: string): string | null; } diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts index ff4c56b3be8..466770be2ff 100644 --- a/modules/module-test-ts/src/lib_submodule.ts +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -1,6 +1,6 @@ /// import { schema, table, t, SyncResponse } from 'spacetimedb/server'; -import { env_get } from 'spacetime:sys@2.3'; +import { env_get } from 'spacetime:sys@2.2'; const libData = table( { name: 'libData', public: true }, From 7d26b8d8a6866bcc7c4454b885eecd5914fffeeb Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 23:34:12 -0400 Subject: [PATCH 05/19] Fix ENV CI coordination and restore C++ logging names --- crates/bindings-cpp/src/abi/wasi_shims.cpp | 15 ++- .../Runtime/Internal/Module.cs | 6 +- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- .../commands/workflow-coordinator/src/main.rs | 95 ++++++++++++++++++- 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index a698b4177f7..37548c40d99 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -4,14 +4,13 @@ #include #include -// Keep this pure ABI translation unit independent of SDK opaque types: their -// standard-library helpers include wasi/api.h in Emscripten, which conflicts -// with these standalone shim definitions. Use a distinct C++ name for the raw -// host logging import; its WebAssembly signature is the same eight i32 values. + +// SpacetimeDB imports we need for console output +// Import from spacetime_10.0 module as required by SpacetimeDB ABI extern "C" __attribute__((import_module("spacetime_10.0"), import_name("console_log"))) -void wasi_console_log(uint8_t level, const uint8_t* target_ptr, uint32_t target_len, - const uint8_t* filename_ptr, uint32_t filename_len, uint32_t line_number, - const uint8_t* message_ptr, uint32_t message_len); +void console_log(uint8_t log_level, const uint8_t* target, uint32_t target_len, + const uint8_t* filename, uint32_t filename_len, uint32_t line_number, + const uint8_t* message, uint32_t message_len); // Helper macro for string literals #define CSTR(s) (uint8_t*)s, sizeof(s) - 1 @@ -151,7 +150,7 @@ __wasi_errno_t __wasi_fd_write(__wasi_fd_t fd, const __wasi_ciovec_t* iovs, // Make a single console_log call with the complete message uint8_t log_level = (fd == STDERR_FILENO) ? 1 : 2; // 1=WARN, 2=INFO - wasi_console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, + console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, buffer, offset); // Clean up heap allocation if needed diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 035291a2f0d..438c3439146 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -87,7 +87,8 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); - internal void RegisterEnvironment(EnvironmentDeclaration declaration) => environment.Add(declaration); + internal void RegisterEnvironment(EnvironmentDeclaration declaration) => + environment.Add(declaration); internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => viewPrimaryKeyDefs.Add(new RawViewPrimaryKeyDefV10(viewSourceName, [.. columns])); @@ -431,7 +432,8 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } - public static void RegisterEnvironment(EnvironmentDeclaration declaration) => moduleDef.RegisterEnvironment(declaration); + public static void RegisterEnvironment(EnvironmentDeclaration declaration) => + moduleDef.RegisterEnvironment(declaration); public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => moduleDef.RegisterViewPrimaryKey(viewSourceName, columns); diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index aa589df3b5a..be8dd719582 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 6); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) diff --git a/tools/ci/commands/workflow-coordinator/src/main.rs b/tools/ci/commands/workflow-coordinator/src/main.rs index 2ff8101d999..521b19245d0 100644 --- a/tools/ci/commands/workflow-coordinator/src/main.rs +++ b/tools/ci/commands/workflow-coordinator/src/main.rs @@ -197,6 +197,8 @@ struct Repository { #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] struct PullRequestRef { sha: String, + #[serde(rename = "ref")] + branch_name: String, repo: Option, } @@ -340,10 +342,29 @@ fn related_private_pr(public_pr_number: Option) -> Result 1 { - bail!("found multiple open linked private PRs"); + let public_branch = if pulls.len() > 1 { + Some(pull_request(PUBLIC_REPO, public_pr_number)?.head.branch_name) + } else { + None + }; + select_related_private_pr(pulls, public_branch.as_deref()) +} + +fn select_related_private_pr(mut pulls: Vec, public_branch: Option<&str>) -> Result> { + if pulls.len() <= 1 { + return Ok(pulls.pop()); + } + + // Timeline references include historical links and links to other layers of + // a PR stack. A unique shared branch name identifies the companion PR; the + // exact public-submodule SHA is still checked before selecting its CI run. + if let Some(public_branch) = public_branch.filter(|branch| !branch.is_empty()) { + pulls.retain(|pull| pull.head.branch_name == public_branch); + if pulls.len() == 1 { + return Ok(pulls.pop()); + } } - Ok(pulls.pop()) + bail!("found multiple open linked private PRs without a unique matching head branch") } fn resolve_private_source(public_pr_number: Option) -> Result { @@ -574,6 +595,7 @@ mod tests { state: "open".to_owned(), head: PullRequestRef { sha: "private-sha".to_owned(), + branch_name: "tyler/environment-variables".to_owned(), repo: Some(Repository { full_name: PRIVATE_REPO.to_owned(), }), @@ -581,6 +603,73 @@ mod tests { } } + #[test] + fn related_private_pr_prefers_the_unique_exact_public_head_branch() { + let matching = pull(); + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/environment-variables-followup".to_owned(); + for candidates in [ + vec![matching.clone(), downstream.clone()], + vec![downstream, matching.clone()], + ] { + assert_eq!( + select_related_private_pr(candidates, Some("tyler/environment-variables")).unwrap(), + Some(matching.clone()) + ); + } + } + + #[test] + fn related_private_pr_preserves_absent_and_single_candidate_behavior() { + assert_eq!(select_related_private_pr(Vec::new(), None).unwrap(), None); + for public_branch in [None, Some("unrelated-branch")] { + assert_eq!( + select_related_private_pr(vec![pull()], public_branch).unwrap(), + Some(pull()) + ); + } + } + + #[test] + fn related_private_pr_rejects_multiple_matching_branches() { + let mut duplicate = pull(); + duplicate.number = 43; + assert!(select_related_private_pr(vec![pull(), duplicate], Some("tyler/environment-variables")).is_err()); + } + + #[test] + fn related_private_pr_rejects_missing_or_unmatched_public_branch() { + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/v10-abi-extensions".to_owned(); + for public_branch in [None, Some(""), Some("tyler/unrelated")] { + assert!(select_related_private_pr(vec![pull(), downstream.clone()], public_branch).is_err()); + } + } + + #[test] + fn selected_companion_still_requires_the_exact_public_submodule() { + let selected = select_related_private_pr(vec![pull()], None).unwrap().unwrap(); + assert!(ensure_public_submodule_matches(selected.number, "old-public-sha", "requested-public-sha").is_err()); + ensure_public_submodule_matches(selected.number, "requested-public-sha", "requested-public-sha").unwrap(); + } + + #[test] + fn pull_request_head_branch_uses_the_github_ref_field() { + let parsed: PullRequest = serde_json::from_value(serde_json::json!({ + "number": 42, + "state": "open", + "head": { + "sha": "private-sha", + "ref": "tyler/environment-variables", + "repo": { "full_name": PRIVATE_REPO } + } + })) + .unwrap(); + assert_eq!(parsed, pull()); + } + fn run(id: u64, title: &str, created_at: &str) -> WorkflowRun { WorkflowRun { id, From 60c17d018b5c66cb1ecac17c8d1154974d0a96cf Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:00:45 -0400 Subject: [PATCH 06/19] Fix environment CI isolation and update handler diagnostics --- crates/bindings-typescript/package.json | 4 +- crates/bindings/tests/ui/http_handlers.stderr | 2 +- crates/cli/src/spacetime_config.rs | 2 +- .../cli/src/spacetime_config/environment.rs | 48 ++++++++++++++++++- crates/smoketests/src/lib.rs | 27 +++++++++-- .../tests/standalone/cli/environment.rs | 18 +++---- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index 8e6f94787de..db598440673 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -25,14 +25,14 @@ "scripts": { "build:js": "tsup", "build:types": "tsc -p tsconfig.build.json", - "build": "pnpm -s build:js && pnpm -s build:types", + "build": "pnpm run build:js && pnpm run build:types", "format": "prettier . --write --ignore-path ../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "test": "vitest run", "test:typecheck": "vitest typecheck --run", "coverage": "vitest run --coverage", "brotli-size": "brotli-size dist/index.js", - "size": "pnpm -s build && size-limit", + "size": "pnpm run build && size-limit", "generate:moduledef": "cargo run -p spacetimedb-codegen --example regen-typescript-moduledef && prettier --write src/lib/autogen", "generate:client-api": "cargo run -p generate-client-api && prettier --write src/sdk/client_api", "generate:test-app": "pnpm --filter @clockworklabs/test-app generate", diff --git a/crates/bindings/tests/ui/http_handlers.stderr b/crates/bindings/tests/ui/http_handlers.stderr index 960b3cdeae9..c28936ac12e 100644 --- a/crates/bindings/tests/ui/http_handlers.stderr +++ b/crates/bindings/tests/ui/http_handlers.stderr @@ -175,7 +175,7 @@ error[E0609]: no field `db` on type `&mut HandlerContext` 53 | let _rows = ctx.db.test_table().iter(); | ^^ unknown field | - = note: available fields are: `timestamp`, `http` + = note: available fields are: `env`, `timestamp`, `http` error[E0308]: mismatched types --> tests/ui/http_handlers.rs:66:4 diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index cce271510ee..32d3030082a 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -855,7 +855,7 @@ impl SpacetimeConfig { let value = environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; let config: Self = environment::decode_config(value) - .map_err(|_| anyhow::anyhow!("Invalid configuration structure in {}", path.display()))?; + .with_context(|| format!("Invalid configuration structure in {}", path.display()))?; Ok(config) } diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs index ecaa3218684..c4493835376 100644 --- a/crates/cli/src/spacetime_config/environment.rs +++ b/crates/cli/src/spacetime_config/environment.rs @@ -92,10 +92,26 @@ pub(super) fn parse(content: &str) -> anyhow::Result { /// Deserialize through JSON text so Serde's flattened-field buffer does not /// receive visit_u128 from Value's deserializer. That buffer cannot represent /// u128, while the arbitrary-precision JSON parser preserves its decimal token. -/// Diagnostics deliberately discard the original error, which may quote values. +/// Diagnostics discard values while retaining a bounded, ordinary unknown field +/// name, which is useful for correcting misspelled configuration options. pub(super) fn decode_config(value: Value) -> anyhow::Result { let encoded = serde_json::to_vec(&value).map_err(|_| anyhow::anyhow!("Invalid configuration structure"))?; - serde_json::from_slice(&encoded).map_err(|_| anyhow::anyhow!("Invalid configuration structure")) + serde_json::from_slice(&encoded).map_err(|error| { + let diagnostic = error.to_string(); + if let Some((field, suffix)) = diagnostic + .strip_prefix("unknown field `") + .and_then(|message| message.split_once('`')) + && suffix.starts_with(", expected ") + && !field.is_empty() + && field.len() <= 64 + && field + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return anyhow::anyhow!("unknown field `{field}`"); + } + anyhow::anyhow!("Invalid configuration structure") + }) } fn is_space(ch: char) -> bool { @@ -283,4 +299,32 @@ mod tests { serde_json::from_value(serde_json::json!({"env":{"A":1},"children":[{"env":null}]})).unwrap(); assert!(config.collect_all_targets_with_inheritance()[1].fields["env"].is_null()); } + + #[test] + fn unknown_configuration_fields_are_actionable_without_exposing_values() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + let secret = "private-configuration-value-sentinel"; + let value = serde_json::json!({"dev": {"run_command": secret}, "env": {"TOKEN": secret}}); + std::fs::write(&path, serde_json::to_vec(&value).unwrap()).unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let diagnostic = format!("{error:#}"); + assert!(diagnostic.contains("unknown field `run_command`")); + assert!(!diagnostic.contains(secret)); + } + + for field in [ + "control\ncharacters".to_owned(), + "x".repeat(65), + "quote`injection".to_owned(), + ] { + let error = decode_config(serde_json::json!({"dev": {field: secret}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } + let error = decode_config(serde_json::json!({"dev": {"run": [secret]}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } } diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 3ec8f5b7141..f9ff97e7fe7 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -802,6 +802,7 @@ pub struct SmoketestBuilder { autopublish: bool, pg_port: Option, server_url_override: Option, + isolated_local_server: bool, cli_path: Option, } @@ -828,6 +829,7 @@ impl SmoketestBuilder { autopublish: true, pg_port: None, server_url_override: None, + isolated_local_server: false, cli_path: None, } } @@ -837,6 +839,14 @@ impl SmoketestBuilder { self } + /// Start an owned local server with a fresh CLI configuration, even in a + /// remote test job. Do not copy the remote job's login or base configuration. + /// This cannot be combined with an explicit server URL. + pub fn isolated_local_server(mut self) -> Self { + self.isolated_local_server = true; + self + } + /// Uses a specific CLI binary instead of the pre-built CLI for this test. pub fn cli_path(mut self, path: impl AsRef) -> Self { self.cli_path = Some(path.as_ref().to_path_buf()); @@ -923,6 +933,15 @@ impl SmoketestBuilder { /// Panics if the CLI/standalone binaries haven't been built or are stale. /// Run `cargo smoketest prepare` to build binaries before running tests. pub fn build(self) -> Smoketest { + assert!( + !self.isolated_local_server || self.server_url_override.is_none(), + "isolated_local_server cannot use an explicit remote server URL" + ); + let inherited_remote = if self.isolated_local_server { + None + } else { + remote_server_url() + }; // Check binaries first - this will panic with a helpful message if missing/stale if self.cli_path.is_none() { let _ = ensure_binaries_built(); @@ -936,7 +955,7 @@ impl SmoketestBuilder { // Check if we're running against a remote server let (guard, server_url, data_dir_fixture) = if let Some(fixture) = self.data_dir_fixture.as_ref() { - if self.server_url_override.is_some() || remote_server_url().is_some() { + if self.server_url_override.is_some() || inherited_remote.is_some() { panic!("data_dir_fixture requires a local server managed by the smoketest harness"); } @@ -963,7 +982,7 @@ impl SmoketestBuilder { } else if let Some(url) = self.server_url_override { eprintln!("[REMOTE] Using explicit server URL: {}", url); (None, url, None) - } else if let Some(remote_url) = remote_server_url() { + } else if let Some(remote_url) = inherited_remote { eprintln!("[REMOTE] Using remote server: {}", remote_url); (None, remote_url, None) } else { @@ -998,7 +1017,9 @@ impl SmoketestBuilder { let module_name = format!("smoketest_module_{}", random_string()); let config_path = project_dir.path().join("config.toml"); - if let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") { + if !self.isolated_local_server + && let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") + { fs::copy(&base_config_path, &config_path) .unwrap_or_else(|err| panic!("failed to copy base smoketest config from {base_config_path}: {err:#}")); } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 1f080bd1da4..2ba96d8adb9 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -27,23 +27,23 @@ struct Fixture { impl Fixture { fn new() -> Self { - // Check before the harness can connect or copy a prior login. These are - // standalone-only tests, including when accidentally run in a remote job. - for key in [ + // Private CI supplies remote cluster settings to the same test binary. + // This fixture must still create its own server and fresh credentials, + // without changing those settings for other tests in the process. + let remote_settings = [ "SPACETIME_REMOTE_SERVER", "SPACETIME_USE_AUTH_HOST", "SPACETIME_SMOKETEST_BASE_CONFIG_PATH", - ] { - assert!( - std::env::var_os(key).is_none(), - "ENV smoke test requires isolated local settings ({key})" - ); - } + ]; + let inherited = remote_settings.map(std::env::var_os); let test = Smoketest::builder() + .isolated_local_server() .precompiled_module("environment-publish") .autopublish(false) .build(); + assert_eq!(remote_settings.map(std::env::var_os), inherited); assert!(test.guard.is_some()); + assert!(!test.config_path.exists(), "local fixture copied inherited credentials"); let address = test .server_url .strip_prefix("http://") From 0293b627a361e2b5e182c469edee1a14dda588d6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:08:51 -0400 Subject: [PATCH 07/19] Document direct HTTP environment publishing and retain query diagnostics --- crates/query/src/lib.rs | 4 +- .../00200-http-api/00300-database.md | 54 ++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 717f9d38092..9225393eb34 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length") + bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,7 +59,7 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length") + bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") } match parse_and_type_sql(sql, tx, auth)? { diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index e6137bea0df..2066a7682c2 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -41,7 +41,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -76,7 +76,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -98,6 +98,56 @@ If a database with the given name exists, but the identity provided in the `Auth } } ``` +### Publishing with environment values + +Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish+json` with this JSON body: + +```json +{ + "module": "", + "environment": { + "API_KEY": "development-only-key", + "MODE": "development" + } +} +``` + +`module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. + +For example, this Python script publishes a Wasm module to a local server. Set `SPACETIME_TOKEN` to a token authorized to publish and `API_KEY` to the complete configuration's required value. Change the module path to the artifact you built. + +```python +import base64 +import json +import os +from pathlib import Path +from urllib.request import Request, urlopen + +body = json.dumps({ + "module": base64.b64encode(Path("module.wasm").read_bytes()).decode("ascii"), + "environment": { + "API_KEY": os.environ["API_KEY"], + "MODE": "development", + }, +}).encode("utf-8") + +request = Request( + "http://127.0.0.1:3000/v1/database/env-example?host_type=wasm", + data=body, + method="PUT", + headers={ + "Authorization": "Bearer " + os.environ["SPACETIME_TOKEN"], + "Content-Type": "application/vnd.spacetimedb.publish+json", + }, +) +with urlopen(request, timeout=60) as response: + print(response.read().decode("utf-8")) +``` + +Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. + +Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. + ## `GET /v1/database/:name_or_identity` Get a database's identity, owner identity, host type, number of replicas and a hash of its WASM module. From 03cb28c18db7a8e93471819093f1fbd70b9b1be5 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:30:03 -0400 Subject: [PATCH 08/19] Add typed Rust environment enums and show values in env list --- crates/bindings-macro/src/environment.rs | 9 +- .../bindings-macro/src/environment/value.rs | 193 ++++++++++++++++++ crates/bindings-macro/src/lib.rs | 5 + crates/bindings/src/lib.rs | 40 +++- crates/bindings/src/rt.rs | 80 ++++++-- crates/bindings/tests/environment.rs | 1 + .../bindings/tests/environment_enum_values.rs | 78 +++++++ crates/bindings/tests/pass/environment.rs | 17 ++ crates/bindings/tests/ui/environment_enum.rs | 54 +++++ .../bindings/tests/ui/environment_enum.stderr | 85 ++++++++ crates/bindings/tests/ui/environment_types.rs | 11 - .../tests/ui/environment_types.stderr | 113 +++------- crates/cli/src/subcommands/env.rs | 93 +++++---- .../tests/standalone/cli/environment.rs | 15 +- crates/testing/tests/environment.rs | 76 +++++++ .../00700-environment-variables.md | 36 +++- .../00100-cli-reference.md | 4 +- modules/environment-test/src/lib.rs | 42 +++- 18 files changed, 777 insertions(+), 175 deletions(-) create mode 100644 crates/bindings-macro/src/environment/value.rs create mode 100644 crates/bindings/tests/environment_enum_values.rs create mode 100644 crates/bindings/tests/ui/environment_enum.rs create mode 100644 crates/bindings/tests/ui/environment_enum.stderr diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index e58899d96db..4640bcb40dc 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -1,3 +1,5 @@ +pub(crate) mod value; + use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::ext::IdentExt as _; @@ -59,7 +61,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::AnyString), + None => quote!(<#ty as ::spacetimedb::rt::EnvironmentValue>::constraint()), Some([value]) => { quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())) } @@ -69,6 +71,11 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::with_constraint(#constraint)) + } else { + constraint + }; declarations.push( quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentDeclaration { name: #name.into(), diff --git a/crates/bindings-macro/src/environment/value.rs b/crates/bindings-macro/src/environment/value.rs new file mode 100644 index 00000000000..2608cc10f69 --- /dev/null +++ b/crates/bindings-macro/src/environment/value.rs @@ -0,0 +1,193 @@ +use proc_macro2::TokenStream; +use quote::quote; +use std::collections::BTreeSet; +use syn::ext::IdentExt as _; +use syn::{Data, DeriveInput, Fields, LitStr}; + +pub(crate) fn derive(item: DeriveInput) -> syn::Result { + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment value enums cannot be generic", + )); + } + if let Some(attr) = item.attrs.iter().find(|attr| attr.path().is_ident("env")) { + return Err(syn::Error::new_spanned( + attr, + "place `#[env(value = \"...\")]` on enum variants", + )); + } + let Data::Enum(data) = &item.data else { + return Err(syn::Error::new_spanned( + &item.ident, + "EnvironmentValue requires an enum with unit variants", + )); + }; + if data.variants.is_empty() || data.variants.len() > 256 { + return Err(syn::Error::new_spanned( + &item.ident, + "environment value enums require 1 to 256 variants", + )); + } + let mut variants = Vec::new(); + let mut values = Vec::new(); + let mut unique = BTreeSet::new(); + for variant in &data.variants { + if !matches!(variant.fields, Fields::Unit) { + return Err(syn::Error::new_spanned( + &variant.fields, + "environment value variants cannot have payloads", + )); + } + let mut value: Option = None; + for attr in variant.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("value") || value.is_some() { + return Err(meta.error("expected one value = \"literal\" mapping")); + } + value = Some(meta.value()?.parse()?); + Ok(()) + })?; + if value.is_none() { + return Err(syn::Error::new_spanned(attr, "expected value = \"literal\" mapping")); + } + } + let value = value.unwrap_or_else(|| LitStr::new(&variant.ident.unraw().to_string(), variant.ident.span())); + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + if !unique.insert(value.value()) { + return Err(syn::Error::new_spanned( + value, + "environment variants must map to distinct strings", + )); + } + variants.push(&variant.ident); + values.push(value); + } + let constraint = match values.as_slice() { + [value] => quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())), + values => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf(::std::vec![#(#values.into()),*]) + ), + }; + let ident = &item.ident; + Ok(quote! { + impl ::spacetimedb::rt::EnvironmentValue for #ident { + const OPTIONAL: bool = false; + + fn constraint() -> ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint { + #constraint + } + + fn from_environment(value: ::std::option::Option<::std::string::String>, key: &str) -> Self { + match value.as_deref() { + #(::std::option::Option::Some(#values) => Self::#variants,)* + ::std::option::Option::Some(_) => ::core::panic!("environment value does not match its declared enum: {}", key), + ::std::option::Option::None => ::core::panic!("required environment key is missing: {}", key), + } + } + } + impl ::spacetimedb::rt::RequiredEnvironmentValue for #ident {} + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_invalid_enum_shapes_mappings_and_limits() { + for input in [ + quote!( + struct Value; + ), + quote!( + enum Value {} + ), + quote!( + enum Value { + Item(T), + } + ), + quote!( + enum Value { + Item(String), + } + ), + quote!( + enum Value { + Item { field: String }, + } + ), + quote!( + enum Value { + #[env()] + Item, + } + ), + quote!( + enum Value { + #[env(values("x"))] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x", value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x")] + #[env(value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "Same")] + First, + Same, + } + ), + quote!( + #[env(value = "x")] + enum Value { + Item, + } + ), + ] { + assert!(derive(syn::parse2(input).unwrap()).is_err()); + } + let oversized = LitStr::new(&"é".repeat(4097), proc_macro2::Span::call_site()); + assert!(derive(syn::parse_quote!( + enum Value { + #[env(value = #oversized)] + Item, + } + )) + .is_err()); + let variants = (0..257).map(|n| quote::format_ident!("V{n}")); + assert!(derive(syn::parse_quote!(enum Value { #(#variants),* })).is_err()); + } + + #[test] + fn accepts_exact_strings_and_ordinary_enum_attributes() { + let output = derive(syn::parse_quote! { + #[derive(Debug, PartialEq)] + enum Value { + #[env(value = "in progress")] InProgress, + #[env(value = "")] Empty, + #[env(value = "héllo\0世界")] Unicode, + r#type, + } + }) + .unwrap(); + syn::parse2::(output).unwrap(); + } +} diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index 02aaacc4028..37069ccbf2c 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -15,6 +15,11 @@ pub fn env(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream { ok_or_compile_error(|| environment::expand(args.into(), syn::parse(item)?)) } +#[proc_macro_derive(EnvironmentValue, attributes(env))] +pub fn derive_environment_value(item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::value::derive(syn::parse(item)?)) +} + mod http; mod procedure; diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 04be557fdf2..190065c5e1f 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -921,28 +921,52 @@ pub use query_builder::{Query, RawQuery}; /// Declare the complete publish-time environment schema and generate named accessors. /// -/// Fields must resolve to `String` or `Option`, including type aliases; -/// `#[env(values("a", "b"))]` -/// constrains exact strings. Values are supplied on every publish, never in metadata. -/// The macro generates an `EnvAccess` extension trait for a struct named `Env`. -/// Import that trait when the declaration lives in a different Rust module. +/// Fields may be `String`, enums deriving [`EnvironmentValue`], or `Option` of +/// either, including type aliases. Values are supplied on every publish, never +/// in metadata. The macro generates an `EnvAccess` extension trait for a struct +/// named `Env`; import that trait if the declaration lives in another module. /// The name `get` is reserved for generic checked access. /// /// ```no_run +/// #[derive(spacetimedb::EnvironmentValue)] +/// pub enum LogLevel { +/// #[env(value = "debug")] +/// Debug, +/// #[env(value = "info")] +/// Info, +/// } /// #[spacetimedb::env] /// pub struct Env { /// pub API_KEY: String, -/// #[env(values("debug", "info"))] -/// pub LOG_LEVEL: Option, +/// pub LOG_LEVEL: Option, /// } /// fn read(ctx: &spacetimedb::ReducerContext) { /// let _: String = ctx.env.API_KEY(); -/// let _: Option = ctx.env.LOG_LEVEL(); +/// let _: Option = ctx.env.LOG_LEVEL(); /// } /// ``` +/// +/// Existing `#[env(values("a", "b"))]` field constraints remain supported for +/// `String` and `Option`; enum constraints come from their variants. #[doc(inline)] pub use spacetimedb_bindings_macro::env; +/// Derive a typed environment value from an enum with unit variants. +/// +/// Each variant accepts its exact Rust name by default. Use +/// `#[env(value = "in progress")]` to map a variant to an arbitrary string, +/// including spaces, capitalization, Unicode or the empty string. Mappings must +/// be distinct, with 1 to 256 variants and at most 8192 UTF-8 bytes per string. +/// Generic enums and variants with payloads are not supported. +/// +/// The schema contains only allowed strings. A named environment accessor returns +/// this enum, or `Option` for an optional field; generic `env.get` still +/// returns `Option`. Missing required values and unmapped strings panic +/// with the key name only. Other derives and the enum's ordinary serialization +/// are unaffected. +#[doc(inline)] +pub use spacetimedb_bindings_macro::EnvironmentValue; + /// Read-only access to this database's environment store. /// /// Reads use the current transaction. In a procedure outside a transaction, diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 486ba90ba62..4ebb26e0b63 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,42 +917,82 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } -mod environment_value_sealed { - pub trait Sealed {} - - impl Sealed for String {} - impl Sealed for Option {} -} - -/// The compiler resolves declaration types, including aliases, before selecting -/// their metadata and accessor. Sealing keeps the accepted types identical to -/// the host's string and optional-string environment model. +/// Implementation support for `#[env]` and `#[derive(EnvironmentValue)]`. +/// +/// The compiler resolves aliases before selecting metadata and accessors. Custom +/// implementations must keep their declared constraint and decoding in agreement. +/// The host independently validates every published string against that constraint. #[doc(hidden)] -#[diagnostic::on_unimplemented(message = "environment fields must be `String` or `Option`")] -pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { +#[diagnostic::on_unimplemented( + message = "environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either" +)] +pub trait EnvironmentValue: Sized { const OPTIONAL: bool; - fn get(environment: &crate::Environment, key: &str) -> Self; + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint; + + /// Decode a checked host result. Errors must identify only the key, never its value. + fn from_environment(value: Option, key: &str) -> Self; + + fn get(environment: &crate::Environment, key: &str) -> Self { + Self::from_environment(environment.get(key), key) + } } +/// Required environment types supported by the optional-value implementation. +/// Derived enums and `String` implement this; `Option` deliberately does not. +#[doc(hidden)] +pub trait RequiredEnvironmentValue: EnvironmentValue {} + impl EnvironmentValue for String { const OPTIONAL: bool = false; - fn get(environment: &crate::Environment, key: &str) -> Self { - environment - .get(key) - .unwrap_or_else(|| panic!("required environment key is missing: {key}")) + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + spacetimedb_lib::environment::EnvironmentConstraint::AnyString + } + + fn from_environment(value: Option, key: &str) -> Self { + value.unwrap_or_else(|| panic!("required environment key is missing: {key}")) } } -impl EnvironmentValue for Option { +impl RequiredEnvironmentValue for String {} + +impl EnvironmentValue for Option { const OPTIONAL: bool = true; - fn get(environment: &crate::Environment, key: &str) -> Self { - environment.get(key) + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + T::constraint() + } + + fn from_environment(value: Option, key: &str) -> Self { + value.map(|value| T::from_environment(Some(value), key)) } } +mod string_environment_value_sealed { + pub trait Sealed {} + + impl Sealed for String {} + impl Sealed for Option {} +} + +/// Legacy field-level string constraints cannot override a typed enum's mapping. +#[doc(hidden)] +#[diagnostic::on_unimplemented( + message = "`#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = \"...\")]` instead" +)] +pub trait StringEnvironmentValue: EnvironmentValue + string_environment_value_sealed::Sealed { + fn with_constraint( + constraint: spacetimedb_lib::environment::EnvironmentConstraint, + ) -> spacetimedb_lib::environment::EnvironmentConstraint { + constraint + } +} + +impl StringEnvironmentValue for String {} +impl StringEnvironmentValue for Option {} + /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] pub fn register_environment(declarations: fn() -> Vec) { diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs index 6636b9160ef..b2b5181bc9d 100644 --- a/crates/bindings/tests/environment.rs +++ b/crates/bindings/tests/environment.rs @@ -3,4 +3,5 @@ fn environment_declaration_accessors_compile_with_exact_types() { let tests = trybuild::TestCases::new(); tests.pass("tests/pass/environment.rs"); tests.compile_fail("tests/ui/environment_types.rs"); + tests.compile_fail("tests/ui/environment_enum.rs"); } diff --git a/crates/bindings/tests/environment_enum_values.rs b/crates/bindings/tests/environment_enum_values.rs new file mode 100644 index 00000000000..ceecb647b5f --- /dev/null +++ b/crates/bindings/tests/environment_enum_values.rs @@ -0,0 +1,78 @@ +use spacetimedb::rt::EnvironmentValue as _; +use spacetimedb::spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use std::collections::BTreeMap; + +#[derive(Debug, PartialEq, Eq, spacetimedb::SpacetimeType, spacetimedb::EnvironmentValue)] +enum Mode { + #[env(value = "in progress")] + InProgress, + Ready, + #[env(value = "")] + Empty, + #[env(value = "héllo\0世界")] + Unicode, +} + +#[derive(Debug, PartialEq, Eq, spacetimedb::EnvironmentValue)] +enum Literal { + #[env(value = "only")] + Only, +} + +#[test] +fn typed_mappings_match_exact_schema_strings_and_optional_absence() { + let cases = [ + ("in progress", Mode::InProgress), + ("Ready", Mode::Ready), + ("", Mode::Empty), + ("héllo\0世界", Mode::Unicode), + ]; + assert_eq!( + Mode::constraint(), + EnvironmentConstraint::OneOf(cases.iter().map(|(s, _)| s.to_string()).collect()) + ); + assert_eq!(Option::::constraint(), Mode::constraint()); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "MODE".into(), + constraint: Mode::constraint(), + optional: false, + }]) + .unwrap(); + for (value, variant) in cases { + schema + .validate_values(&BTreeMap::from([("MODE".into(), value.into())])) + .unwrap(); + assert_eq!(Mode::from_environment(Some(value.into()), "MODE"), variant); + assert_eq!( + Option::::from_environment(Some(value.into()), "MODE"), + Some(variant) + ); + } + assert_eq!(Option::::from_environment(None, "MODE"), None); + assert_eq!(Literal::constraint(), EnvironmentConstraint::Literal("only".into())); + assert_eq!(Literal::from_environment(Some("only".into()), "VALUE"), Literal::Only); + for rejected in ["InProgress", "ready", "in progress ", "private-unmapped-value"] { + assert!(schema + .validate_values(&BTreeMap::from([("MODE".into(), rejected.into())])) + .is_err()); + } +} + +#[test] +fn decode_errors_report_the_key_without_the_supplied_value() { + for value in [None, Some("private-unmapped-value".into())] { + let error = std::panic::catch_unwind(|| Mode::from_environment(value, "MODE")).unwrap_err(); + let message = error.downcast_ref::().unwrap(); + assert!(message.contains("MODE")); + assert!(!message.contains("private-unmapped-value")); + assert!(!message.contains("in progress")); + } +} + +#[test] +fn environment_mapping_does_not_change_ordinary_enum_serialization() { + use spacetimedb::spacetimedb_lib::bsatn; + assert_eq!(bsatn::to_vec(&Mode::InProgress).unwrap(), vec![0]); + assert_eq!(bsatn::to_vec(&Mode::Ready).unwrap(), vec![1]); + assert_eq!(bsatn::from_slice::(&[2]).unwrap(), Mode::Empty); +} diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs index c0f80526ea4..136d631dc89 100644 --- a/crates/bindings/tests/pass/environment.rs +++ b/crates/bindings/tests/pass/environment.rs @@ -6,12 +6,27 @@ use std::string::String as RenamedString; type RequiredAlias = RenamedString; type OptionalAlias = Maybe; +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Ready, +} +type ModeAlias = Mode; +type MaybeMode = Maybe; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + const _: [(); 0] = [(); ::OPTIONAL as usize]; const _: [(); 1] = [(); ::OPTIONAL as usize]; #[spacetimedb::env] pub struct Env { pub REQUIRED: RequiredAlias, + pub MODE: ModeAlias, + pub OPTIONAL_MODE: MaybeMode, #[env(values("false", "true"))] pub FLAG: String, #[env(values(""))] @@ -22,6 +37,8 @@ pub struct Env { fn reads(env: spacetimedb::Environment) { let _: String = env.REQUIRED(); + let _: Mode = env.MODE(); + let _: Option = env.OPTIONAL_MODE(); let _: String = env.FLAG(); let _: Option = env.OPTIONAL(); let _: Option = env.get("get"); diff --git a/crates/bindings/tests/ui/environment_enum.rs b/crates/bindings/tests/ui/environment_enum.rs new file mode 100644 index 00000000000..95a6387f5c4 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.rs @@ -0,0 +1,54 @@ +#[derive(spacetimedb::EnvironmentValue)] +struct NotEnum; + +#[derive(spacetimedb::EnvironmentValue)] +enum Empty {} + +#[derive(spacetimedb::EnvironmentValue)] +enum Generic { + Value(T), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Payload { + Value(String), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Duplicate { + #[env(value = "Same")] + First, + Same, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum DuplicateAttribute { + #[env(value = "x", value = "y")] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongAttribute { + #[env(values("x"))] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongLiteral { + #[env(value = 1)] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + Ready, +} + +#[spacetimedb::env] +pub struct Env { + #[env(values("other"))] + pub MODE: Mode, + pub NESTED: Option>, +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_enum.stderr b/crates/bindings/tests/ui/environment_enum.stderr new file mode 100644 index 00000000000..f3f3d25ddc0 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.stderr @@ -0,0 +1,85 @@ +error: EnvironmentValue requires an enum with unit variants + --> tests/ui/environment_enum.rs:2:8 + | +2 | struct NotEnum; + | ^^^^^^^ + +error: environment value enums require 1 to 256 variants + --> tests/ui/environment_enum.rs:5:6 + | +5 | enum Empty {} + | ^^^^^ + +error: environment value enums cannot be generic + --> tests/ui/environment_enum.rs:8:13 + | +8 | enum Generic { + | ^^^ + +error: environment value variants cannot have payloads + --> tests/ui/environment_enum.rs:14:10 + | +14 | Value(String), + | ^^^^^^^^ + +error: environment variants must map to distinct strings + --> tests/ui/environment_enum.rs:21:5 + | +21 | Same, + | ^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:26:24 + | +26 | #[env(value = "x", value = "y")] + | ^^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:32:11 + | +32 | #[env(values("x"))] + | ^^^^^^ + +error: expected string literal + --> tests/ui/environment_enum.rs:38:19 + | +38 | #[env(value = 1)] + | ^ + +error[E0277]: the trait bound `Option: RequiredEnvironmentValue` is not satisfied + --> tests/ui/environment_enum.rs:51:17 + | +51 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `Option` + | +help: the following other types implement trait `RequiredEnvironmentValue` + --> tests/ui/environment_enum.rs:42:10 + | +42 | #[derive(spacetimedb::EnvironmentValue)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Mode` + | + ::: src/rt.rs + | + | impl RequiredEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + = note: required for `Option>` to implement `EnvironmentValue` + = note: this error originates in the derive macro `spacetimedb::EnvironmentValue` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: `#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = "...")]` instead + --> tests/ui/environment_enum.rs:50:15 + | +50 | pub MODE: Mode, + | ^^^^ unsatisfied trait bound + | +help: the trait `StringEnvironmentValue` is not implemented for `Mode` + --> tests/ui/environment_enum.rs:43:1 + | +43 | pub enum Mode { + | ^^^^^^^^^^^^^ +help: the following other types implement trait `StringEnvironmentValue` + --> src/rt.rs + | + | impl StringEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + | impl StringEnvironmentValue for Option {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Option` diff --git a/crates/bindings/tests/ui/environment_types.rs b/crates/bindings/tests/ui/environment_types.rs index 4fa3e3aae0b..27693688e4f 100644 --- a/crates/bindings/tests/ui/environment_types.rs +++ b/crates/bindings/tests/ui/environment_types.rs @@ -24,15 +24,4 @@ pub struct Unsupported { pub get: u32, } -struct Custom; - -// External code cannot extend the set of supported environment types. -impl spacetimedb::rt::EnvironmentValue for Custom { - const OPTIONAL: bool = false; - - fn get(_: &spacetimedb::Environment, _: &str) -> Self { - Self - } -} - fn main() {} diff --git a/crates/bindings/tests/ui/environment_types.stderr b/crates/bindings/tests/ui/environment_types.stderr index 3c7346dcb4e..f884a635897 100644 --- a/crates/bindings/tests/ui/environment_types.stderr +++ b/crates/bindings/tests/ui/environment_types.stderr @@ -1,57 +1,24 @@ -error[E0277]: the trait bound `Custom: spacetimedb::rt::environment_value_sealed::Sealed` is not satisfied - --> tests/ui/environment_types.rs:30:44 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^ unsatisfied trait bound - | -help: the trait `spacetimedb::rt::environment_value_sealed::Sealed` is not implemented for `Custom` - --> tests/ui/environment_types.rs:27:1 - | -27 | struct Custom; - | ^^^^^^^^^^^^^ -help: the following other types implement trait `spacetimedb::rt::environment_value_sealed::Sealed` - --> src/rt.rs - | - | impl Sealed for String {} - | ^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` - | impl Sealed for Option {} - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -note: required by a bound in `EnvironmentValue` - --> src/rt.rs - | - | pub trait EnvironmentValue: environment_value_sealed::Sealed + Sized { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `EnvironmentValue` - = note: `EnvironmentValue` is a "sealed trait", because to implement it you also need to implement `spacetimedb::rt::environment_value_sealed::Sealed`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it - = help: the following types implement the trait: - std::string::String - std::option::Option - -error[E0277]: environment fields must be `String` or `Option` - --> tests/ui/environment_types.rs:6:20 - | - 6 | pub VALUE: String, - | ^^^^^^ unsatisfied trait bound - | +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:6:20 + | +6 | pub VALUE: String, + | ^^^^^^ unsatisfied trait bound + | help: the trait `EnvironmentValue` is not implemented for `shadowed_string::String` - --> tests/ui/environment_types.rs:2:5 - | - 2 | pub struct String; - | ^^^^^^^^^^^^^^^^^ + --> tests/ui/environment_types.rs:2:5 + | +2 | pub struct String; + | ^^^^^^^^^^^^^^^^^ help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs - | - | impl EnvironmentValue for String { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:15:20 | 15 | pub VALUE: Option, @@ -63,67 +30,53 @@ help: the trait `EnvironmentValue` is not implemented for `shadowed_option::Opti 11 | pub struct Option(T); | ^^^^^^^^^^^^^^^^^^^^ help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:21:15 | 21 | pub BOOL: bool, | ^^^^ the trait `EnvironmentValue` is not implemented for `bool` | help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: the trait bound `std::option::Option: RequiredEnvironmentValue` is not satisfied --> tests/ui/environment_types.rs:22:17 | 22 | pub NESTED: Option>, - | ^^^^^^^^^^^^^^^^^^^^^^ the trait `EnvironmentValue` is not implemented for `std::option::Option>` + | ^^^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `std::option::Option` | -help: the trait `EnvironmentValue` is implemented for `std::option::Option` +help: the trait `RequiredEnvironmentValue` is implemented for `std::string::String` --> src/rt.rs | - | impl EnvironmentValue for Option { + | impl RequiredEnvironmentValue for String {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: required for `std::option::Option>` to implement `EnvironmentValue` -error[E0277]: environment fields must be `String` or `Option` +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either --> tests/ui/environment_types.rs:24:14 | 24 | pub get: u32, | ^^^ the trait `EnvironmentValue` is not implemented for `u32` | help: the following other types implement trait `EnvironmentValue` - --> tests/ui/environment_types.rs:30:1 - | -30 | impl spacetimedb::rt::EnvironmentValue for Custom { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Custom` - | - ::: src/rt.rs + --> src/rt.rs | | impl EnvironmentValue for String { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` ... - | impl EnvironmentValue for Option { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index 20bc11a5b9b..01a1f25afe9 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -41,7 +41,7 @@ pub fn cli() -> Command { ), )) .subcommand(target( - Command::new("list").about("List published environment keys (never values)"), + Command::new("list").about("List published environment keys and values"), )) } @@ -53,7 +53,7 @@ enum Query { impl Query { fn sql(&self) -> anyhow::Result { match self { - Self::List => Ok("SELECT key FROM st_env".into()), + Self::List => Ok("SELECT key, value FROM st_env".into()), Self::Get(key) => { // POSIX names cannot contain quotes or SQL syntax. validate_key(key).map_err(|_| anyhow::anyhow!("Invalid environment key name"))?; @@ -117,41 +117,40 @@ async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result } fn render(body: &[u8], query: &Query) -> anyhow::Result { - // Only project the requested single string column; do not dump an error or - // unexpected response which could contain unrequested secret values. + // Validate the requested projection before rendering any response values. let results: Vec>> = serde_json::from_slice(body).map_err(|_| anyhow::anyhow!("Invalid environment read response"))?; ensure!(results.len() == 1, "Invalid environment read result count"); let result = &results[0]; - let expected = match query { - Query::List => "key", - Query::Get(_) => "value", + let expected: &[&str] = match query { + Query::List => &["key", "value"], + Query::Get(_) => &["value"], }; ensure!( - result.schema.elements.len() == 1 - && result.schema.elements[0].name.as_deref() == Some(expected) - && result.schema.elements[0].algebraic_type == spacetimedb_lib::AlgebraicType::String, + result.schema.elements.len() == expected.len() + && result.schema.elements.iter().zip(expected).all(|(column, name)| { + column.name.as_deref() == Some(*name) && column.algebraic_type == spacetimedb_lib::AlgebraicType::String + }), "Invalid environment read projection" ); - let mut values = Vec::new(); + ensure!(result.rows.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); for row in &result.rows { - ensure!(row.len() == 1, "Invalid environment read row"); + ensure!(row.len() == expected.len(), "Invalid environment read row"); if matches!(query, Query::List) { validate_key(&row[0]).context("Invalid environment key in response")?; } ensure!( - row[0].len() <= MAX_ENV_VALUE_BYTES, + row.last().unwrap().len() <= MAX_ENV_VALUE_BYTES, "Environment read value exceeds limit" ); - values.push(row[0].as_str()); } match query { Query::List => { - ensure!(values.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); - values.sort_unstable(); - let rows = values - .into_iter() - .map(|key| Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![key])); + let mut rows: Vec<_> = result.rows.iter().collect(); + rows.sort_unstable_by(|a, b| a[0].cmp(&b[0])); + let rows = rows.into_iter().map(|row| { + Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![row[0].as_str(), row[1].as_str()]) + }); let table = sql::build_table( spacetimedb_lib::sats::satn::PsqlClient::SpacetimeDB, &result.schema, @@ -160,8 +159,8 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { Ok(format!("{table}\n")) } Query::Get(_) => { - ensure!(values.len() == 1, "Environment key is absent"); - Ok(format!("{}\n", values[0])) + ensure!(result.rows.len() == 1, "Environment key is absent"); + Ok(format!("{}\n", result.rows[0][0])) } } } @@ -169,9 +168,13 @@ fn render(body: &[u8], query: &Query) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - fn body(column: &'static str, rows: Vec>) -> Vec { + fn body(columns: &[&'static str], rows: Vec>) -> Vec { serde_json::to_vec(&[spacetimedb_client_api_messages::http::SqlStmtResult { - schema: spacetimedb_lib::sats::ProductType::from([(column, spacetimedb_lib::AlgebraicType::String)]), + schema: spacetimedb_lib::sats::ProductType::from_iter( + columns + .iter() + .map(|column| (*column, spacetimedb_lib::AlgebraicType::String)), + ), rows, total_duration_micros: 0, stats: Default::default(), @@ -197,7 +200,7 @@ mod tests { let get = matches.subcommand_matches("get").unwrap(); assert_eq!(get.get_one::("database").unwrap(), "db"); assert_eq!(get.get_one::("key").unwrap(), "KEY"); - assert_eq!(Query::List.sql().unwrap(), "SELECT key FROM st_env"); + assert_eq!(Query::List.sql().unwrap(), "SELECT key, value FROM st_env"); assert_eq!( Query::Get("KEY".into()).sql().unwrap(), "SELECT value FROM st_env WHERE key = 'KEY'" @@ -205,22 +208,27 @@ mod tests { assert!(Query::Get("x';DELETE FROM st_env;--".into()).sql().is_err()); } #[test] - fn list_projects_keys_and_rejects_unexpected_secret_columns() { + fn list_renders_sorted_keys_and_values_and_rejects_unexpected_columns() { assert_eq!( - render(&body("key", vec![vec!["Z"], vec!["A"]]), &Query::List) - .unwrap() - .lines() - .map(str::trim) - .collect::>(), - ["key", "-----", "\"A\"", "\"Z\""] + render( + &body(&["key", "value"], vec![vec!["Z", "last"], vec!["A", "first"]]), + &Query::List + ) + .unwrap() + .lines() + .enumerate() + .filter(|(index, _)| *index != 1) + .map(|(_, line)| line.split('|').map(str::trim).collect::>()) + .collect::>(), + [["key", "value"], ["\"A\"", "\"first\""], ["\"Z\"", "\"last\""]] ); - let err = render(&body("value", vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); + let err = render(&body(&["value"], vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); assert!(!format!("{err:#}").contains("generated-secret-sentinel")); assert_eq!( - render(&body("value", vec![vec![""]]), &Query::Get("A".into())).unwrap(), + render(&body(&["value"], vec![vec![""]]), &Query::Get("A".into())).unwrap(), "\n" ); - assert!(render(&body("value", vec![]), &Query::Get("A".into())).is_err()); + assert!(render(&body(&["value"], vec![]), &Query::Get("A".into())).is_err()); } #[tokio::test] async fn actual_loopback_queries_and_error_redaction() { @@ -229,13 +237,13 @@ mod tests { ( Query::List, "200 OK", - body("key", vec![vec!["KEY"]]), - Some(" key \n-------\n \"KEY\" \n"), + body(&["key", "value"], vec![vec!["KEY", "generated-list-sentinel"]]), + Some("generated-list-sentinel"), ), ( Query::Get("KEY".into()), "200 OK", - body("value", vec![vec!["generated-read-sentinel"]]), + body(&["value"], vec![vec!["generated-read-sentinel"]]), Some("generated-read-sentinel\n"), ), ( @@ -290,8 +298,17 @@ mod tests { .redirect(reqwest::redirect::Policy::none()) .build() .unwrap(); - let result = fetch(client.post(format!("http://{address}/v1/database/owned/sql")), query).await; + let result = fetch( + client.post(format!("http://{address}/v1/database/owned/sql")), + query.clone(), + ) + .await; match expected { + Some(expected) if matches!(query, Query::List) => { + let output = result.unwrap(); + assert!(output.contains("key") && output.contains("value")); + assert!(output.contains("KEY") && output.contains(expected)); + } Some(expected) => assert_eq!(result.unwrap(), expected), None => assert!(!format!("{:#}", result.unwrap_err()).contains("generated-error-secret")), } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 2ba96d8adb9..f044c7a0a11 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -186,10 +186,19 @@ impl Fixture { &[], ); let mut lines = output.lines().map(str::trim); - assert_eq!(lines.next(), Some("key")); + assert_eq!( + lines.next().unwrap().split('|').map(str::trim).collect::>(), + ["key", "value"] + ); lines - .filter(|line| !line.is_empty() && !line.chars().all(|c| c == '-')) - .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|line| !line.is_empty() && !line.chars().all(|c| matches!(c, '-' | '+'))) + .map(|line| { + let (key, value) = line.split_once('|').unwrap(); + let key: String = serde_json::from_str(key.trim()).unwrap(); + let value: String = serde_json::from_str(value.trim()).unwrap(); + assert_eq!(self.get(&key), format!("{value}\n")); + key + }) .collect() } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 96a29b29bf9..5d01ca55ab5 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -483,3 +483,79 @@ fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { }, ); } + +// The actual host keeps ENV as exact strings; generated Rust accessors decode +// the selected enum variant after initial publish and complete replacements. +#[test] +#[serial] +fn rust_environment_enums_preserve_exact_typed_mappings() { + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + let mut values = initial; + for (value, index) in [ + ("ready", 0u8), + ("other", 1), + ("in progress", 2), + ("Ready", 3), + ("", 4), + ("héllo\0世界", 5), + ] { + values.insert("MODE".into(), value.into()); + values.insert("TYPED".into(), value.into()); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![index, Some(index)]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + assert_eq!( + read(&module, "MODE").await, + AlgebraicValue::from(Some(value.to_string())) + ); + } + for rejected in ["InProgress", "READY", "in progress "] { + let mut invalid = values.clone(); + invalid.insert("TYPED".into(), rejected.into()); + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "TYPED").await, + AlgebraicValue::from(Some("héllo\0世界".to_string())) + ); + } + values.remove("TYPED"); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![5u8, None::]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 34161f9095a..dfcf6d245a1 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -56,16 +56,30 @@ Within an environment declaration, a simple enum specifies allowed strings. Enum -Add one environment declaration to the module: +Declare allowed strings with enums, then add one environment declaration to the module: ```rust +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "development")] + Development, + #[env(value = "production")] + Production, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum LogLevel { + #[env(value = "info")] + Info, + #[env(value = "debug")] + Debug, +} + #[spacetimedb::env] pub struct Env { pub API_KEY: String, - #[env(values("development", "production"))] - pub MODE: String, - #[env(values("info", "debug"))] - pub LOG_LEVEL: Option, + pub MODE: Mode, + pub LOG_LEVEL: Option, } ``` @@ -73,12 +87,14 @@ Inside a reducer, procedure, or view, read values from its context: ```rust let api_key: String = ctx.env.API_KEY(); -let mode: String = ctx.env.MODE(); -let log_level: Option = ctx.env.LOG_LEVEL(); +let mode: Mode = ctx.env.MODE(); +let log_level: Option = ctx.env.LOG_LEVEL(); let checked: Option = ctx.env.get("LOG_LEVEL"); ``` -`String` requires a value, and `Option` permits absence. `#[env(values(...))]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. +`String` accepts any string. An enum restricts values to its variants, and `Option` permits absence. Type aliases work for these types. Without an attribute, a variant accepts its exact Rust name. An explicit mapping such as `#[env(value = "in progress")] InProgress` supports spaces, capitalization, Unicode, or the empty string. Mappings must be unique; variants cannot have payloads. A one-variant enum declares a single allowed string. These mappings affect environment reads and declarations only, not the enum's ordinary serialization. + +Existing `#[env(values(...))]` constraints on `String` and `Option` fields remain supported. Use enum variants to constrain typed enum fields. @@ -198,6 +214,8 @@ Previously stored values are **not** defaults for the next publish. Every publis The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. +For publishing from an HTTP client or a module procedure, see the [HTTP publish format and example](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values). Supply the module and complete environment in the request body; project configuration and shell overrides are CLI conveniences. + ## Inspect published values The database owner and collaborators with private-table read access can inspect the environment. For the local database above: @@ -207,7 +225,7 @@ spacetime env list env-example --server http://127.0.0.1:3000 spacetime env get env-example MODE --server http://127.0.0.1:3000 ``` -`env list` prints keys only. `env get` prints the requested value and fails if it is absent. Reading a secret with `env get` therefore exposes that secret in the command's output. +`env list` prints a table of keys and values. `env get` prints the requested value and fails if it is absent. Both are explicit inspection commands and include secrets in their output. Automatic publish output still omits values. Values are stored in the private system table `st_env`. Authorized SQL reads are also supported: diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 7c9ee00f3f7..c820d565bb6 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -150,7 +150,7 @@ Inspect published database environment variables ###### **Subcommands:** * `get` — Read one published environment value -* `list` — List published environment keys (never values) +* `list` — List published environment keys and values @@ -180,7 +180,7 @@ Read one published environment value ## `spacetime env list` -List published environment keys (never values) +List published environment keys and values **Usage:** `spacetime env list [OPTIONS] ` diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 8d6efa37a11..99846b25301 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -6,11 +6,29 @@ static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); type RequiredString = String; type OptionalString = Option; +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "ready")] + Ready, + #[env(value = "other")] + Other, + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Capitalized, + #[env(value = "")] + Empty, + #[env(value = "héllo\0世界")] + Unicode, +} + +type OptionalMode = Option; + #[spacetimedb::env] pub struct Env { pub REQUIRED: RequiredString, - #[env(values("ready", "other"))] - pub MODE: String, + pub MODE: Mode, + pub TYPED: OptionalMode, pub MISSING: OptionalString, pub EMPTY: Option, pub UTF8: Option, @@ -24,7 +42,25 @@ pub struct Env { #[spacetimedb::reducer(init)] pub fn init(ctx: &ReducerContext) { assert_eq!(ctx.env.REQUIRED(), "initial-required"); - assert_eq!(ctx.env.MODE(), "ready"); + assert_eq!(ctx.env.MODE(), Mode::Ready); + assert_eq!(ctx.env.TYPED(), None); +} + +#[spacetimedb::reducer] +pub fn expect_typed_environment(ctx: &ReducerContext, required: u8, optional: Option) { + fn index(mode: Mode) -> u8 { + match mode { + Mode::Ready => 0, + Mode::Other => 1, + Mode::InProgress => 2, + Mode::Capitalized => 3, + Mode::Empty => 4, + Mode::Unicode => 5, + } + } + assert_eq!(index(ctx.env.MODE()), required); + assert_eq!(ctx.env.TYPED().map(index), optional); + assert_eq!(index(ctx.as_read_only().env.MODE()), required); } #[spacetimedb::reducer] From 2bd321b989c21d96a111cdbfd50ff73d6463be39 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 00:46:22 -0400 Subject: [PATCH 09/19] Fix ENV test CLI inspection and isolated root discovery --- .../tests/standalone/cli/environment.rs | 4 ++ crates/testing/src/lib.rs | 49 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index f044c7a0a11..1dde0899f72 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -88,6 +88,10 @@ impl Fixture { .env("NO_PROXY", "*") .env("no_proxy", "*") .envs(shell.iter().copied()) + // Avoid platform directory discovery after clearing the environment, + // including Windows' known-folder lookup for LocalAppData. + .arg("--root-dir") + .arg(self.test.project_dir.path()) .arg("--config-path") .arg(&self.test.config_path) .args(args) diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 13cb9872b96..63799d77e72 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -49,6 +49,55 @@ pub fn invoke_cli(paths: &SpacetimePaths, args: &[&str]) { } // If CUSTOM_SPACETIMEDB_PATH is missing, fall through to the default behavior. + if cmd == "publish" { + // Publishing inspects exact module bytes in a child process. This + // function is linked into a libtest under target/*/deps, so current_exe + // cannot locate the standalone companion of the actual runtime CLI. + // Use the same explicit build artifacts as the test server guard. + let cli = spacetimedb_guard::ensure_binaries_built(); + let inspector = cli + .with_file_name("spacetimedb-standalone") + .with_extension(std::env::consts::EXE_EXTENSION); + assert!( + inspector.is_file(), + "SDK tests require the standalone schema inspector beside the CLI: {}", + inspector.display() + ); + assert!( + sub_args.get_one::("server").is_some(), + "SDK publication must use its explicit test server" + ); + let root = paths + .to_root_dir() + .expect("SDK tests require an isolated root directory"); + let status = RUNTIME.block_on(async { + let mut child = tokio::process::Command::new(cli) + .arg("--root-dir") + .arg(root) + .arg("--config-path") + .arg(paths.cli_config_dir.cli_toml()) + .args(args) + .arg("--no-config") + .env("SPACETIMEDB_SCHEMA_EXTRACTOR", inspector) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("Failed to start the pre-built publish CLI"); + // Output streams directly to the test log, without accumulating a + // second buffer. The inspector retains its own tighter bounds. + match tokio::time::timeout(std::time::Duration::from_secs(120), child.wait()).await { + Ok(status) => status.expect("Failed to reap the pre-built publish CLI"), + Err(_) => { + let _ = child.start_kill(); + child.wait().await.expect("Failed to reap timed-out publish CLI"); + panic!("SDK module publication timed out"); + } + } + }); + assert!(status.success(), "SDK module publication failed"); + return; + } + // Default: run in-process CLI (fast/path-friendly for tests). let config = Config::new_with_localhost(paths.cli_config_dir.cli_toml()); RUNTIME From 93f89e9099258f65d8a165bf49f53c49aa75d8a8 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:00:37 -0400 Subject: [PATCH 10/19] Add Rust module-test environment example and runtime coverage --- .../snapshots/codegen__codegen_csharp.snap | 155 +++++++++++++++++ .../snapshots/codegen__codegen_rust.snap | 156 ++++++++++++++++++ .../codegen__codegen_typescript.snap | 44 +++++ crates/testing/tests/environment.rs | 6 + modules/module-test/src/environment.rs | 26 +++ modules/module-test/src/lib.rs | 2 + 6 files changed, 389 insertions(+) create mode 100644 modules/module-test/src/environment.rs diff --git a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap index ab899e7f663..21fe014ee8d 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap @@ -69,6 +69,84 @@ namespace SpacetimeDB } } ''' +"Procedures/ReadEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteProcedures : RemoteBase + { + public void ReadEnvironment(string key, ProcedureCallback callback) + { + // Convert the clean callback to the wrapper callback + InternalReadEnvironment(key, (ctx, result) => { + if (result.IsSuccess && result.Value != null) + { + callback(ctx, ProcedureCallbackResult.Success(result.Value.Value)); + } + else + { + callback(ctx, ProcedureCallbackResult.Failure(result.Error!)); + } + }); + } + + private void InternalReadEnvironment(string key, ProcedureCallback callback) + { + conn.InternalCallProcedure(new Procedure.ReadEnvironmentArgs(key), callback); + } + + } + + public abstract partial class Procedure + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironment + { + [DataMember(Name = "Value")] + public string? Value; + + public ReadEnvironment(string? Value) + { + this.Value = Value; + } + + public ReadEnvironment() + { + } + } + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironmentArgs : Procedure, IProcedureArgs + { + [DataMember(Name = "key")] + public string Key; + + public ReadEnvironmentArgs(string Key) + { + this.Key = Key; + } + + public ReadEnvironmentArgs() + { + this.Key = ""; + } + + string IProcedureArgs.ProcedureName => "read_environment"; + } + + } +} +''' "Procedures/ReturnValue.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -683,6 +761,82 @@ namespace SpacetimeDB } } ''' +"Reducers/ExpectEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void ExpectEnvironmentHandler(ReducerEventContext ctx, string key, string? expected); + public event ExpectEnvironmentHandler? OnExpectEnvironment; + + public void ExpectEnvironment(string key, string? expected) + { + conn.InternalCallReducer(new Reducer.ExpectEnvironment(key, expected)); + } + + public bool InvokeExpectEnvironment(ReducerEventContext ctx, Reducer.ExpectEnvironment args) + { + if (OnExpectEnvironment == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch(ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnExpectEnvironment( + ctx, + args.Key, + args.Expected + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ExpectEnvironment : Reducer, IReducerArgs + { + [DataMember(Name = "key")] + public string Key; + [DataMember(Name = "expected")] + public string? Expected; + + public ExpectEnvironment( + string Key, + string? Expected + ) + { + this.Key = Key; + this.Expected = Expected; + } + + public ExpectEnvironment() + { + this.Key = ""; + } + + string IReducerArgs.ReducerName => "expect_environment"; + } + } +} +''' "Reducers/ListOverAge.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -1681,6 +1835,7 @@ namespace SpacetimeDB Reducer.AssertCallerIdentityIsModuleIdentity args => Reducers.InvokeAssertCallerIdentityIsModuleIdentity(eventContext, args), Reducer.DeletePlayer args => Reducers.InvokeDeletePlayer(eventContext, args), Reducer.DeletePlayersByName args => Reducers.InvokeDeletePlayersByName(eventContext, args), + Reducer.ExpectEnvironment args => Reducers.InvokeExpectEnvironment(eventContext, args), Reducer.ListOverAge args => Reducers.InvokeListOverAge(eventContext, args), Reducer.LogModuleIdentity args => Reducers.InvokeLogModuleIdentity(eventContext, args), Reducer.QueryPrivate args => Reducers.InvokeQueryPrivate(eventContext, args), diff --git a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap index 1cda8c8d75b..af115ee46b5 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap @@ -488,6 +488,88 @@ impl delete_players_by_name for super::RemoteReducers { } } +''' +"expect_environment_reducer.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct ExpectEnvironmentArgs { + pub key: String, + pub expected: Option::, +} + +impl From for super::Reducer { + fn from(args: ExpectEnvironmentArgs) -> Self { + Self::ExpectEnvironment { + key: args.key, + expected: args.expected, +} +} +} + +impl __sdk::InModule for ExpectEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `expect_environment`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait expect_environment { + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`expect_environment:expect_environment_then`] to run a callback after the reducer completes. + fn expect_environment(&self, key: String, +expected: Option::, +) -> __sdk::Result<()> { + self.expect_environment_then(key, expected, |_, _| {}) + } + + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl expect_environment for super::RemoteReducers { + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp.invoke_reducer_with_callback(ExpectEnvironmentArgs { key, expected, }, callback) + } +} + ''' "foobar_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE @@ -1116,6 +1198,7 @@ pub mod add_private_reducer; pub mod assert_caller_identity_is_module_identity_reducer; pub mod delete_player_reducer; pub mod delete_players_by_name_reducer; +pub mod expect_environment_reducer; pub mod list_over_age_reducer; pub mod log_module_identity_reducer; pub mod query_private_reducer; @@ -1129,6 +1212,7 @@ pub mod test_d_table; pub mod test_f_table; pub mod my_player_table; pub mod get_my_schema_via_http_procedure; +pub mod read_environment_procedure; pub mod return_value_procedure; pub mod sleep_one_second_procedure; pub mod with_tx_procedure; @@ -1163,6 +1247,7 @@ pub use add_private_reducer::add_private; pub use assert_caller_identity_is_module_identity_reducer::assert_caller_identity_is_module_identity; pub use delete_player_reducer::delete_player; pub use delete_players_by_name_reducer::delete_players_by_name; +pub use expect_environment_reducer::expect_environment; pub use list_over_age_reducer::list_over_age; pub use log_module_identity_reducer::log_module_identity; pub use query_private_reducer::query_private; @@ -1170,6 +1255,7 @@ pub use say_hello_reducer::say_hello; pub use test_reducer::test; pub use test_btree_index_args_reducer::test_btree_index_args; pub use get_my_schema_via_http_procedure::get_my_schema_via_http; +pub use read_environment_procedure::read_environment; pub use return_value_procedure::return_value; pub use sleep_one_second_procedure::sleep_one_second; pub use with_tx_procedure::with_tx; @@ -1198,6 +1284,10 @@ pub enum Reducer { } , DeletePlayersByName { name: String, +} , + ExpectEnvironment { + key: String, + expected: Option::, } , ListOverAge { age: u8, @@ -1228,6 +1318,7 @@ impl __sdk::Reducer for Reducer { Reducer::AssertCallerIdentityIsModuleIdentity => "assert_caller_identity_is_module_identity", Reducer::DeletePlayer { .. } => "delete_player", Reducer::DeletePlayersByName { .. } => "delete_players_by_name", + Reducer::ExpectEnvironment { .. } => "expect_environment", Reducer::ListOverAge { .. } => "list_over_age", Reducer::LogModuleIdentity => "log_module_identity", Reducer::QueryPrivate => "query_private", @@ -1268,6 +1359,13 @@ Reducer::DeletePlayer{ name, } => __sats::bsatn::to_vec(&delete_players_by_name_reducer::DeletePlayersByNameArgs { name: name.clone(), +}), + Reducer::ExpectEnvironment{ + key, + expected, +} => __sats::bsatn::to_vec(&expect_environment_reducer::ExpectEnvironmentArgs { + key: key.clone(), + expected: expected.clone(), }), Reducer::ListOverAge{ age, @@ -3296,6 +3394,64 @@ impl query_private for super::RemoteReducers { } } +''' +"read_environment_procedure.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] + struct ReadEnvironmentArgs { + pub key: String, +} + + +impl __sdk::InModule for ReadEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `read_environment`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait read_environment { + fn read_environment(&self, key: String, +) { + self.read_environment_then(key, |_, _| {}); + } + + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ); +} + +impl read_environment for super::RemoteProcedures { + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ) { + self.imp.invoke_procedure_with_callback::<_, Option::>( + "read_environment", + ReadEnvironmentArgs { key, }, + __callback, + ); + } +} + ''' "remove_table_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE diff --git a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap index 9ca1e9926a5..addaeab2185 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap @@ -103,6 +103,24 @@ export default { name: __t.string(), }; ''' +"expect_environment_reducer.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + key: __t.string(), + expected: __t.option(__t.string()), +}; +''' "get_my_schema_via_http_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -162,6 +180,7 @@ import AddPrivateReducer from "./add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "./assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "./delete_player_reducer"; import DeletePlayersByNameReducer from "./delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "./expect_environment_reducer"; import ListOverAgeReducer from "./list_over_age_reducer"; import LogModuleIdentityReducer from "./log_module_identity_reducer"; import QueryPrivateReducer from "./query_private_reducer"; @@ -171,6 +190,7 @@ import TestBtreeIndexArgsReducer from "./test_btree_index_args_reducer"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "./get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "./read_environment_procedure"; import * as ReturnValueProcedure from "./return_value_procedure"; import * as SleepOneSecondProcedure from "./sleep_one_second_procedure"; import * as WithTxProcedure from "./with_tx_procedure"; @@ -270,6 +290,7 @@ const reducersSchema = __reducers( __reducerSchema("assert_caller_identity_is_module_identity", AssertCallerIdentityIsModuleIdentityReducer), __reducerSchema("delete_player", DeletePlayerReducer), __reducerSchema("delete_players_by_name", DeletePlayersByNameReducer), + __reducerSchema("expect_environment", ExpectEnvironmentReducer), __reducerSchema("list_over_age", ListOverAgeReducer), __reducerSchema("log_module_identity", LogModuleIdentityReducer), __reducerSchema("query_private", QueryPrivateReducer), @@ -281,6 +302,7 @@ const reducersSchema = __reducers( /** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ const proceduresSchema = __procedures( __procedureSchema("get_my_schema_via_http", GetMySchemaViaHttpProcedure.params, GetMySchemaViaHttpProcedure.returnType), + __procedureSchema("read_environment", ReadEnvironmentProcedure.params, ReadEnvironmentProcedure.returnType), __procedureSchema("return_value", ReturnValueProcedure.params, ReturnValueProcedure.returnType), __procedureSchema("sleep_one_second", SleepOneSecondProcedure.params, SleepOneSecondProcedure.returnType), __procedureSchema("with_tx", WithTxProcedure.params, WithTxProcedure.returnType), @@ -531,6 +553,23 @@ import { export default {}; ''' +"read_environment_procedure.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const params = { + key: __t.string(), +}; +export const returnType = __t.option(__t.string())''' "return_value_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -827,12 +866,15 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "../get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "../read_environment_procedure"; import * as ReturnValueProcedure from "../return_value_procedure"; import * as SleepOneSecondProcedure from "../sleep_one_second_procedure"; import * as WithTxProcedure from "../with_tx_procedure"; export type GetMySchemaViaHttpArgs = __Infer; export type GetMySchemaViaHttpResult = __Infer; +export type ReadEnvironmentArgs = __Infer; +export type ReadEnvironmentResult = __Infer; export type ReturnValueArgs = __Infer; export type ReturnValueResult = __Infer; export type SleepOneSecondArgs = __Infer; @@ -856,6 +898,7 @@ import AddPrivateReducer from "../add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "../assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "../delete_player_reducer"; import DeletePlayersByNameReducer from "../delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "../expect_environment_reducer"; import ListOverAgeReducer from "../list_over_age_reducer"; import LogModuleIdentityReducer from "../log_module_identity_reducer"; import QueryPrivateReducer from "../query_private_reducer"; @@ -869,6 +912,7 @@ export type AddPrivateParams = __Infer; export type AssertCallerIdentityIsModuleIdentityParams = __Infer; export type DeletePlayerParams = __Infer; export type DeletePlayersByNameParams = __Infer; +export type ExpectEnvironmentParams = __Infer; export type ListOverAgeParams = __Infer; export type LogModuleIdentityParams = __Infer; export type QueryPrivateParams = __Infer; diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 5d01ca55ab5..838b26e43b5 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -363,6 +363,12 @@ fn rust_environment_publish_is_atomic_and_reads_follow_declared_configuration() exercise_fixture("environment-test"); } +#[test] +#[serial] +fn rust_module_test_environment_publish_and_checked_reads() { + exercise_fixture("module-test"); +} + #[test] #[serial] fn typescript_environment_publish_and_checked_reads() { diff --git a/modules/module-test/src/environment.rs b/modules/module-test/src/environment.rs new file mode 100644 index 00000000000..6adce646498 --- /dev/null +++ b/modules/module-test/src/environment.rs @@ -0,0 +1,26 @@ +//! Optional declarations keep this general-purpose test module publishable +//! without configuration, just like the C#, C++, and TypeScript examples. + +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::env] +pub struct Env { + pub MISSING: Option, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, +} + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.EMPTY(), ctx.env.get("EMPTY")); + assert_eq!(ctx.env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index fc1851b21b0..9a2d6790d53 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -10,6 +10,8 @@ use spacetimedb::{ }; use spacetimedb::{log, ProcedureContext}; +pub mod environment; + pub type TestAlias = TestA; // ───────────────────────────────────────────────────────────────────────────── From 60992adb61f75daa5d62001f52f59dc3a280463b Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:00:38 -0400 Subject: [PATCH 11/19] Apply ENV system table and test naming review cleanup --- crates/cli/src/spacetime_config/environment.rs | 2 ++ .../src/locking_tx_datastore/committed_state.rs | 9 ++++----- crates/datastore/src/locking_tx_datastore/datastore.rs | 4 ++-- crates/datastore/src/system_tables.rs | 3 ++- crates/datastore/src/system_tables/environment.rs | 3 --- crates/smoketests/tests/standalone/cli/environment.rs | 10 +++++----- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs index c4493835376..4fea7a548c7 100644 --- a/crates/cli/src/spacetime_config/environment.rs +++ b/crates/cli/src/spacetime_config/environment.rs @@ -1,4 +1,6 @@ //! Preserve JSON numeric values before the JSON5 deserializer can round them. +//! For example, converting `9007199254740993` through `f64` yields +//! `9007199254740992`, changing an environment value before it reaches the module. //! This also permits existing comments, unquoted names and trailing commas. use serde_json::Value; diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 9169a96b45c..3f61903e645 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -29,9 +29,9 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, ST_TABLE_ACCESSOR_ID, - ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, - ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, + ST_ENV_ID, ST_ENV_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, + ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, }; use anyhow::anyhow; @@ -357,8 +357,7 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); - let env = crate::system_tables::st_env_schema(); - self.create_table(env.table_id, env.into()); + self.create_table(ST_ENV_ID, schemas[ST_ENV_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index cf5fd662c63..1082e0ae9c6 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1079,7 +1079,6 @@ pub(crate) mod tests { use super::*; use crate::error::IndexError; use crate::locking_tx_datastore::tx_state::PendingSchemaChange; - use crate::system_tables::ST_ENV_ID; use crate::system_tables::{ system_tables, StColumnRow, StConnectionCredentialsFields, StConstraintData, StConstraintFields, StConstraintRow, StEventTableFields, StIndexAlgorithm, StIndexFields, StIndexRow, StRowLevelSecurityFields, @@ -1093,6 +1092,7 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ST_ENV_ID, ST_ENV_NAME}; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1559,7 +1559,7 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, - TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_ENV_ID.into(), name: ST_ENV_NAME, ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index d39cd14923c..61dcd11be9b 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -279,6 +279,7 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; +pub(crate) const ST_ENV_IDX: usize = 20; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -700,8 +701,8 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_EVENT_TABLE_NAME); validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); - environment::validate_table(&result); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + validate_system_table::(&result, ST_ENV_NAME); result } diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs index 82395f4bd77..d79f676edb8 100644 --- a/crates/datastore/src/system_tables/environment.rs +++ b/crates/datastore/src/system_tables/environment.rs @@ -30,9 +30,6 @@ pub(super) fn register_table(builder: &mut RawModuleDefV9Builder) { .with_unique_constraint(ColId(0)) .with_index_no_accessor_name(btree(ColId(0))); } -pub(super) fn validate_table(def: &ModuleDef) { - validate_system_table::(def, ST_ENV_NAME); -} pub(crate) fn st_env_schema() -> TableSchema { st_schema(ST_ENV_NAME, ST_ENV_ID) } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 1dde0899f72..957ace39286 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -19,13 +19,13 @@ const KEYS: &[&str] = &[ "SMOKE_FLAG", ]; -struct Fixture { +struct EnvironmentFixture { test: Smoketest, database: String, wasm: PathBuf, } -impl Fixture { +impl EnvironmentFixture { fn new() -> Self { // Private CI supplies remote cluster settings to the same test binary. // This fixture must still create its own server and fresh credentials, @@ -296,7 +296,7 @@ fn bounded_output(mut command: Command) -> Output { #[test] fn cli_environment_layers_shell_and_exact_precompiled_declarations() { - let f = Fixture::new(); + let f = EnvironmentFixture::new(); f.write( "spacetime.json", json!({"database":"unused-parent", "env":{ @@ -362,7 +362,7 @@ fn cli_environment_layers_shell_and_exact_precompiled_declarations() { #[test] fn cli_environment_replacement_rejection_and_read_only_commands() { - let f = Fixture::new(); + let f = EnvironmentFixture::new(); f.config(Some( json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), )); @@ -436,7 +436,7 @@ fn cli_environment_replacement_rejection_and_read_only_commands() { #[test] fn cli_environment_initial_rejection_clear_and_omitted_payload() { - let mut f = Fixture::new(); + let mut f = EnvironmentFixture::new(); f.config(None); assert!(!f.publish(&[], &[]).status.success()); f.config(Some(json!({"SMOKE_REQUIRED":"clear-initial","SMOKE_MODE":"ready"}))); From b491ae43867d15ad4e854e5568eef756dc4d31d2 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:26:59 -0400 Subject: [PATCH 12/19] Simplify ENV schema declaration state and example naming --- crates/schema/src/def.rs | 20 ++++++++++++-------- crates/schema/src/def/validate/v10.rs | 24 +++++++++++------------- crates/schema/src/def/validate/v9.rs | 3 +-- crates/schema/src/error.rs | 4 ++-- modules/module-test-ts/src/index.ts | 6 ++++-- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 2516894ceea..e324d93b0ea 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -18,6 +18,7 @@ use std::collections::BTreeMap; use std::fmt::{self, Debug, Write}; use std::hash::Hash; +use std::sync::LazyLock; use crate::error::{IdentifierError, ValidationErrors}; use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; @@ -44,6 +45,7 @@ use spacetimedb_lib::db::raw_def::v9::{ RawUniqueConstraintDataV9, RawViewDefV9, TableAccess, TableType, }; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; +use spacetimedb_lib::environment::EnvironmentSchema; use spacetimedb_lib::{ProductType, RawModuleDef}; use spacetimedb_primitives::{ ColId, ColList, ColOrCols, ColSet, HttpHandlerId, ProcedureId, ReducerId, TableId, ViewFnPtr, @@ -180,8 +182,8 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, - environment: spacetimedb_lib::environment::EnvironmentSchema, - environment_declared: bool, + /// `None` means undeclared; an explicitly empty declaration is `Some(empty)`. + environment: Option, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -194,13 +196,17 @@ pub enum RawModuleDefVersion { impl ModuleDef { /// The validated root environment schema. Legacy modules have an empty schema. - pub fn environment(&self) -> &spacetimedb_lib::environment::EnvironmentSchema { - &self.environment + pub fn environment(&self) -> &EnvironmentSchema { + static EMPTY: LazyLock = LazyLock::new(EnvironmentSchema::default); + match &self.environment { + Some(schema) => schema, + None => &EMPTY, + } } /// Whether the raw module explicitly required environment support. pub fn environment_declared(&self) -> bool { - self.environment_declared + self.environment.is_some() } /// The raw module definition version this module was authored under. @@ -1012,7 +1018,6 @@ impl From for RawModuleDefV9 { raw_module_def_version: _, submodules: _, environment: _, - environment_declared: _, } = val; // Extract column defaults from tables before consuming tables @@ -1074,11 +1079,10 @@ impl From for RawModuleDefV10 { raw_module_def_version: _, submodules, environment, - environment_declared, } = val; let mut sections = Vec::new(); - if environment_declared { + if let Some(environment) = environment { sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); } let mut explicit_names = ExplicitNames::default(); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index f164642a615..084f6a1aac5 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -298,13 +298,10 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let ( - (tables, types, reducers, procedures, views, (http_handlers, http_routes)), - submodules, - (environment, environment_declared), - ) = (tables_types_reducers_procedures_views, submodules, environment) - .combine_errors() - .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; + let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules, environment) = + (tables_types_reducers_procedures_views, submodules, environment) + .combine_errors() + .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); @@ -327,7 +324,6 @@ pub fn validate(def: RawModuleDefV10) -> Result { raw_module_def_version: RawModuleDefVersion::V10, submodules, environment, - environment_declared, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -340,20 +336,20 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } -fn validate_environment(def: &RawModuleDefV10) -> Result<(spacetimedb_lib::environment::EnvironmentSchema, bool)> { +fn validate_environment(def: &RawModuleDefV10) -> Result> { let mut sections = def.sections.iter().filter_map(|section| match section { RawModuleDefV10Section::Environment(declarations) => Some(declarations), _ => None, }); let Some(declarations) = sections.next() else { - return Ok((Default::default(), false)); + return Ok(None); }; if sections.next().is_some() { - return Err(ValidationError::RepeatedEnvironmentSection.into()); + return Err(ValidationError::RepeatedEnvironmentDeclaration.into()); } let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) .map_err(|error| ValidationError::Environment { error })?; - Ok((schema, true)) + Ok(Some(schema)) } /// Validate that each submodule's namespace is a valid identifier of at most 63 characters, @@ -2844,6 +2840,8 @@ mod environment_tests { let legacy = validate(RawModuleDefV10::default()).unwrap(); assert!(legacy.environment().is_empty()); assert!(!legacy.environment_declared()); + let raw: RawModuleDefV10 = legacy.into(); + assert!(!validate(raw).unwrap().environment_declared()); let explicit = validate(RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Environment(vec![])], }) @@ -2870,7 +2868,7 @@ mod environment_tests { assert!(validate(duplicate) .unwrap_err() .into_iter() - .any(|error| matches!(error, ValidationError::RepeatedEnvironmentSection))); + .any(|error| matches!(error, ValidationError::RepeatedEnvironmentDeclaration))); let nested = RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { namespace: "outer".into(), diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 1ba7c03c7b9..c6eef4f74f9 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -171,8 +171,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), - environment: Default::default(), - environment_declared: false, + environment: None, }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 770c44c7a3e..691fef4a3f5 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,8 +22,8 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { - #[error("module has repeated environment sections")] - RepeatedEnvironmentSection, + #[error("module has repeated environment declarations")] + RepeatedEnvironmentDeclaration, #[error("invalid environment declaration: {error}")] Environment { error: spacetimedb_lib::environment::EnvironmentSchemaError, diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 2d903356b00..000e25be0e4 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -574,7 +574,8 @@ export const router = spacetimedb.httpRouter( ); // Dedicated environment ABI integration exercised by crates/testing. -export const expect_environment = spacetimedb.reducer( +export const expectEnvironment = spacetimedb.reducer( + { name: 'expect_environment' }, { key: t.string(), expected: t.option(t.string()) }, (ctx, { key, expected }) => { if (libSubmodule.readRootEnvironmentHelper() !== ctx.env.get('EMPTY')) throw new Error('helper environment scope mismatch'); @@ -584,7 +585,8 @@ export const expect_environment = spacetimedb.reducer( } } ); -export const read_environment = spacetimedb.procedure( +export const readEnvironment = spacetimedb.procedure( + { name: 'read_environment' }, { key: t.string() }, t.option(t.string()), (ctx, { key }) => { From b0c498b1346b6c5864b3ff16c4a8f63a767e0aad Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 01:41:23 -0400 Subject: [PATCH 13/19] Clarify standalone publication locking and ENV recovery invariants --- .../standalone/src/control_db/environment.rs | 14 +++++++++++ crates/standalone/src/lib.rs | 24 ++++++++++++------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs index ede423c3935..454dd0e1d70 100644 --- a/crates/standalone/src/control_db/environment.rs +++ b/crates/standalone/src/control_db/environment.rs @@ -1,4 +1,18 @@ //! Private bootstrap inputs, separate from the historical public Database encoding. +//! +//! Creation and reset atomically persist both database indexes, the generation +//! and initial-program binding, the complete ENV input, and the nominated leader. +//! The transaction is flushed before host launch, so a lost request response can +//! recover the same generation through ordinary leader lookup. +//! +//! Reading the input rechecks the persisted owner, program, and generation in +//! the same transaction. A legacy generation with neither metadata nor input has +//! an empty environment; missing input for a recorded generation is an error. +//! +//! The host reads this input only before the database's first initialization. +//! Reopening an initialized database uses its committed program and `st_env`, +//! including later module updates. Reset replaces the bootstrap input, and +//! database deletion removes it atomically with both indexes and the binding. use super::*; use spacetimedb_client_api_messages::publish::PublishRequest; use spacetimedb_lib::Hash; diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index efe11d860fa..c0ad9638d92 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -188,7 +188,7 @@ impl NodeDelegate for StandaloneEnv { let owner = self.weak_self.upgrade().expect("standalone owner exists during lookup"); tokio::spawn(async move { let _guard = guard; - owner.leader_under_publication(database_id).await + owner.leader_with_publication_lock_held(database_id).await }) .await .map_err(|error| GetLeaderHostError::LaunchError { source: error.into() })? @@ -403,7 +403,11 @@ impl StandaloneEnv { .await? } - async fn leader_under_publication(&self, database_id: u64) -> Result { + /// Look up or start the current leader while the caller holds the publication + /// lock. Retain the read or write guard through this future's completion. + /// Ordinary lookup holds a read guard; publication and reset hold a write + /// guard, so calling `leader()` here would acquire the lock again and deadlock. + async fn leader_with_publication_lock_held(&self, database_id: u64) -> Result { let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { return Err(GetLeaderHostError::NoSuchReplica); }; @@ -477,7 +481,7 @@ impl StandaloneEnv { let database_id = database.id; let database_identity = database.database_identity; - let leader = self.leader_under_publication(database_id).await?; + let leader = self.leader_with_publication_lock_held(database_id).await?; let update_result = leader .update_with_environment( database, @@ -574,7 +578,11 @@ impl StandaloneEnv { None => { // A reset without an artifact retains the currently committed // module, not the original bootstrap program or its old values. - let module = self.leader_under_publication(database.id).await?.module().await?; + let module = self + .leader_with_publication_lock_held(database.id) + .await? + .module() + .await?; module .relational_db() .program()? @@ -671,16 +679,14 @@ impl StandaloneEnv { async fn on_insert_replica(&self, instance: &Replica) -> Result<(), anyhow::Error> { if instance.leader { - let database = self - .control_db - .get_database_by_id(instance.database_id)? + self.leader_with_publication_lock_held(instance.database_id) + .await .with_context(|| { format!( - "unknown database: id: {}, instance: {}", + "failed to start leader for database {}, replica {}", instance.database_id, instance.id ) })?; - self.leader_under_publication(database.id).await?; } Ok(()) From 0b1abdb24f5862041488af5a91a3a7068263fa9a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 08:56:26 -0400 Subject: [PATCH 14/19] Regenerate canonical C# environment metadata bindings --- .../Autogen/EnvironmentConstraint.g.cs | 15 +++-- .../Autogen/EnvironmentDeclaration.g.cs | 55 +++++++++++-------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs index f4226020799..87031068dec 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs @@ -1,11 +1,16 @@ -// Canonical module-definition metadata; declaration constraints contain no runtime values. +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + #nullable enable -namespace SpacetimeDB.Internal; -[SpacetimeDB.Type] -public partial record EnvironmentConstraint - : SpacetimeDB.TaggedEnum<( +using System; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + public partial record EnvironmentConstraint : SpacetimeDB.TaggedEnum<( SpacetimeDB.Unit AnyString, string Literal, System.Collections.Generic.List OneOf )>; +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs index 554d77ba55a..504d1a846a9 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs @@ -1,29 +1,40 @@ -#nullable enable -namespace SpacetimeDB.Internal; - -[SpacetimeDB.Type] -[System.Runtime.Serialization.DataContract] -public sealed partial class EnvironmentDeclaration -{ - [System.Runtime.Serialization.DataMember(Name = "name")] - public string Name; +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - [System.Runtime.Serialization.DataMember(Name = "constraint")] - public EnvironmentConstraint Constraint; +#nullable enable - [System.Runtime.Serialization.DataMember(Name = "optional")] - public bool Optional; +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; - public EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional) +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class EnvironmentDeclaration { - this.Name = Name; - this.Constraint = Constraint; - this.Optional = Optional; - } + [DataMember(Name = "name")] + public string Name; + [DataMember(Name = "constraint")] + public EnvironmentConstraint Constraint; + [DataMember(Name = "optional")] + public bool Optional; - public EnvironmentDeclaration() - { - Name = ""; - Constraint = new EnvironmentConstraint.AnyString(default); + public EnvironmentDeclaration( + string Name, + EnvironmentConstraint Constraint, + bool Optional + ) + { + this.Name = Name; + this.Constraint = Constraint; + this.Optional = Optional; + } + + public EnvironmentDeclaration() + { + this.Name = ""; + this.Constraint = null!; + } } } From 87169bd3fd331ec136228f7682682c75eba3f582 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 20:41:46 -0400 Subject: [PATCH 15/19] Clarify environment usage and lower rejected SQL logging to debug --- crates/client-api/src/lib.rs | 2 +- .../00100-databases/00600-submodules.md | 6 +++ .../00700-environment-variables.md | 43 +++++++++++++++---- .../00200-http-api/00300-database.md | 43 +++++++------------ 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 9423849ddf8..7d3175e751b 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -166,7 +166,7 @@ impl Host { .await .map_err(|e| { // Parser diagnostics can quote values. Return them only to the caller. - log::warn!("SQL request rejected"); + log::debug!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; diff --git a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md index 1bfb9a2a3a4..78b1db00454 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md +++ b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md @@ -251,6 +251,12 @@ export default authSchema; +## Environment variables + +Only the root module can declare a nonempty [environment](./00700-environment-variables.md). Including a submodule with environment declarations causes publication to fail. A module with such declarations can still be published independently as a root module. + +Submodules have no separate environment-variable namespace, and their host-dispatched entry points cannot read the root module's environment. Root module code can pass configuration values to helpers explicitly. Ordinary helper calls retain the calling entry point's access, including calls to helpers defined in submodules. + ## Client Subscriptions Client subscriptions use the same namespace structure as server-side access. Submodule tables and views are queried as `.`. diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index dfcf6d245a1..b1fc7fe9d9f 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -53,6 +53,12 @@ const checked: string | null = ctx.env.get('LOG_LEVEL'); Within an environment declaration, a simple enum specifies allowed strings. Enums used elsewhere in the module retain their usual tagged representation. An enum with one case restricts the value to that string. Enums with payloads cannot be used as environment constraints. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors return `undefined`; `ctx.env.get` returns `null` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get('get')`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `withTx`. + @@ -96,6 +102,12 @@ let checked: Option = ctx.env.get("LOG_LEVEL"); Existing `#[env(values(...))]` constraints on `String` and `Option` fields remain supported. Use enum variants to constrain typed enum fields. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `None` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get("get")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. + @@ -126,6 +138,12 @@ string? checkedValue = ctx.Env.Get("LOG_LEVEL"); `string` requires a value, and `string?` permits absence. `[SpacetimeDB.EnvValues(...)]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.Env.Get` return `null` for an absent optional value. + +Key names are exact and case-sensitive. Names that collide with `Get`, `ModuleEnvironment`, or inherited `Object` methods are available through the string-key getter, for example `ctx.Env.Get("GetType")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `WithTx`. + @@ -161,14 +179,14 @@ std::optional checked = ctx.env.get("LOG_LEVEL"); `std::string` requires a value, and `std::optional` permits absence. The optional third element restricts the allowed strings. Supplying one string makes it an exact-value constraint. - - +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `std::nullopt` for an absent optional value. -The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails; it does not return an absent value. Named optional accessors return `None`, `undefined`, `null`, or `std::nullopt`, depending on the language. The TypeScript string-key getter uses `null` for absence. +Key names are exact and case-sensitive. Names reserved by the generated accessor type, including `get`, remain available through the string-key getter, for example `ctx.env.get("get")`. A module with no environment declarations accepts no keys. -Key names are exact and case-sensitive. The getter name `get`, or `Get` in C#, is reserved: a key with that name remains accessible through the string-key getter. A module with no environment declarations accepts no keys. +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. -Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx` in Rust or C++, `withTx` in TypeScript, or `WithTx` in C#. + + ## Supply values when publishing @@ -192,7 +210,14 @@ With a local server running on port 3000, publish from the directory containing API_KEY='development-only-key' spacetime publish ``` -For real credentials, supply the value through the publishing process's environment or an appropriately ignored local configuration file. Keep secrets out of checked-in configuration and module source. +For real credentials, supply the value through the publishing process's environment, `spacetime.local.json`, or `spacetime.{environment}.local.json`. Keep checked-in `spacetime.json` and `spacetime.{environment}.json` limited to non-secret defaults. Ensure the local files are ignored by Git: + +```gitignore +spacetime.local.json +spacetime.*.local.json +``` + +The `.local` naming convention does not itself prevent a file from being committed. The CLI resolves each declared key in this order: @@ -204,7 +229,7 @@ A shell value overrides JSON even when it is an empty string or the key is absen The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. -JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. For an absent optional value, omit its property from the JSON object instead of setting it to `null`. Also remove any inherited or shell value for that key, as described below. ### Every publish replaces the complete environment @@ -238,7 +263,7 @@ SQL writes to `st_env`, module-side writes, and separate CLI setters are not sup ## Access and limits -Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. +Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. There is no separate environment-variable namespace to configure for a submodule. A module with environment declarations can be published independently as a root module, but cannot be included as a submodule with those declarations. See [Submodules](./00600-submodules.md) for this restriction. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. @@ -250,4 +275,4 @@ Use an ordinary [private table](../00300-tables/00400-access-permissions.md) whe Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. -Unlike environment variables, these values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. +Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index 2066a7682c2..52faf7df576 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -114,36 +114,23 @@ Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish `module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. -For example, this Python script publishes a Wasm module to a local server. Set `SPACETIME_TOKEN` to a token authorized to publish and `API_KEY` to the complete configuration's required value. Change the module path to the artifact you built. - -```python -import base64 -import json -import os -from pathlib import Path -from urllib.request import Request, urlopen - -body = json.dumps({ - "module": base64.b64encode(Path("module.wasm").read_bytes()).decode("ascii"), - "environment": { - "API_KEY": os.environ["API_KEY"], - "MODE": "development", - }, -}).encode("utf-8") - -request = Request( - "http://127.0.0.1:3000/v1/database/env-example?host_type=wasm", - data=body, - method="PUT", - headers={ - "Authorization": "Bearer " + os.environ["SPACETIME_TOKEN"], - "Content-Type": "application/vnd.spacetimedb.publish+json", - }, -) -with urlopen(request, timeout=60) as response: - print(response.read().decode("utf-8")) +For example, use `curl`, `jq`, and `base64` to publish a Wasm module to a local server. Export `SPACETIME_TOKEN` with a token authorized to publish and `API_KEY` with the required value. Change `module.wasm` to the artifact you built. + +```bash +base64 < module.wasm | + jq --raw-input --slurp '{ + module: gsub("[\\r\\n]"; ""), + environment: { API_KEY: env.API_KEY, MODE: "development" } + }' | + curl --fail-with-body --request PUT \ + 'http://127.0.0.1:3000/v1/database/env-example?host_type=wasm' \ + --header "Authorization: Bearer $SPACETIME_TOKEN" \ + --header 'Content-Type: application/vnd.spacetimedb.publish+json' \ + --data-binary @- ``` +`jq` handles JSON escaping for the supplied value, including quotes and newlines. For a procedure or another HTTP client, construct the same JSON object with that language's JSON serializer and Base64 encoder. + Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. From 291e02d6ce84f929de54d9eee8e915439c2267e7 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 20:42:31 -0400 Subject: [PATCH 16/19] Avoid inherited member collisions in C# environment accessors --- .../Codegen.Tests/EnvironmentTests.cs | 46 +++++++++++++++++++ crates/bindings-csharp/Codegen/Environment.cs | 15 +++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs index d7dd4d4382e..efb8997828a 100644 --- a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -25,6 +25,8 @@ public static class Host { return key switch { "REQUIRED" => Reads.ToString(), "OPTIONAL" => null, "MODE" => "prod", "Get" => "reserved", "class" => "keyword", + "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString" or + "Finalize" or "GetType" or "MemberwiseClone" => key, _ => throw new System.InvalidOperationException("undeclared environment key") }; } @@ -104,6 +106,50 @@ declarations[0].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.Any Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty("Get")); } + [Theory] + [InlineData("Get")] + [InlineData("ModuleEnvironment")] + [InlineData("Equals")] + [InlineData("GetHashCode")] + [InlineData("ToString")] + [InlineData("Finalize")] + [InlineData("GetType")] + [InlineData("MemberwiseClone")] + public static void ReservedAccessorNamesRemainAvailableThroughCheckedGet(string name) + { + var (compilation, result) = Generate( + $$""" + [SpacetimeDB.Env] public struct Declarations { + public string {{name}}; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.Get("{{name}}") != "{{(name == "Get" ? "reserved" : name)}}") + throw new System.Exception("bad reserved accessor"); + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 1 || declarations[0].Name != "{{name}}") + throw new System.Exception("missing reserved declaration"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => + diagnostic.Severity is DiagnosticSeverity.Warning or DiagnosticSeverity.Error + && diagnostic.Location.SourceTree is { } tree + && result.GeneratedTrees.Contains(tree) + ); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty(name)); + } + [Theory] [InlineData("public int BAD;")] [InlineData("public static string BAD;")] diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs index e049abb5fda..a9f0992da9c 100644 --- a/crates/bindings-csharp/Codegen/Environment.cs +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -123,8 +123,19 @@ void Report(ISymbol symbol, string message) => $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" ); // Preserve the checked generic method, including a key literally - // named Get. Keywords are escaped without renaming stored keys. - if (name is "Get" or "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString") + // named Get, and inherited object members. Keywords are escaped + // without renaming stored keys. + if ( + name + is "Get" + or "ModuleEnvironment" + or "Equals" + or "GetHashCode" + or "ToString" + or "Finalize" + or "GetType" + or "MemberwiseClone" + ) continue; var read = $"Get({Literal(name)})"; if (!optional) From 81a4190ab36c43717f93fa41b80fdb2bf25a26b8 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 21:29:45 -0400 Subject: [PATCH 17/19] Separate generic view handling from ENV changes --- crates/core/src/host/module_host.rs | 127 ++---------------- .../src/host/wasm_common/module_host_actor.rs | 9 +- crates/testing/tests/environment.rs | 126 +++++------------ modules/environment-test/src/lib.rs | 15 --- 4 files changed, 49 insertions(+), 228 deletions(-) diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 685a6046e13..495f6a61fa9 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1630,18 +1630,6 @@ impl fmt::Debug for ViewCallResult { } impl ViewCallResult { - fn into_materialized_tx(self, db: &RelationalDB, trapped: bool) -> Result { - let error = match self.outcome { - ViewOutcome::Success if !trapped => return Ok(self.tx), - ViewOutcome::Success => "View instance trapped during materialization".to_owned(), - ViewOutcome::Failed(error) => error, - ViewOutcome::BudgetExceeded => "View terminated due to insufficient budget".to_owned(), - }; - let (_, metrics, reducer) = db.rollback_mut_tx(self.tx); - db.report_mut_tx_metrics(reducer, metrics, None); - Err(ViewCallError::InternalError(error)) - } - pub fn default(tx: MutTxId) -> Self { Self { outcome: ViewOutcome::Success, @@ -1721,9 +1709,6 @@ pub enum ClientConnectedError { pub struct RefInstance<'a, I: WasmInstance> { pub common: &'a mut InstanceCommon, pub instance: &'a mut I, - // Invocation-local disposal state survives errors propagated through SQL or - // subscription helpers, whose Result error does not carry a success tuple. - pub(crate) trapped: bool, } macro_rules! call_view_command_method { @@ -2961,26 +2946,17 @@ impl ModuleHost { /// Passing [`Workload::Sql`] will update the instance's last-used timestamp. /// Passing [`Workload::Subscribe`] will also increment the subscriber's refcount. pub fn materialize_views( - tx: MutTxId, + mut tx: MutTxId, instance: &mut RefInstance<'_, I>, view_collector: &impl CollectViews, caller: Identity, workload: Workload, ) -> Result<(MutTxId, bool), ViewCallError> { use FunctionArgs::*; - let db = instance.instance.replica_ctx().relational_db().clone(); - // Keep all earlier view materializations and subscription refcounts in - // the same rollback boundary if any later view fails. - let mut tx = scopeguard::guard(Some(tx), |tx| { - if let Some(tx) = tx { - let (_, metrics, reducer) = db.rollback_mut_tx(tx); - db.report_mut_tx_metrics(reducer, metrics, None); - } - }); let mut view_ids = HashSet::new(); view_collector.collect_views(&mut view_ids); for view_id in view_ids { - let st_view_row = tx.as_ref().unwrap().lookup_st_view(view_id)?; + let st_view_row = tx.lookup_st_view(view_id)?; let view_name: NamespacedIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; @@ -2992,30 +2968,25 @@ impl ModuleHost { }; let view_call = ViewCallInfo::from_args(view_id, args); let sender = args.sender(); - let is_materialized = tx.as_ref().unwrap().is_view_materialized(&view_call)?; + let is_materialized = tx.is_view_materialized(&view_call)?; if !is_materialized { - let (res, trapped) = Self::call_view( - instance, - tx.take().unwrap(), - &view_name, - view_id, - table_id, - Nullary, - caller, - sender, - )?; - *tx = Some(res.into_materialized_tx(&db, trapped)?); + let (res, trapped) = + Self::call_view(instance, tx, &view_name, view_id, table_id, Nullary, caller, sender)?; + tx = res.tx; + if trapped { + return Ok((tx, true)); + } } - let tx = tx.as_mut().unwrap(); - // These changes commit only after every requested view succeeds. + // If this is a sql call, we only update this view's "last called" timestamp if let Workload::Sql = workload { tx.update_view_timestamp(view_call.clone(), args)?; } + // If this is a subscribe call, we also increment this view's subscriber count if let Workload::Subscribe = workload { tx.subscribe_view(view_call, args, caller)?; } } - Ok((ScopeGuard::into_inner(tx).unwrap(), false)) + Ok((tx, false)) } /// Refreshes every view made stale by `tx`. @@ -3162,11 +3133,6 @@ impl ModuleHost { sender: Option, timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { - let db = instance.instance.replica_ctx().relational_db().clone(); - let tx = scopeguard::guard(tx, |tx| { - let (_, metrics, reducer) = db.rollback_mut_tx(tx); - db.report_mut_tx_metrics(reducer, metrics, None); - }); let module_def = &instance.common.info().module_def; let (global_fn_ptr, view_def, owning_def) = module_def .view_by_name_with_global_fn_ptr(view_name) @@ -3178,7 +3144,7 @@ impl ModuleHost { Ok(Self::call_view_inner( instance, - ScopeGuard::into_inner(tx), + tx, view_name, view_id, table_id, @@ -3220,9 +3186,7 @@ impl ModuleHost { view_typespace, }; - let (result, trapped) = instance.common.call_view_with_tx(tx, params, instance.instance); - instance.trapped |= trapped; - (result, trapped) + instance.common.call_view_with_tx(tx, params, instance.instance) } pub async fn init_database(&self, program: Program) -> Result { @@ -3724,69 +3688,6 @@ mod tests { use spacetimedb_sats::product; use std::sync::Arc; - #[test] - fn failed_view_materialization_rolls_back_prior_rows_and_subscriber_counts() -> anyhow::Result<()> { - use super::{ViewCallResult, ViewOutcome}; - use crate::db::relational_db::tests_utils::begin_mut_tx; - use spacetimedb_datastore::locking_tx_datastore::{ViewCallInfo, ViewInstanceArgs}; - use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; - use spacetimedb_lib::ProductType; - use spacetimedb_schema::def::ModuleDef; - - let db = TestDB::in_memory()?; - let mut builder = RawModuleDefV10Builder::new(); - let row = builder.add_algebraic_type( - [], - "Row", - AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::U8)])), - true, - ); - builder.add_view( - "earlier", - 0, - true, - true, - ProductType::unit(), - AlgebraicType::array(row.into()), - ); - let module: ModuleDef = builder.finish().try_into()?; - let mut tx = begin_mut_tx(&db); - let (view_id, table_id) = db.create_view(&mut tx, &module, module.view("earlier").unwrap())?; - db.commit_tx(tx)?; - let call = ViewCallInfo::anonymous(view_id); - - for (outcome, trapped) in [ - (ViewOutcome::Failed("denied".into()), false), - (ViewOutcome::Failed("trap".into()), true), - (ViewOutcome::BudgetExceeded, true), - (ViewOutcome::Success, true), - ] { - let mut tx = begin_mut_tx(&db); - // A preceding view succeeded in this same multi-view request. - db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; - tx.subscribe_view(call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; - assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); - let mut result = ViewCallResult::default(tx); - result.outcome = outcome; - assert!(result.into_materialized_tx(&db, trapped).is_err()); - - let tx = begin_mut_tx(&db); - assert!(tx.active_subscribers_for_view(view_id).is_empty()); - assert!(!tx.is_view_materialized(&call)?); - assert_eq!(db.iter_mut(&tx, table_id)?.count(), 0); - let _ = db.rollback_mut_tx(tx); - } - // Success retains the owned transaction for the normal commit path. - let mut tx = begin_mut_tx(&db); - db.materialize_view_call(&mut tx, table_id, call.clone(), vec![product![9_u8]])?; - tx.subscribe_view(call, ViewInstanceArgs::Anonymous, Identity::ZERO)?; - db.commit_tx(ViewCallResult::default(tx).into_materialized_tx(&db, false)?)?; - let tx = begin_mut_tx(&db); - assert_eq!(tx.active_subscribers_for_view(view_id), vec![(Identity::ZERO, 1)]); - let _ = db.rollback_mut_tx(tx); - Ok(()) - } - fn v2_client_config() -> ClientConfig { ClientConfig { protocol: Protocol::Binary, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 2733d4cefe3..7abc0aaf186 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1210,7 +1210,6 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, - trapped: false, }; let (res, trapped) = match cmds { ViewCommand::AddSingleSubscription { @@ -1297,7 +1296,7 @@ impl InstanceCommon { if let Err(err) = &res { error_target.send(&info.subscriptions, err); } - (res, trapped || inst.trapped) + (res, trapped) } pub(in crate::host) fn handle_sql_cmd( @@ -1308,7 +1307,6 @@ impl InstanceCommon { let mut inst = RefInstance { instance: inst, common: self, - trapped: false, }; let SqlCommand { db, @@ -1325,9 +1323,9 @@ impl InstanceCommon { result: Ok(result), head, }, - trapped || inst.trapped, + trapped, ), - Err(err) => (SqlCommandResult { result: Err(err), head }, inst.trapped), + Err(err) => (SqlCommandResult { result: Err(err), head }, false), } } @@ -1494,7 +1492,6 @@ impl InstanceCommon { let mut instance = RefInstance { common: self, instance: inst, - trapped: false, }; ModuleHost::call_views_with_tx_at(tx, &mut instance, caller, timestamp) } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index 838b26e43b5..a8187619fca 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -95,30 +95,18 @@ async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { &mut vec![], ) .await; - let error = format!( - "{:#}", - result.expect_err("view bypassed the environment read interface") - ); - assert!(!error.contains("not found"), "view failed before dispatch: {error}"); - // A valid view and a forbidden view share one actual subscription - // request. It must fail as a whole, without an initial-success message. - let request_id = 880; - let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { - query_strings: ["SELECT * FROM my_player".into(), format!("SELECT * FROM {view}").into()].into(), - request_id, - }); - let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; - let message = next_message(handle).await; - let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { - panic!("failed view subscription returned a success or unexpected message: {message:?}"); - }; - assert_eq!(message.request_id, Some(request_id)); - let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { - panic!("forbidden view subscription returned rows"); - }; - assert!(!error.message.is_empty()); - assert!(!error.message.contains("not found")); + // Host access must be denied, independently of the general view-error + // transport contract tracked in #5912. A failed view may yield no rows. + match result { + Ok(result) => assert!(result.rows.is_empty(), "forbidden view exposed rows"), + Err(error) => { + let error = format!("{error:#}"); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + assert!(!error.contains("root-visible"), "view error exposed environment data"); + } + } } + // HTTP routes are root entries today. Calling an exported child callback // as an ordinary helper retains that root entry's authority. assert_eq!( @@ -127,63 +115,25 @@ async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { ); } -// SQL, subscription materialization, and ordinary reducers all use the same -// main Wasmtime instance. The fixture view sets a guest-global marker and traps; -// the next reducer checks the marker is absent in the replacement instance. -async fn check_wasm_trap_disposal(handle: &mut ModuleHandle) { - let module = handle.client.module(); - let view = "SELECT * FROM environment_trap"; - let result = spacetimedb::sql::execute::run( - module.relational_db().clone(), - view.to_string(), - AuthCtx::for_current(Identity::ZERO), - Some(module.info.subscriptions.clone()), - Some(module.clone()), - &mut vec![], - ) - .await; - let error = format!("{:#}", result.expect_err("trapped view returned SQL success")); - assert!(!error.contains("not found"), "view failed before dispatch: {error}"); - expect_clean_wasm_instance(&module).await; - - // Keep the existing successful ENV-view subscription while adding this - // separate query. The failing request must report an error, not rows. - let request_id = 890; - let subscribe = ws_v1::ClientMessage::::SubscribeSingle(ws_v1::SubscribeSingle { - query: view.into(), - request_id, - query_id: ws_v1::QueryId::new(890), - }); - let _ = handle.send(bsatn::to_vec(&subscribe).unwrap()).await; - let message = next_message(handle).await; - let OutboundMessage::V1(SerializableMessage::Subscription(message)) = message else { - panic!("trapped view subscription returned success or an unexpected message: {message:?}"); +// Qualification can pin locally built inputs without invoking a nested build. +// Ordinary test runs keep the existing compilation path when no pin is supplied. +fn compiled_fixture(name: &str) -> CompiledModule { + use spacetimedb::messages::control_db::HostType; + let input = match name { + "environment-test" => Some(("SPACETIMEDB_ENV_RUST_MODULE", HostType::Wasm)), + "module-test-ts" => Some(("SPACETIMEDB_ENV_TYPESCRIPT_MODULE", HostType::Js)), + "module-test-cs" => Some(("SPACETIMEDB_ENV_CSHARP_MODULE", HostType::Wasm)), + _ => None, }; - assert_eq!(message.request_id, Some(request_id)); - let spacetimedb::client::messages::SubscriptionResult::Error(error) = message.result else { - panic!("trapped view subscription returned rows"); - }; - assert!(!error.message.is_empty()); - assert!(!error.message.contains("not found")); - expect_clean_wasm_instance(&module).await; -} - -async fn expect_clean_wasm_instance(module: &ModuleHost) { - module - .call_reducer( - Identity::ZERO, - None, - None, - None, - None, - "expect_environment", - FunctionArgs::Bsatn(bsatn::to_vec(&product!["MISSING", None::]).unwrap().into()), - ) - .await - .unwrap() - .outcome - .into_result() - .unwrap(); + if let Some((key, host_type)) = input + && let Some(path) = std::env::var_os(key) + { + let path = std::path::PathBuf::from(path); + assert!(path.is_absolute() && path.is_file(), "invalid explicit module artifact"); + CompiledModule::from_artifact(name, host_type, path) + } else { + CompiledModule::compile(name, CompilationMode::Debug) + } } fn exercise_fixture(name: &str) { @@ -195,18 +145,7 @@ fn exercise_fixture(name: &str) { } else { Values::new() }; - // NativeAOT's WebAssembly compiler requires a supported compiler host. - // Allow this one fixture to consume the exact artifact built there while - // exercising all normal publication and runtime paths below. - let artifact = (name == "module-test-cs") - .then(|| std::env::var_os("SPACETIMEDB_ENV_CSHARP_MODULE")) - .flatten(); - let compiled = match artifact { - Some(path) => { - CompiledModule::from_artifact(name, spacetimedb::messages::control_db::HostType::Wasm, path.into()) - } - None => CompiledModule::compile(name, CompilationMode::Debug), - }; + let compiled = compiled_fixture(name); compiled.with_module_async_with_environment(DEFAULT_CONFIG, initial.clone(), |mut handle| async move { let mut values = initial; for (key, expected) in [ @@ -333,7 +272,6 @@ fn exercise_fixture(name: &str) { .into_result() .unwrap(); } - check_wasm_trap_disposal(&mut handle).await; } if name == "module-test-ts" { check_submodule_scope(&mut handle, &mut values).await; @@ -398,7 +336,7 @@ fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + compiled_fixture("environment-test").with_module_async_with_environment( DEFAULT_CONFIG, initial.clone(), |handle| async move { @@ -499,7 +437,7 @@ fn rust_environment_enums_preserve_exact_typed_mappings() { ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - CompiledModule::compile("environment-test", CompilationMode::Debug).with_module_async_with_environment( + compiled_fixture("environment-test").with_module_async_with_environment( DEFAULT_CONFIG, initial.clone(), |handle| async move { diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs index 99846b25301..c72d1bd8625 100644 --- a/modules/environment-test/src/lib.rs +++ b/modules/environment-test/src/lib.rs @@ -1,7 +1,4 @@ use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; -use std::sync::atomic::{AtomicBool, Ordering}; - -static VIEW_TRAP_ENTERED: AtomicBool = AtomicBool::new(false); type RequiredString = String; type OptionalString = Option; @@ -65,10 +62,6 @@ pub fn expect_typed_environment(ctx: &ReducerContext, required: u8, optional: Op #[spacetimedb::reducer] pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { - assert!( - !VIEW_TRAP_ENTERED.load(Ordering::Relaxed), - "trapped Wasm view instance was reused" - ); assert_eq!(ctx.env.get(&key), expected); assert_eq!(ctx.as_read_only().env.get(&key), expected); assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); @@ -111,14 +104,6 @@ pub fn environment_value(ctx: &AnonymousViewContext) -> Option Some(EnvironmentValue { value }) } -/// A Rust panic is a Wasm trap. The next main-instance reducer verifies that -/// the guest-global marker did not survive the failed SQL/subscription call. -#[spacetimedb::view(accessor = environment_trap, public)] -pub fn environment_trap(_ctx: &AnonymousViewContext) -> Option { - VIEW_TRAP_ENTERED.store(true, Ordering::Relaxed); - panic!("intentional environment view trap"); -} - /// Hand-written ABI callers cannot retain unbounded host allocations. #[spacetimedb::reducer] pub fn bounded_environment_sources(_ctx: &ReducerContext) { From a7c7799705ea918717bbaeacf0f26c3dcc0a7213 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Thu, 10 Sep 2026 08:34:10 -0700 Subject: [PATCH 18/19] Add to C++ HandlerContext --- crates/bindings-cpp/include/spacetimedb/handler_context.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h index cb9b4abe84c..80f42d821c5 100644 --- a/crates/bindings-cpp/include/spacetimedb/handler_context.h +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ namespace SpacetimeDB { struct HandlerContext { Timestamp timestamp; + Environment env; HttpClient http; private: From e98ef53982b73694d437b3cbed2161c534acbd6f Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 19:13:25 -0400 Subject: [PATCH 19/19] Preserve Windows inspector PATH and clarify secret visibility --- crates/cli/src/schema_extract.rs | 20 ++++++--- crates/cli/src/schema_extract/tests.rs | 45 +++++++++++++++++++ .../00700-environment-variables.md | 4 ++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs index c00481ae40f..76b2ec64159 100644 --- a/crates/cli/src/schema_extract.rs +++ b/crates/cli/src/schema_extract.rs @@ -97,6 +97,16 @@ impl Observation { } } +fn configure_inspector_environment(command: &mut tokio::process::Command) { + command.env_clear(); + // Windows needs PATH to resolve the inspector's dependent DLLs. Keep + // other inherited variables, including publish-time secrets, isolated. + #[cfg(windows)] + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } +} + async fn inspect_observed( extractor: PathBuf, program: Vec, @@ -114,18 +124,18 @@ async fn inspect_observed( tokio::fs::write(&module, program) .await .context("Cannot prepare module inspection input")?; - let mut child = tokio::process::Command::new(extractor) + let mut command = tokio::process::Command::new(extractor); + command .arg("extract-schema") .arg(&module) .arg("--host-type") .arg(host_type.to_ascii_lowercase()) - .env_clear() .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) - .kill_on_drop(true) - .spawn() - .context("Cannot start local module schema inspection")?; + .kill_on_drop(true); + configure_inspector_environment(&mut command); + let mut child = command.spawn().context("Cannot start local module schema inspection")?; observation.started(child.id()); let mut output = Vec::new(); let mut stdout = child diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs index d8c6daea04a..7738fb8a5f3 100644 --- a/crates/cli/src/schema_extract/tests.rs +++ b/crates/cli/src/schema_extract/tests.rs @@ -3,6 +3,51 @@ use spacetimedb_lib::environment::{ EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration, EnvironmentSchema, }; +#[tokio::test] +async fn inspector_environment_excludes_secrets_and_preserves_windows_dll_search() { + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "schema_extract::tests::inspector_environment_child", + "--ignored", + ]) + .env("STDB_INSPECTOR_TEST_SECRET", "must-not-reach-inspector"); + configure_inspector_environment(&mut command); + let explicit: Vec<_> = command.as_std().get_envs().collect(); + #[cfg(windows)] + { + let path = std::env::var_os("PATH"); + let expected: Vec<_> = path + .as_deref() + .map(|value| (std::ffi::OsStr::new("PATH"), Some(value))) + .into_iter() + .collect(); + assert_eq!(explicit, expected); + } + #[cfg(not(windows))] + assert!(explicit.is_empty()); + let output = command.output().await.unwrap(); + assert!( + output.status.success(), + "isolated inspector process failed: {}", + String::from_utf8_lossy(&output.stdout) + ); +} + +#[test] +#[ignore = "invoked by the isolated inspector environment regression"] +fn inspector_environment_child() { + // CoreFoundation adds this variable during process initialization on macOS, + // even when the parent supplies an empty environment. + #[cfg(target_os = "macos")] + assert!(std::env::vars_os().all(|(key, _)| key == "__CF_USER_TEXT_ENCODING")); + #[cfg(windows)] + assert!(std::env::vars_os().all(|(key, _)| key.as_encoded_bytes().eq_ignore_ascii_case(b"PATH"))); + #[cfg(all(not(windows), not(target_os = "macos")))] + assert!(std::env::vars_os().next().is_none(), "unexpected inherited variable"); +} + fn schema() -> EnvironmentSchema { EnvironmentSchema::new(vec![ Declaration { diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index b1fc7fe9d9f..d05c96ed526 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -275,4 +275,8 @@ Use an ordinary [private table](../00300-tables/00400-access-permissions.md) whe Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. +:::warning Secret visibility +Private tables can store dynamically editable secrets, but changing a table to public can expose its contents to clients. Environment variables have no public-table visibility setting. With either approach, module code can still expose secrets through return values or logs. +::: + Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish.