diff --git a/Cargo.lock b/Cargo.lock index 6f9455400ec..384c70585bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2298,6 +2298,13 @@ dependencies = [ "log", ] +[[package]] +name = "environment-test" +version = "0.0.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 9f7852863c8..afdde4e3253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/environment-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index dc5d1433fce..695c91f5b5f 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -28,6 +28,33 @@ add_library(spacetimedb::spacetimedb_cpp_library ALIAS spacetimedb_cpp_library) target_sources(spacetimedb_cpp_library PRIVATE ${LIBRARY_SOURCES}) +# A declared Environment adds named methods to the context member type. Every +# SDK and module translation unit must see the same prelude before any context. +if(NOT DEFINED SPACETIMEDB_ENV_HEADER) + set(SPACETIMEDB_ENV_HEADER "" CACHE FILEPATH "Absolute module environment declaration header") +endif() +if(SPACETIMEDB_ENV_HEADER) + if(NOT IS_ABSOLUTE "${SPACETIMEDB_ENV_HEADER}" OR NOT EXISTS "${SPACETIMEDB_ENV_HEADER}") + message(FATAL_ERROR "SPACETIMEDB_ENV_HEADER must name an existing absolute declaration header") + endif() + # The standalone WASI ABI shims have no context or Environment type. Their + # hand-written ABI declarations must not include the C++ standard library's + # WASI declarations through the module prelude. + set_source_files_properties(src/abi/wasi_shims.cpp PROPERTIES + COMPILE_DEFINITIONS SPACETIMEDB_WASI_SHIMS=1) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/include/spacetimedb/environment_prelude.h.in + ${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h + @ONLY) + set(SPACETIMEDB_ENV_PRELUDE "${CMAKE_CURRENT_BINARY_DIR}/include/spacetimedb/environment_prelude.h") + target_compile_definitions(spacetimedb_cpp_library PUBLIC SPACETIMEDB_ENV_HEADER_ACTIVE=1) + if(MSVC) + target_compile_options(spacetimedb_cpp_library PUBLIC "/FI${SPACETIMEDB_ENV_PRELUDE}") + else() + target_compile_options(spacetimedb_cpp_library PUBLIC -include "${SPACETIMEDB_ENV_PRELUDE}") + endif() +endif() + # Require C++20 for consumers of this library without forcing global flags target_compile_features(spacetimedb_cpp_library PUBLIC cxx_std_20) target_compile_definitions(spacetimedb_cpp_library PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..155fafedfda 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -275,3 +275,51 @@ See the `modules/*-cpp/src/` directory for example modules: This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. + +### Declared environment + +Declare the complete environment schema in a dedicated header, before other SDK +includes. Values are supplied at publish time and are never embedded in this +header. For example, `environment.h`: + +```cpp +#pragma once +#include +SPACETIMEDB_ENV( + (API_URL, std::string), + (MODE, std::string, ("prod", "dev")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +Select it in your project's `CMakeLists.txt` **before** adding the SDK directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +add_subdirectory(path/to/bindings-cpp sdk) +add_executable(my_module src/lib.cpp) +target_link_libraries(my_module PRIVATE spacetimedb_cpp_library) +``` + +The SDK's CMake target includes the declaration header before every SDK and module +translation unit that uses contexts. The standalone WASI ABI shim is excluded +because it has no SDK context types. This keeps the context type consistent; +including the header in only one source file is insufficient. The normal module +source can then use the usual umbrella include: + +```cpp +#include +void example(SpacetimeDB::ReducerContext ctx) { + std::string mode = ctx.env.MODE(); + std::optional level = ctx.env.LOG_LEVEL(); +} +``` + +An empty `SPACETIMEDB_ENV()` or an omitted declaration selects an empty schema. +`get`, C++ keywords, and names colliding with the accessor type do not create named +methods; use the checked `ctx.env.get("get")` form for such declared names. Unknown +keys fail at runtime, including when the module declares no environment variables. +Only the host's root module entry may read environment values. Ordinary C++ helper +calls retain their caller's host scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index 32990f156d4..f3ad213452f 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -73,6 +73,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; +using ::env_get; // ===== Procedure Transactions ===== using ::procedure_start_mut_tx; diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 99dc067c147..502cd408be3 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,6 +39,9 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -59,6 +62,9 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +STDB_IMPORT_10_6(env_get) +Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); + // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) Status table_id_from_name(const uint8_t* name_ptr, size_t name_len, TableId* out); diff --git a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h index 5fcba095c76..a61fc3bcc59 100644 --- a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h +++ b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h @@ -124,10 +124,10 @@ namespace SpacetimeDB::bsatn { template std::optional read_optional() { uint8_t tag = read_u8(); - if (tag == 0) { - return std::nullopt; - } else if (tag == 1) { + if (tag == 0) { // Some, matching the canonical BSATN option type. return SpacetimeDB::bsatn::deserialize(*this); + } else if (tag == 1) { // None. + return std::nullopt; } else { std::abort(); // Invalid optional tag in BSATN deserialization } diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h new file mode 100644 index 00000000000..06020dcee2f --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -0,0 +1,107 @@ +#ifndef SPACETIMEDB_ENVIRONMENT_H +#define SPACETIMEDB_ENVIRONMENT_H +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { +/// Read-only database environment. Reads use the current transaction, or a +/// short snapshot in a procedure outside a transaction. Values are not cached. +class EnvironmentBase { +public: + std::optional get(std::string_view key) const { + if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); + BytesSource source{0}; + if (FFI::env_get(reinterpret_cast(key.data()), static_cast(key.size()), &source) != Status(0)) + LOG_PANIC("environment read failed"); + if (source == BytesSource{0}) return std::nullopt; + std::array buffer; + std::string value; + for (;;) { + size_t len = buffer.size(); + const auto status = FFI::bytes_source_read(source, buffer.data(), &len); + if ((status != 0 && status != -1) || len > buffer.size()) LOG_PANIC("environment source read failed"); + value.append(reinterpret_cast(buffer.data()), len); + if (status == -1) return value; + if (len == 0) LOG_PANIC("environment source made no progress"); + } + } +}; + +namespace Internal { +inline std::vector& environment_declarations() { + static std::vector declarations; + return declarations; +} + +template +inline constexpr bool environment_string = std::is_same_v || std::is_same_v>; + +template +T read_environment(std::string_view key) { + static_assert(environment_string, "Environment declarations require string or optional"); + auto value = EnvironmentBase{}.get(key); + if constexpr (std::is_same_v) { + if (!value) LOG_PANIC("required environment value is absent"); + return std::move(*value); + } else { return value; } +} + +template +EnvironmentDeclaration declare_environment(std::string name) { + static_assert(environment_string, "Environment declarations require string or optional"); + EnvironmentConstraint constraint; + constraint.set<0>(std::monostate{}); + return {std::move(name), std::move(constraint), std::is_same_v>}; +} + +// Preserve the complete source literal, including embedded NUL bytes. Implicit +// conversion through std::string(const char*) would truncate those constraints. +struct EnvironmentLiteral { + std::string value; + template + EnvironmentLiteral(const char (&text)[N]) : value(text, N - 1) {} + EnvironmentLiteral(std::string text) : value(std::move(text)) {} +}; + +template +EnvironmentDeclaration declare_environment(std::string name, std::initializer_list allowed) { + auto declaration = declare_environment(std::move(name)); + if (allowed.size() == 0) LOG_PANIC("environment literal union cannot be empty"); + std::unordered_set unique; + std::vector values; + values.reserve(allowed.size()); + for (const auto& literal : allowed) { + const auto& value = literal.value; + if (value.size() > 8192 || !unique.insert(value).second) LOG_PANIC("invalid environment literal union"); + values.push_back(value); + } + if (values.size() == 1) declaration.constraint.template set<1>(std::move(values.front())); + else declaration.constraint.template set<2>(std::move(values)); + return declaration; +} + +inline void validate_environment_declarations() { + const auto& declarations = environment_declarations(); + if (declarations.size() > 256) LOG_PANIC("too many environment declarations"); + std::unordered_set keys; + const auto initial = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; }; + for (const auto& declaration : declarations) { + const auto& key = declaration.name; + if (key.empty() || key.size() > 256 || !initial(key[0]) || !keys.insert(key).second) LOG_PANIC("invalid environment declaration name"); + for (char c : key) if (!initial(c) && !(c >= '0' && c <= '9')) LOG_PANIC("invalid environment declaration name"); + } +} +} + +#ifndef SPACETIMEDB_ENV_DECLARATION +class Environment : public EnvironmentBase {}; +#endif +} +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/environment_declaration.h b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h new file mode 100644 index 00000000000..b3df8028d5b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_declaration.h @@ -0,0 +1,153 @@ +#pragma once +// Put SPACETIMEDB_ENV in a module declaration header selected by CMake's +// SPACETIMEDB_ENV_HEADER, before including any SDK context or umbrella header. +#ifndef SPACETIMEDB_ENV_HEADER_ACTIVE +#error "Configure SPACETIMEDB_ENV_HEADER before adding the SDK CMake directory" +#endif +#ifdef SPACETIMEDB_ENVIRONMENT_H +#error "The environment declaration header must precede every SDK include" +#endif +#define SPACETIMEDB_ENV_DECLARATION 1 +#include + +#define STDB_ENV_CAT_I(a, b) a##b +#define STDB_ENV_CAT(a, b) STDB_ENV_CAT_I(a, b) +#define STDB_ENV_SECOND(a, b, ...) b +#define STDB_ENV_PROBE() ignored, 1 +#define STDB_ENV_CHECK(...) STDB_ENV_SECOND(__VA_ARGS__, 0) +#define STDB_ENV_RESERVED(name) STDB_ENV_CHECK(STDB_ENV_CAT(STDB_ENV_RESERVED_, name)) +#define STDB_ENV_KEEP_0(...) __VA_ARGS__ +#define STDB_ENV_KEEP_1(...) +#define STDB_ENV_RESERVED_alignas STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_alignof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_and_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_asm STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_cancel STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_commit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_atomic_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_auto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitand STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bitor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_bool STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_break STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_case STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_catch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char8_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char16_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_char32_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_class STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_compl STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_concept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_consteval STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_constinit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_const_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_continue STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_await STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_co_yield STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_decltype STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_default STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_delete STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_do STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_double STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_dynamic_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_else STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_enum STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_explicit STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_export STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_extern STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_false STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_float STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_for STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_friend STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_goto STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_if STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_inline STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_int STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_long STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_mutable STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_namespace STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_new STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_noexcept STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_not_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_nullptr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_operator STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_or_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_private STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_protected STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_public STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reflexpr STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_register STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_reinterpret_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_requires STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_return STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_short STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_signed STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_sizeof STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_assert STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_static_cast STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_struct STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_switch STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_synchronized STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_template STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_this STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_thread_local STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_throw STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_true STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_try STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typedef STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typeid STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_typename STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_union STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_unsigned STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_using STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_virtual STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_void STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_volatile STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_wchar_t STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_while STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_xor_eq STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_get STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_Environment STDB_ENV_PROBE() +#define STDB_ENV_RESERVED_EnvironmentBase STDB_ENV_PROBE() + +#define STDB_ENV_PARENS () +#define STDB_ENV_EVAL1(...) __VA_ARGS__ +#define STDB_ENV_EVAL2(...) STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(STDB_ENV_EVAL1(__VA_ARGS__)))) +#define STDB_ENV_EVAL3(...) STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(STDB_ENV_EVAL2(__VA_ARGS__)))) +#define STDB_ENV_EVAL4(...) STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(STDB_ENV_EVAL3(__VA_ARGS__)))) +#define STDB_ENV_EVAL(...) STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(STDB_ENV_EVAL4(__VA_ARGS__)))) +#define STDB_ENV_EACH(macro, ...) __VA_OPT__(STDB_ENV_EVAL(STDB_ENV_EACH_I(macro, __VA_ARGS__))) +#define STDB_ENV_EACH_I(macro, tuple, ...) macro tuple __VA_OPT__(STDB_ENV_AGAIN STDB_ENV_PARENS (macro, __VA_ARGS__)) +#define STDB_ENV_AGAIN() STDB_ENV_EACH_I +#define STDB_ENV_UNPAREN(...) __VA_ARGS__ +#define STDB_ENV_MEMBER(name, type, ...) \ + STDB_ENV_CAT(STDB_ENV_KEEP_, STDB_ENV_RESERVED(name))( \ + type name() const { return ::SpacetimeDB::Internal::read_environment(#name); } \ + ) +#define STDB_ENV_METADATA(name, type, ...) \ + ::SpacetimeDB::Internal::declare_environment(#name __VA_OPT__(, { STDB_ENV_UNPAREN __VA_ARGS__ })), + +/// Declare the complete schema, never live values. One declaration per module. +/// Empty SPACETIMEDB_ENV() is supported. Reserved names retain generic get(). +#define SPACETIMEDB_ENV(...) \ + namespace SpacetimeDB { \ + class Environment : public EnvironmentBase { \ + public: STDB_ENV_EACH(STDB_ENV_MEMBER, __VA_ARGS__) \ + }; \ + namespace Internal { \ + inline const bool environment_schema_registered = [] { \ + environment_declarations() = { STDB_ENV_EACH(STDB_ENV_METADATA, __VA_ARGS__) }; \ + validate_environment_declarations(); \ + return true; \ + }(); \ + } \ + } diff --git a/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in new file mode 100644 index 00000000000..4d812bb6d05 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment_prelude.h.in @@ -0,0 +1,7 @@ +#pragma once + +// The ABI shim translation unit has no SDK contexts. All context-bearing SDK +// and module translation units see the same declared Environment definition. +#ifndef SPACETIMEDB_WASI_SHIMS +#include "@SPACETIMEDB_ENV_HEADER@" +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h index cb9b4abe84c..80f42d821c5 100644 --- a/crates/bindings-cpp/include/spacetimedb/handler_context.h +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ namespace SpacetimeDB { struct HandlerContext { Timestamp timestamp; + Environment env; HttpClient http; private: diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h new file mode 100644 index 00000000000..0b2e45c8ae6 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentConstraint.g.h @@ -0,0 +1,7 @@ +#pragma once +#include "../autogen_base.h" +#include +#include +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_TAGGED_ENUM(EnvironmentConstraint, std::monostate, std::string, std::vector) +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h new file mode 100644 index 00000000000..2d83f3c9d3b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/EnvironmentDeclaration.g.h @@ -0,0 +1,15 @@ +#pragma once +#include "EnvironmentConstraint.g.h" +namespace SpacetimeDB::Internal { +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(EnvironmentDeclaration) { + std::string name; + EnvironmentConstraint constraint; + bool optional; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, name); + ::SpacetimeDB::bsatn::serialize(writer, constraint); + ::SpacetimeDB::bsatn::serialize(writer, optional); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(name, constraint, optional) +}; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index ea2e4b5ec85..551fb3bc636 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -27,8 +27,9 @@ #include "RawScheduleDefV10.g.h" #include "RawViewPrimaryKeyDefV10.g.h" #include "RawHttpHandlerDefV10.g.h" +#include "EnvironmentDeclaration.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index ebfd958af7c..35c93c31475 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -15,6 +15,8 @@ #include #include +#include + namespace SpacetimeDB { /** @@ -57,6 +59,7 @@ struct ProcedureContext { Identity sender_; public: + Environment env; // Timestamp when the procedure was invoked Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 41865b14f3a..14a6028c606 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -13,6 +13,8 @@ // Include database for DatabaseContext #include +#include + namespace SpacetimeDB { // Enhanced ReducerContext with database access - matches Rust pattern @@ -21,6 +23,7 @@ struct ReducerContext { Identity sender_; public: + Environment env; // Core fields - sender is exposed via sender() like Rust, other fields remain directly accessible std::optional connection_id; Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/tx_context.h b/crates/bindings-cpp/include/spacetimedb/tx_context.h index 1a04ef027e1..c874a22b4ff 100644 --- a/crates/bindings-cpp/include/spacetimedb/tx_context.h +++ b/crates/bindings-cpp/include/spacetimedb/tx_context.h @@ -56,6 +56,7 @@ struct TxContext { // In C++, we explicitly expose references where possible and provide // accessors for fields exposed as methods on ReducerContext. DatabaseContext& db; + const Environment& env; const Timestamp& timestamp; const std::optional& connection_id; @@ -63,6 +64,7 @@ struct TxContext { explicit TxContext(ReducerContext& ctx) : ctx_(ctx), db(ctx.db), + env(ctx.env), timestamp(ctx.timestamp), connection_id(ctx.connection_id) {} diff --git a/crates/bindings-cpp/include/spacetimedb/view_context.h b/crates/bindings-cpp/include/spacetimedb/view_context.h index 63102699e42..6985c2e55b0 100644 --- a/crates/bindings-cpp/include/spacetimedb/view_context.h +++ b/crates/bindings-cpp/include/spacetimedb/view_context.h @@ -7,6 +7,8 @@ #include // For ReadOnlyDatabaseContext #include +#include + namespace SpacetimeDB { /** @@ -41,6 +43,7 @@ struct ViewContext { public: // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors @@ -76,6 +79,7 @@ struct ViewContext { struct AnonymousViewContext { // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; QueryBuilder from; // Constructors diff --git a/crates/bindings-cpp/src/abi/wasi_shims.cpp b/crates/bindings-cpp/src/abi/wasi_shims.cpp index 430a1f9a858..37548c40d99 100644 --- a/crates/bindings-cpp/src/abi/wasi_shims.cpp +++ b/crates/bindings-cpp/src/abi/wasi_shims.cpp @@ -150,7 +150,7 @@ __wasi_errno_t __wasi_fd_write(__wasi_fd_t fd, const __wasi_ciovec_t* iovs, // Make a single console_log call with the complete message uint8_t log_level = (fd == STDERR_FILENO) ? 1 : 2; // 1=WARN, 2=INFO - console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, + console_log(log_level, CSTR("wasi"), CSTR(__FILE__), __LINE__, buffer, offset); // Clean up heap allocation if needed @@ -190,4 +190,4 @@ void emscripten_notify_memory_growth(int32_t) { // No-op - memory growth is handled by the runtime } -} // extern "C" \ No newline at end of file +} // extern "C" diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index a931b79a7c2..be5bbd65035 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -1,3 +1,4 @@ +#include "spacetimedb/environment.h" #include "spacetimedb/internal/v10_builder.h" #include "spacetimedb/internal/autogen/AlgebraicType.g.h" #include "spacetimedb/internal/autogen/ProductType.g.h" @@ -315,6 +316,10 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { v10_module.sections.push_back(std::move(section_rls)); } + validate_environment_declarations(); + RawModuleDefV10Section section_environment; + section_environment.set<15>(environment_declarations()); + v10_module.sections.push_back(std::move(section_environment)); return v10_module; } diff --git a/crates/bindings-cpp/tests/environment/CMakeLists.txt b/crates/bindings-cpp/tests/environment/CMakeLists.txt new file mode 100644 index 00000000000..34223081b4a --- /dev/null +++ b/crates/bindings-cpp/tests/environment/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.16) +project(environment_declaration_tests LANGUAGES CXX) +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/declarations.h") +add_subdirectory(../.. sdk) +add_executable(environment_declaration_tests main.cpp other.cpp) +target_link_libraries(environment_declaration_tests PRIVATE spacetimedb_cpp_library) +enable_testing() +add_test(NAME environment_declaration_tests COMMAND environment_declaration_tests) +# This separate target deliberately has no forced prelude, modeling an existing +# module with no declarations and the generic checked accessor only. +add_executable(environment_fallback_tests fallback.cpp) +target_include_directories(environment_fallback_tests PRIVATE ../../include) +target_compile_features(environment_fallback_tests PRIVATE cxx_std_20) +add_test(NAME environment_fallback_tests COMMAND environment_fallback_tests) +set_tests_properties(environment_declaration_tests environment_fallback_tests PROPERTIES TIMEOUT 10) diff --git a/crates/bindings-cpp/tests/environment/declarations.h b/crates/bindings-cpp/tests/environment/declarations.h new file mode 100644 index 00000000000..ab7e2bba1b5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/declarations.h @@ -0,0 +1,11 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (FOOBAR, std::string), + (ENABLE_EMAIL, std::string, ("true", "false")), + (LOG_LEVEL, std::optional, ("debug", "info", "error")), + (DEPLOYMENT_KIND, std::string, ("production")), + (get, std::optional), + (class, std::string), + (NUL_LITERAL, std::string, ("a\0b")) +) diff --git a/crates/bindings-cpp/tests/environment/fallback.cpp b/crates/bindings-cpp/tests/environment/fallback.cpp new file mode 100644 index 00000000000..27d94126011 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/fallback.cpp @@ -0,0 +1,8 @@ +#include +#include +#include + +int main() { + static_assert(std::is_same_v>); + assert(SpacetimeDB::Internal::environment_declarations().empty()); +} diff --git a/crates/bindings-cpp/tests/environment/main.cpp b/crates/bindings-cpp/tests/environment/main.cpp new file mode 100644 index 00000000000..7d0273a8bc5 --- /dev/null +++ b/crates/bindings-cpp/tests/environment/main.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include + +using namespace SpacetimeDB; +std::string from_another_translation_unit(); +namespace { std::string payload; size_t position; unsigned calls; } +extern "C" Status env_get(const uint8_t* key, uint32_t length, BytesSource* out) { + const std::string name(reinterpret_cast(key), length); + ++calls; + if (name == "LOG_LEVEL") { *out = BytesSource{0}; return Status{0}; } + if (name == "FOOBAR") payload = calls == 1 ? "first" : "updated"; + else if (name == "ENABLE_EMAIL") payload = "false"; + else if (name == "DEPLOYMENT_KIND") payload = "production"; + else if (name == "get") payload = "reserved"; + else if (name == "class") payload = "keyword"; + else if (name == "NUL_LITERAL") payload = std::string("a\0b", 3); + else return Status{1}; + position = 0; + *out = BytesSource{1}; + return Status{0}; +} +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* size) { + *size = std::min(*size, payload.size() - position); + std::memcpy(out, payload.data() + position, *size); + position += *size; + return position == payload.size() ? -1 : 0; +} +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, uint32_t, const uint8_t*, size_t) {} + +int main() { + Environment env; + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + assert(env.FOOBAR() == "first"); + assert(from_another_translation_unit() == "updated"); + assert(env.ENABLE_EMAIL() == "false"); + assert(!env.LOG_LEVEL()); + assert(env.DEPLOYMENT_KIND() == "production"); + assert(env.get("get") == "reserved"); + assert(env.get("class") == "keyword"); + assert(env.NUL_LITERAL() == std::string("a\0b", 3)); + const auto& entries = Internal::environment_declarations(); + assert(entries.size() == 7); + assert(entries[0].name == "FOOBAR" && entries[0].constraint.get_tag() == 0 && !entries[0].optional); + assert(entries[1].constraint.get_tag() == 2 && entries[1].constraint.get<2>() == std::vector({"true", "false"})); + assert(entries[2].optional); + assert(entries[3].constraint.get_tag() == 1 && entries[3].constraint.get<1>() == "production"); + assert(entries[6].constraint.get<1>() == std::string("a\0b", 3)); + Internal::RawModuleDefV10Section section; + section.set<15>(entries); + assert(section.get_tag() == 15); + assert(section.get<15>() == entries); +} diff --git a/crates/bindings-cpp/tests/environment/other.cpp b/crates/bindings-cpp/tests/environment/other.cpp new file mode 100644 index 00000000000..c9ffe40f9fd --- /dev/null +++ b/crates/bindings-cpp/tests/environment/other.cpp @@ -0,0 +1,6 @@ +#include +#include +std::string from_another_translation_unit() { + return SpacetimeDB::Environment{}.FOOBAR(); +} +static_assert(std::is_same_v().env.FOOBAR()), std::string>); diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 0ced4e0194c..da7b8705e1c 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,6 +11,7 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp + environment_unit_tests.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp new file mode 100644 index 00000000000..c6bd0f58cd8 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/environment_unit_tests.cpp @@ -0,0 +1,50 @@ +#include "test_harness.h" +#include "spacetimedb/environment.h" +#include "spacetimedb/bsatn/reader.h" +#include +#include + +using namespace SpacetimeDB; + +namespace { +size_t payload_offset; +std::string payload; +} + +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, payload.size() - payload_offset); + std::memcpy(out, payload.data() + payload_offset, *len); + payload_offset += *len; + return payload_offset == payload.size() ? -1 : 0; +} + +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + payload = std::string(8192, 'x'); + ASSERT_EQ(payload, env.get("LARGE").value()); + payload = std::string("a\0b", 3); + ASSERT_EQ(payload, env.get("NUL").value()); + payload = "updated"; + ASSERT_EQ(payload, env.get("NUL").value()); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs new file mode 100644 index 00000000000..efb8997828a --- /dev/null +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -0,0 +1,183 @@ +namespace SpacetimeDB.Codegen.Tests; + +using System.Reflection; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +public static class EnvironmentTests +{ + // A controlled host seam allows the generated accessor code to execute. + // Live value access remains delegated to DatabaseEnvironment.Get. + private const string Host = """ + #nullable enable + namespace SpacetimeDB { + [System.AttributeUsage(System.AttributeTargets.Struct)] + public sealed class EnvAttribute : System.Attribute {} + [System.AttributeUsage(System.AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : System.Attribute {} + public readonly struct DatabaseEnvironment { + public string? Get(string key) => Host.Get(key); + } + public static class Host { + public static int Reads; + public static string? Get(string key) { + Reads++; + return key switch { + "REQUIRED" => Reads.ToString(), "OPTIONAL" => null, + "MODE" => "prod", "Get" => "reserved", "class" => "keyword", + "ModuleEnvironment" or "Equals" or "GetHashCode" or "ToString" or + "Finalize" or "GetType" or "MemberwiseClone" => key, + _ => throw new System.InvalidOperationException("undeclared environment key") + }; + } + } + } + namespace SpacetimeDB.Internal { + public abstract record EnvironmentConstraint { + public sealed record AnyString(System.ValueTuple Value) : EnvironmentConstraint; + public sealed record Literal(string Value) : EnvironmentConstraint; + public sealed record OneOf(System.Collections.Generic.List Value) : EnvironmentConstraint; + } + public sealed record EnvironmentDeclaration(string Name, EnvironmentConstraint Constraint, bool Optional); + public static class Module { + public static System.Collections.Generic.List Declarations = new(); + public static void RegisterEnvironment(EnvironmentDeclaration value) => Declarations.Add(value); + } + } + """; + + private static (Compilation Compilation, GeneratorDriverRunResult Result) Generate( + string declaration + ) + { + var references = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!) + .Split(Path.PathSeparator) + .Select(path => MetadataReference.CreateFromFile(path)); + var parse = new CSharpParseOptions(LanguageVersion.Preview); + var compilation = CSharpCompilation.Create( + "EnvironmentFixture" + Guid.NewGuid().ToString("N"), + [CSharpSyntaxTree.ParseText(Host + declaration, parse)], + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary) + ); + GeneratorDriver driver = CSharpGeneratorDriver.Create( + [new EnvironmentGenerator().AsSourceGenerator()], + parseOptions: parse + ); + driver = driver.RunGeneratorsAndUpdateCompilation(compilation, out var output, out _); + return (output, driver.GetRunResult()); + } + + [Fact] + public static void NamedAccessorsKeepCheckedReadsAndRegisterCanonicalConstraints() + { + var (compilation, result) = Generate( + """ + [SpacetimeDB.Env] public struct Declarations { + public string REQUIRED; + public string? OPTIONAL; + [SpacetimeDB.EnvValues("prod", "dev")] public string MODE; + [SpacetimeDB.EnvValues("reserved")] public string Get; + public string @class; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.REQUIRED == env.REQUIRED || env.OPTIONAL != null || env.MODE != "prod" || + env.Get("Get") != "reserved" || env.@class != "keyword") throw new System.Exception("bad accessor"); + try { env.Get("UNKNOWN"); throw new System.Exception("unchecked generic read"); } + catch (System.InvalidOperationException) {} + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 5 || declarations[0].Optional || !declarations[1].Optional || + declarations[0].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.AnyString || + declarations[2].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.OneOf { Value.Count: 2 } || + declarations[3].Constraint is not SpacetimeDB.Internal.EnvironmentConstraint.Literal { Value: "reserved" }) + throw new System.Exception("bad metadata"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty("Get")); + } + + [Theory] + [InlineData("Get")] + [InlineData("ModuleEnvironment")] + [InlineData("Equals")] + [InlineData("GetHashCode")] + [InlineData("ToString")] + [InlineData("Finalize")] + [InlineData("GetType")] + [InlineData("MemberwiseClone")] + public static void ReservedAccessorNamesRemainAvailableThroughCheckedGet(string name) + { + var (compilation, result) = Generate( + $$""" + [SpacetimeDB.Env] public struct Declarations { + public string {{name}}; + } + public static class Usage { + public static void Check() { + var env = new SpacetimeDB.ModuleEnvironment(); + if (env.Get("{{name}}") != "{{(name == "Get" ? "reserved" : name)}}") + throw new System.Exception("bad reserved accessor"); + var declarations = SpacetimeDB.Internal.Module.Declarations; + if (declarations.Count != 1 || declarations[0].Name != "{{name}}") + throw new System.Exception("missing reserved declaration"); + } + } + """ + ); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => + diagnostic.Severity is DiagnosticSeverity.Warning or DiagnosticSeverity.Error + && diagnostic.Location.SourceTree is { } tree + && result.GeneratedTrees.Contains(tree) + ); + using var stream = new MemoryStream(); + var emitted = compilation.Emit(stream); + Assert.True(emitted.Success, string.Join("\n", emitted.Diagnostics)); + var assembly = Assembly.Load(stream.ToArray()); + assembly.GetType("Usage")!.GetMethod("Check")!.Invoke(null, null); + Assert.Null(assembly.GetType("SpacetimeDB.ModuleEnvironment")!.GetProperty(name)); + } + + [Theory] + [InlineData("public int BAD;")] + [InlineData("public static string BAD;")] + [InlineData("[SpacetimeDB.EnvValues()] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(\"x\", \"x\")] public string BAD;")] + [InlineData("[SpacetimeDB.EnvValues(null)] public string BAD;")] + public static void InvalidDeclarationsAreCompileErrors(string field) + { + var (_, result) = Generate("[SpacetimeDB.Env] public struct Declarations {" + field + "}"); + Assert.Contains( + result.Diagnostics, + diagnostic => + diagnostic.Id == "STDBENV001" && diagnostic.Severity == DiagnosticSeverity.Error + ); + } + + [Fact] + public static void EmptySchemaRetainsOnlyGenericAccess() + { + var (compilation, result) = Generate(""); + Assert.Empty(result.Diagnostics); + Assert.DoesNotContain( + compilation.GetDiagnostics(), + diagnostic => diagnostic.Severity == DiagnosticSeverity.Error + ); + Assert.Contains( + "public string? Get(string key)", + result.GeneratedTrees.Single().ToString() + ); + } +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 4133819ef9c..5ecccedb31b 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -88,9 +88,13 @@ public async Task RunAndCheckGenerators( params IIncrementalGenerator[] generators ) => SampleCompilation.AddSyntaxTrees( - (await Task.WhenAll(generators.Select(RunAndCheckGenerator))).SelectMany(output => - output - ) + (await Task.WhenAll(generators.Select(RunAndCheckGenerator))) + .SelectMany(output => output) + .Concat( + generators.Any(generator => generator is SpacetimeDB.Codegen.Module) + ? RunGeneratorAndGetResult(new EnvironmentGenerator()).GeneratedTrees + : [] + ) ); } @@ -333,6 +337,7 @@ public static void @params(ProcedureContext ctx) [ new SpacetimeDB.Codegen.Type().AsSourceGenerator(), new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + new EnvironmentGenerator().AsSourceGenerator(), ], driverOptions: new( disabledOutputs: IncrementalGeneratorOutputKind.None, diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 6638bb12fc8..e81f122020d 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,6 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -731,6 +732,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -809,6 +811,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -849,6 +852,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -858,6 +863,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -892,6 +899,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -905,6 +913,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index 015ec80b3ad..ad729dadac1 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,6 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -133,6 +134,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -211,6 +213,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -251,6 +254,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -260,6 +265,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -275,6 +282,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -288,6 +296,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 82e3d7bdecf..95bbf9c0516 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,6 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -575,6 +576,7 @@ public Uuid NewUuidV7() public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext( @@ -653,6 +655,7 @@ public Uuid NewUuidV7() public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -693,6 +696,8 @@ public Uuid NewUuidV7() public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal ProcedureTxContext(Internal.TxContext inner) : base(inner) { } @@ -702,6 +707,8 @@ internal ProcedureTxContext(Internal.TxContext inner) [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; + internal HandlerTxContext(Internal.TxContext inner) : base(inner) { } @@ -726,6 +733,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -739,6 +747,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs new file mode 100644 index 00000000000..a9f0992da9c --- /dev/null +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -0,0 +1,169 @@ +namespace SpacetimeDB.Codegen; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +/// Compile-time declarations and read-only accessors, never live values. +[Generator] +public sealed class EnvironmentGenerator : IIncrementalGenerator +{ + private static readonly DiagnosticDescriptor InvalidDeclaration = + new( + "STDBENV001", + "Invalid environment declaration", + "{0}", + "SpacetimeDB", + DiagnosticSeverity.Error, + isEnabledByDefault: true + ); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var declarations = context + .SyntaxProvider.ForAttributeWithMetadataName( + "SpacetimeDB.EnvAttribute", + (_, _) => true, + (ctx, _) => (INamedTypeSymbol)ctx.TargetSymbol + ) + .Collect(); + context.RegisterSourceOutput(declarations, Generate); + } + + private static void Generate( + SourceProductionContext context, + ImmutableArray types + ) + { + void Report(ISymbol symbol, string message) => + context.ReportDiagnostic( + Diagnostic.Create(InvalidDeclaration, symbol.Locations.FirstOrDefault(), message) + ); + if (types.Length > 1) + { + foreach (var type in types) + Report(type, "A module may have only one [SpacetimeDB.Env] declaration struct."); + } + var fields = types + .SelectMany(type => type.GetMembers().OfType()) + .Where(field => !field.IsImplicitlyDeclared) + .ToArray(); + if (fields.Length > 256 && types.Length != 0) + Report(types[0], "An environment schema may declare at most 256 variables."); + + var keys = new HashSet(StringComparer.Ordinal); + var properties = new List(); + var registrations = new List(); + foreach (var field in fields) + { + var name = field.Name; + if (field.IsStatic || field.Type.SpecialType != SpecialType.System_String) + { + Report( + field, + "Environment fields must be instance string or nullable string declarations." + ); + continue; + } + if ( + !Regex.IsMatch(name, "^[A-Za-z_][A-Za-z0-9_]*$") + || Encoding.UTF8.GetByteCount(name) > 256 + || !keys.Add(name) + ) + { + Report( + field, + "Environment names must be unique POSIX identifiers of at most 256 UTF-8 bytes." + ); + continue; + } + var optional = field.NullableAnnotation == NullableAnnotation.Annotated; + var attr = field + .GetAttributes() + .FirstOrDefault(a => + a.AttributeClass?.ToDisplayString() == "SpacetimeDB.EnvValuesAttribute" + ); + var constraint = + "new global::SpacetimeDB.Internal.EnvironmentConstraint.AnyString(default)"; + if (attr is not null) + { + var values = attr.ConstructorArguments.FirstOrDefault(); + if ( + values.Kind != TypedConstantKind.Array + || values.IsNull + || values.Values.Length == 0 + || values.Values.Any(value => + value.Value is not string text || Encoding.UTF8.GetByteCount(text) > 8192 + ) + ) + { + Report( + field, + "EnvValues requires a nonempty list of string literals of at most 8192 UTF-8 bytes each." + ); + continue; + } + var strings = values.Values.Select(value => (string)value.Value!).ToArray(); + if (strings.Distinct(StringComparer.Ordinal).Count() != strings.Length) + { + Report(field, "EnvValues must not repeat an allowed literal."); + continue; + } + constraint = + strings.Length == 1 + ? $"new global::SpacetimeDB.Internal.EnvironmentConstraint.Literal({Literal(strings[0])})" + : $"new global::SpacetimeDB.Internal.EnvironmentConstraint.OneOf(new global::System.Collections.Generic.List {{ {string.Join(", ", strings.Select(Literal))} }})"; + } + registrations.Add( + $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" + ); + // Preserve the checked generic method, including a key literally + // named Get, and inherited object members. Keywords are escaped + // without renaming stored keys. + if ( + name + is "Get" + or "ModuleEnvironment" + or "Equals" + or "GetHashCode" + or "ToString" + or "Finalize" + or "GetType" + or "MemberwiseClone" + ) + continue; + var read = $"Get({Literal(name)})"; + if (!optional) + read += + " ?? throw new global::System.InvalidOperationException(\"Required environment value is absent\")"; + properties.Add($"public string{(optional ? "?" : "")} @{name} => {read};"); + } + context.AddSource( + "Environment.g.cs", + $$""" + // + #nullable enable + #pragma warning disable CS0436 + namespace SpacetimeDB { + public readonly struct ModuleEnvironment { + public string? Get(string key) => default(global::SpacetimeDB.DatabaseEnvironment).Get(key); + {{string.Join("\n", properties)}} + } + internal static class EnvironmentRegistration { + [global::System.Runtime.CompilerServices.ModuleInitializer] + internal static void Register() { + {{string.Join("\n", registrations)}} + } + } + } + """ + ); + } + + private static string Literal(string value) => SymbolDisplay.FormatLiteral(value, quote: true); +} diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index aad1bf54c02..29fc16d2cfc 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2557,6 +2557,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2628,6 +2629,7 @@ public Uuid NewUuidV7() } public sealed partial class ProcedureContext : global::SpacetimeDB.ProcedureContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal ProcedureContext(Identity identity, ConnectionId? connectionId, Random random, Timestamp time) @@ -2698,6 +2700,7 @@ public Uuid NewUuidV7() } public sealed partial class HandlerContext : global::SpacetimeDB.HandlerContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; private readonly Local _db = new(); internal HandlerContext(Random random, Timestamp time) @@ -2735,6 +2738,7 @@ public Uuid NewUuidV7() } public sealed class ProcedureTxContext : global::SpacetimeDB.ProcedureTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2742,6 +2746,7 @@ internal ProcedureTxContext(Internal.TxContext inner) : base(inner) {} [Experimental("STDB_UNSTABLE")] public sealed class HandlerTxContext : global::SpacetimeDB.HandlerTxContextBase { + public new global::SpacetimeDB.ModuleEnvironment Env => default; internal HandlerTxContext(Internal.TxContext inner) : base(inner) {} public new Local Db => (Local)base.Db; @@ -2755,6 +2760,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2766,6 +2772,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.ModuleEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..a97596d4ec9 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -20,3 +20,30 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. + +### Declared environment + +A module may declare one `[SpacetimeDB.Env]` struct. `string` is required and +`string?` is optional. An optional `EnvValues` attribute restricts the allowed +strings; one value is a literal constraint. Values are supplied on publish, never +in module source: + +```csharp +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_URL; + [SpacetimeDB.EnvValues("prod", "dev")] + public string MODE; + public string? LOG_LEVEL; +} +``` + +Context access is read-only: `ctx.Env.MODE` returns `string`, while +`ctx.Env.LOG_LEVEL` returns `string?`. `ctx.Env.Get("MODE")` uses the same checked +host read. A key named `Get` keeps generic access rather than replacing the method; +C# keywords are escaped, for example `ctx.Env.@class`. Empty or absent declarations +allow no environment keys. Undeclared reads and reads from host-dispatched +submodules fail at runtime. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 3865f925a2a..afcfcc0688e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -1,5 +1,16 @@ namespace SpacetimeDB { + /// Declares the complete environment schema for this module. + [AttributeUsage(AttributeTargets.Struct)] + public sealed class EnvAttribute : Attribute { } + + /// Restricts one declared string to these exact permitted values. + [AttributeUsage(AttributeTargets.Field)] + public sealed class EnvValuesAttribute(params string[] values) : Attribute + { + public string[] Values { get; } = values; + } + namespace Internal { [Flags] diff --git a/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs new file mode 100644 index 00000000000..59ca2c26ec2 --- /dev/null +++ b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs @@ -0,0 +1,26 @@ +namespace SpacetimeDB; + +/// +/// Read-only database environment. Values are plaintext and accessible to database +/// collaborators. Procedure reads outside a transaction use a short snapshot. +/// +public readonly struct DatabaseEnvironment +{ + internal static readonly DatabaseEnvironment Instance = new(); + + /// Return null for a missing key, or an empty string for a present empty value. + public unsafe string? Get(string key) + { + ArgumentNullException.ThrowIfNull(key); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + fixed (byte* ptr = bytes) + { + Internal.FFI.env_get(ptr, checked((uint)bytes.Length), out var source); + if (source == Internal.BytesSource.INVALID) + { + return null; + } + return System.Text.Encoding.UTF8.GetString(Internal.Module.Consume(source)); + } + } +} diff --git a/crates/bindings-csharp/Runtime/HandlerContext.cs b/crates/bindings-csharp/Runtime/HandlerContext.cs index 76b13230426..9fc02d8b858 100644 --- a/crates/bindings-csharp/Runtime/HandlerContext.cs +++ b/crates/bindings-csharp/Runtime/HandlerContext.cs @@ -7,6 +7,7 @@ namespace SpacetimeDB; public abstract class HandlerContextBase { public Random Rng => txState.Rng; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => txState.Timestamp; // NOTE: The host rejects procedure HTTP requests while a mut transaction is open @@ -90,6 +91,7 @@ public abstract class HandlerTxContextBase(Internal.TxContext inner) : IRefresha void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => Inner.Timestamp; public Random Rng => Inner.Rng; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs new file mode 100644 index 00000000000..87031068dec --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentConstraint.g.cs @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + public partial record EnvironmentConstraint : SpacetimeDB.TaggedEnum<( + SpacetimeDB.Unit AnyString, + string Literal, + System.Collections.Generic.List OneOf + )>; +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs new file mode 100644 index 00000000000..504d1a846a9 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs @@ -0,0 +1,40 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class EnvironmentDeclaration + { + [DataMember(Name = "name")] + public string Name; + [DataMember(Name = "constraint")] + public EnvironmentConstraint Constraint; + [DataMember(Name = "optional")] + public bool Optional; + + public EnvironmentDeclaration( + string Name, + EnvironmentConstraint Constraint, + bool Optional + ) + { + this.Name = Name; + this.Constraint = Constraint; + this.Optional = Optional; + } + + public EnvironmentDeclaration() + { + this.Name = ""; + this.Constraint = null!; + } + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 61212c98e89..52f750e3903 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -23,6 +23,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, System.Collections.Generic.List ViewPrimaryKeys, - System.Collections.Generic.List Submodules + System.Collections.Generic.List Submodules, + System.Collections.Generic.List Environment )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index afc56abbc8f..8957759d8eb 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,6 +109,22 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_6 = +#if EXPERIMENTAL_WASM_AOT || NET10_0_OR_GREATER + "spacetime_10.6" +#else + "bindings" +#endif + ; + + [WasmImportLinkage] + [LibraryImport(StdbNamespace10_6)] + public static unsafe partial CheckedStatus env_get( + byte* key, + uint keyLen, + out BytesSource source + ); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 177fddd785a..438c3439146 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -20,6 +20,7 @@ partial class RawModuleDefV10 private readonly List httpRouteDefs = []; private readonly List viewDefs = []; private readonly List viewPrimaryKeyDefs = []; + private readonly List environment = []; private readonly List rowLevelSecurityDefs = []; private readonly Dictionary> defaultValuesByTable = new(StringComparer.Ordinal); @@ -86,6 +87,9 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); + internal void RegisterEnvironment(EnvironmentDeclaration declaration) => + environment.Add(declaration); + internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => viewPrimaryKeyDefs.Add(new RawViewPrimaryKeyDefV10(viewSourceName, [.. columns])); @@ -162,6 +166,7 @@ internal RawModuleDefV10 BuildModuleDefinition() var sections = new List { new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Environment(environment), }; if (typeDefs.Count > 0) @@ -427,6 +432,9 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } + public static void RegisterEnvironment(EnvironmentDeclaration declaration) => + moduleDef.RegisterEnvironment(declaration); + public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => moduleDef.RegisterViewPrimaryKey(viewSourceName, columns); diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index a86711f2814..63ec0238ca0 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -3,6 +3,7 @@ namespace SpacetimeDB; #pragma warning disable STDB_UNSTABLE public abstract class ProcedureContextBase : Internal.IInternalProcedureContext { + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public static Identity Identity => Internal.IProcedureContext.GetIdentity(); public Identity Sender { get; } public ConnectionId? ConnectionId { get; } @@ -100,6 +101,7 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) : IRefres void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 6118ba6be11..b877876b37f 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,6 +135,10 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.6" +IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index e3b4bd0e942..45d4b570d58 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -65,6 +65,8 @@ + + diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs new file mode 100644 index 00000000000..4640bcb40dc --- /dev/null +++ b/crates/bindings-macro/src/environment.rs @@ -0,0 +1,202 @@ +pub(crate) mod value; + +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::ext::IdentExt as _; +use syn::punctuated::Punctuated; +use syn::{Fields, ItemStruct, LitStr, Token}; + +pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result { + if !args.is_empty() { + return Err(syn::Error::new_spanned(args, "env does not accept arguments")); + } + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment declarations cannot be generic", + )); + } + if matches!(item.fields, Fields::Unnamed(_)) { + return Err(syn::Error::new_spanned( + &item.fields, + "environment declarations require named fields", + )); + } + let mut declarations = Vec::new(); + let mut signatures = Vec::new(); + let mut methods = Vec::new(); + for field in &mut item.fields { + let ident = field.ident.as_ref().expect("named fields checked"); + let name = ident.unraw().to_string(); + if name.len() > 256 || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(syn::Error::new_spanned( + ident, + "environment keys must be POSIX names of at most 256 bytes", + )); + } + let ty = &field.ty; + let mut values: Option> = None; + for attr in field.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("values") || values.is_some() { + return Err(meta.error("expected one values(\"literal\", ...) constraint")); + } + let content; + syn::parenthesized!(content in meta.input); + let parsed = Punctuated::::parse_terminated(&content)?; + if parsed.is_empty() { + return Err(meta.error("environment string unions must not be empty")); + } + for value in &parsed { + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + } + values = Some(parsed.into_iter().collect()); + Ok(()) + })?; + } + field.attrs.retain(|attr| !attr.path().is_ident("env")); + let constraint = match values.as_deref() { + None => quote!(<#ty as ::spacetimedb::rt::EnvironmentValue>::constraint()), + Some([value]) => { + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())) + } + Some(values) => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf( + ::std::vec![#(#values.into()),*] + ) + ), + }; + let constraint = if values.is_some() { + quote!(<#ty as ::spacetimedb::rt::StringEnvironmentValue>::with_constraint(#constraint)) + } else { + constraint + }; + declarations.push( + quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentDeclaration { + name: #name.into(), + constraint: #constraint, + optional: <#ty as ::spacetimedb::rt::EnvironmentValue>::OPTIONAL, + }), + ); + if name == "get" { + continue; + } + signatures.push(quote! { + #[doc = concat!("Read the declared environment key `", #name, "` through the checked host ABI.")] + fn #ident(&self) -> #ty; + }); + methods.push(quote! { + fn #ident(&self) -> #ty { + <#ty as ::spacetimedb::rt::EnvironmentValue>::get(self, #name) + } + }); + } + let vis = &item.vis; + let access = format_ident!("{}Access", item.ident.unraw()); + let symbol = format!("__preinit__20_register_environment_{}", item.ident.unraw()); + Ok(quote! { + #[allow(non_snake_case)] + #item + + /// Named read-only accessors for this module's environment declaration. + #[allow(non_snake_case)] + #vis trait #access { + #(#signatures)* + } + #[allow(non_snake_case)] + impl #access for ::spacetimedb::Environment { + #(#methods)* + } + const _: () = { + #[unsafe(export_name = #symbol)] + extern "C" fn __register_environment() { + ::spacetimedb::rt::register_environment(|| ::std::vec![#(#declarations),*]); + } + }; + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_empty_union_invalid_names_and_unsupported_struct_shapes() { + for item in [ + quote!( + struct Env { + #[env(values())] + VALUE: String, + } + ), + quote!( + struct Env { + #[env(values("a"), values("b"))] + VALUE: String, + } + ), + quote!( + struct Env { + ι›ͺ: String, + } + ), + quote!( + struct Env(String); + ), + quote!( + struct Env { + VALUE: T, + } + ), + ] { + assert!(expand(TokenStream::new(), syn::parse2(item).unwrap()).is_err()); + } + } + + #[test] + fn accepts_strings_optionals_constraints_and_generic_accessor_collision() { + let output = expand( + TokenStream::new(), + syn::parse_quote! { + pub struct Env { + VALUE: String, + #[env(values("false", "true"))] ENABLED: std::string::String, + #[env(values(""))] OPTIONAL: Option, + get: Option, + r#type: String, + } + }, + ) + .unwrap(); + let parsed: syn::File = syn::parse2(output).unwrap(); + let trait_item = parsed + .items + .iter() + .find_map(|item| match item { + syn::Item::Trait(item) => Some(item), + _ => None, + }) + .unwrap(); + let names: Vec<_> = trait_item + .items + .iter() + .filter_map(|item| match item { + syn::TraitItem::Fn(method) => Some(method.sig.ident.unraw().to_string()), + _ => None, + }) + .collect(); + assert_eq!(names, ["VALUE", "ENABLED", "OPTIONAL", "type"]); + assert!(expand( + TokenStream::new(), + syn::parse_quote!( + pub struct Empty {} + ) + ) + .is_ok()); + } +} diff --git a/crates/bindings-macro/src/environment/value.rs b/crates/bindings-macro/src/environment/value.rs new file mode 100644 index 00000000000..2608cc10f69 --- /dev/null +++ b/crates/bindings-macro/src/environment/value.rs @@ -0,0 +1,193 @@ +use proc_macro2::TokenStream; +use quote::quote; +use std::collections::BTreeSet; +use syn::ext::IdentExt as _; +use syn::{Data, DeriveInput, Fields, LitStr}; + +pub(crate) fn derive(item: DeriveInput) -> syn::Result { + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { + return Err(syn::Error::new_spanned( + &item.generics, + "environment value enums cannot be generic", + )); + } + if let Some(attr) = item.attrs.iter().find(|attr| attr.path().is_ident("env")) { + return Err(syn::Error::new_spanned( + attr, + "place `#[env(value = \"...\")]` on enum variants", + )); + } + let Data::Enum(data) = &item.data else { + return Err(syn::Error::new_spanned( + &item.ident, + "EnvironmentValue requires an enum with unit variants", + )); + }; + if data.variants.is_empty() || data.variants.len() > 256 { + return Err(syn::Error::new_spanned( + &item.ident, + "environment value enums require 1 to 256 variants", + )); + } + let mut variants = Vec::new(); + let mut values = Vec::new(); + let mut unique = BTreeSet::new(); + for variant in &data.variants { + if !matches!(variant.fields, Fields::Unit) { + return Err(syn::Error::new_spanned( + &variant.fields, + "environment value variants cannot have payloads", + )); + } + let mut value: Option = None; + for attr in variant.attrs.iter().filter(|attr| attr.path().is_ident("env")) { + attr.parse_nested_meta(|meta| { + if !meta.path.is_ident("value") || value.is_some() { + return Err(meta.error("expected one value = \"literal\" mapping")); + } + value = Some(meta.value()?.parse()?); + Ok(()) + })?; + if value.is_none() { + return Err(syn::Error::new_spanned(attr, "expected value = \"literal\" mapping")); + } + } + let value = value.unwrap_or_else(|| LitStr::new(&variant.ident.unraw().to_string(), variant.ident.span())); + if value.value().len() > 8192 { + return Err(syn::Error::new_spanned( + value, + "environment literal exceeds 8192 UTF-8 bytes", + )); + } + if !unique.insert(value.value()) { + return Err(syn::Error::new_spanned( + value, + "environment variants must map to distinct strings", + )); + } + variants.push(&variant.ident); + values.push(value); + } + let constraint = match values.as_slice() { + [value] => quote!(::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::Literal(#value.into())), + values => quote!( + ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint::OneOf(::std::vec![#(#values.into()),*]) + ), + }; + let ident = &item.ident; + Ok(quote! { + impl ::spacetimedb::rt::EnvironmentValue for #ident { + const OPTIONAL: bool = false; + + fn constraint() -> ::spacetimedb::spacetimedb_lib::environment::EnvironmentConstraint { + #constraint + } + + fn from_environment(value: ::std::option::Option<::std::string::String>, key: &str) -> Self { + match value.as_deref() { + #(::std::option::Option::Some(#values) => Self::#variants,)* + ::std::option::Option::Some(_) => ::core::panic!("environment value does not match its declared enum: {}", key), + ::std::option::Option::None => ::core::panic!("required environment key is missing: {}", key), + } + } + } + impl ::spacetimedb::rt::RequiredEnvironmentValue for #ident {} + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_invalid_enum_shapes_mappings_and_limits() { + for input in [ + quote!( + struct Value; + ), + quote!( + enum Value {} + ), + quote!( + enum Value { + Item(T), + } + ), + quote!( + enum Value { + Item(String), + } + ), + quote!( + enum Value { + Item { field: String }, + } + ), + quote!( + enum Value { + #[env()] + Item, + } + ), + quote!( + enum Value { + #[env(values("x"))] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x", value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "x")] + #[env(value = "y")] + Item, + } + ), + quote!( + enum Value { + #[env(value = "Same")] + First, + Same, + } + ), + quote!( + #[env(value = "x")] + enum Value { + Item, + } + ), + ] { + assert!(derive(syn::parse2(input).unwrap()).is_err()); + } + let oversized = LitStr::new(&"Γ©".repeat(4097), proc_macro2::Span::call_site()); + assert!(derive(syn::parse_quote!( + enum Value { + #[env(value = #oversized)] + Item, + } + )) + .is_err()); + let variants = (0..257).map(|n| quote::format_ident!("V{n}")); + assert!(derive(syn::parse_quote!(enum Value { #(#variants),* })).is_err()); + } + + #[test] + fn accepts_exact_strings_and_ordinary_enum_attributes() { + let output = derive(syn::parse_quote! { + #[derive(Debug, PartialEq)] + enum Value { + #[env(value = "in progress")] InProgress, + #[env(value = "")] Empty, + #[env(value = "hΓ©llo\0δΈ–η•Œ")] Unicode, + r#type, + } + }) + .unwrap(); + syn::parse2::(output).unwrap(); + } +} diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index a3efdd3d2f4..37069ccbf2c 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -8,6 +8,18 @@ // // (private documentation for the macro authors is totally fine here and you SHOULD write that!) +mod environment; + +#[proc_macro_attribute] +pub fn env(args: StdTokenStream, item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::expand(args.into(), syn::parse(item)?)) +} + +#[proc_macro_derive(EnvironmentValue, attributes(env))] +pub fn derive_environment_value(item: StdTokenStream) -> StdTokenStream { + ok_or_compile_error(|| environment::value::derive(syn::parse(item)?)) +} + mod http; mod procedure; diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index 0295b4616e5..c683d4c8a59 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -883,6 +883,17 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } + #[link(wasm_import_module = "spacetime_10.6")] + unsafe extern "C" { + /// Read a UTF-8 environment value. Writes INVALID for a missing key; + /// present empty strings have a valid BytesSource. Returns ordinary errno. + /// Invalid keys return HOST_CALL_FAILURE. NO_SPACE means 256 byte + /// sources remain unconsumed; consume a source before retrying. + /// Calls outside a reducer/view + /// transaction or procedure return NOT_IN_TRANSACTION. + pub fn env_get(key: *const u8, key_len: usize, out: *mut BytesSource) -> u16; + } + /// What strategy does the database index use? /// /// See also: @@ -1493,6 +1504,14 @@ pub fn get_jwt(connection_id: [u8; 16]) -> Option { } } +/// Read a database environment value without exposing the system table. +#[inline] +pub fn env_get(key: &str) -> Option { + let source = unsafe { call(|out| raw::env_get(key.as_ptr(), key.len(), out)) } + .unwrap_or_else(|errno: Errno| panic!("Error reading environment: {errno}")); + (source != raw::BytesSource::INVALID).then_some(source) +} + pub struct RowIter { raw: raw::RowIter, } diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 48e7cfd1535..a006b8489fd 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -209,3 +209,29 @@ To run the tests, do: ```sh pnpm build && pnpm test ``` + +### Declared environment + +Pass the complete environment schema to `schema`. Values are supplied at publish +time, not embedded in the module: + +```ts +const db = schema(tables, { + env: { + API_URL: t.string(), + MODE: t.enum('Mode', ['prod', 'dev']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, +}); +``` + +`ctx.env.MODE` has type `'prod' | 'dev'`; `ctx.env.LOG_LEVEL` also permits +`undefined`. Simple enum cases mean allowed strings only in this declaration; +ordinary enum values elsewhere retain their tagged representation. Payload enums +are rejected. The checked `ctx.env.get('LOG_LEVEL')` returns `null` for an omitted +optional value. A declared key named `get` remains accessible through the generic +method. Omitted or empty schemas allow no keys. Undeclared reads and reads entered +by the host in a submodule fail at runtime; ordinary helpers retain their caller's +scope. Values are private, durable database configuration for secrets and other settings. +Database owners and authorized collaborators can read them; module code can expose +them through its own outputs. diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index 8e6f94787de..db598440673 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -25,14 +25,14 @@ "scripts": { "build:js": "tsup", "build:types": "tsc -p tsconfig.build.json", - "build": "pnpm -s build:js && pnpm -s build:types", + "build": "pnpm run build:js && pnpm run build:types", "format": "prettier . --write --ignore-path ../../.prettierignore", "lint": "eslint . && prettier . --check --ignore-path ../../.prettierignore", "test": "vitest run", "test:typecheck": "vitest typecheck --run", "coverage": "vitest run --coverage", "brotli-size": "brotli-size dist/index.js", - "size": "pnpm -s build && size-limit", + "size": "pnpm run build && size-limit", "generate:moduledef": "cargo run -p spacetimedb-codegen --example regen-typescript-moduledef && prettier --write src/lib/autogen", "generate:client-api": "cargo run -p generate-client-api && prettier --write src/sdk/client_api", "generate:test-app": "pnpm --filter @clockworklabs/test-app generate", diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 3cee51f03d5..642c0431285 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -393,9 +393,38 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get Submodules() { return __t.array(RawSubmoduleV10); }, + get Environment() { + return __t.array(EnvironmentDeclaration); + }, }); export type RawModuleDefV10Section = __Infer; +export const EnvironmentConstraint = __t.enum('EnvironmentConstraint', { + get AnyString() { + return __t.unit(); + }, + get Literal() { + return __t.string(); + }, + get OneOf() { + return __t.array(__t.string()); + }, +}); +export type EnvironmentConstraint = __Infer; + +export const EnvironmentDeclaration = __t.object('EnvironmentDeclaration', { + get name() { + return __t.string(); + }, + get constraint() { + return EnvironmentConstraint; + }, + get optional() { + return __t.bool(); + }, +}); +export type EnvironmentDeclaration = __Infer; + export const RawModuleDefV8 = __t.object('RawModuleDefV8', { get typespace() { return Typespace; diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts new file mode 100644 index 00000000000..d065d5bb0aa --- /dev/null +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -0,0 +1,56 @@ +import type { + OptionBuilder, + StringBuilder, + TypeBuilder, + t, +} from './type_builders'; + +/** Only strings and simple, payload-free enums constrain environment strings. */ +export type EnvironmentString = + | StringBuilder + | (TypeBuilder & { + readonly variants: Record>; + }); +export type EnvironmentSchema = Record< + string, + EnvironmentString | OptionBuilder +>; + +export type EnvironmentValue = + T extends OptionBuilder + ? EnvironmentValue | undefined + : T extends { readonly variants: infer Variants } + ? keyof Variants & string + : string; + +/** Values are read from the host on each access. Undeclared names are errors. */ +export type Environment< + Declarations extends EnvironmentSchema | undefined = undefined, +> = Declarations extends EnvironmentSchema + ? { + readonly [Key in keyof Declarations as Key extends 'get' + ? never + : Key]: EnvironmentValue; + } & { + get( + key: Key & + (string extends Key + ? unknown + : Key extends keyof Declarations + ? unknown + : never) + ): Key extends keyof Declarations + ? + | Exclude, undefined> + | (undefined extends EnvironmentValue + ? null + : never) + : string | null; + } + : { get(key: string): string | null }; + +export type EnvironmentFor = Schema extends { + env: infer Declarations extends EnvironmentSchema; +} + ? Environment + : Environment; diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 27c57f0721b..f2d470152ec 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,3 +1,4 @@ +import type { EnvironmentFor } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -115,6 +116,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + env: EnvironmentFor; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index eb3821f6606..67b263d8397 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -205,6 +205,7 @@ export class ModuleContext { entries: [], }, submodules: [], + environment: [], }; get moduleDef(): ModuleDef { @@ -275,6 +276,7 @@ export class ModuleContext { value: module.submodules, } ); + push({ tag: 'Environment', value: module.environment }); return { sections }; } diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts new file mode 100644 index 00000000000..05458efae8a --- /dev/null +++ b/crates/bindings-typescript/src/server/environment.ts @@ -0,0 +1,91 @@ +import { env_get } from 'spacetime:sys@2.2'; +import type { Environment, EnvironmentSchema } from '../lib/environment'; +import type { + EnvironmentDeclaration, + EnvironmentConstraint, + AlgebraicType, +} from '../lib/autogen/types'; +import { OptionBuilder, StringBuilder } from '../lib/type_builders'; + +// These UTF-8 byte/count limits match spacetimedb_lib::environment. +const MAX_ENV_KEY_BYTES = 256; +const MAX_ENV_VALUE_BYTES = 8 * 1024; +const MAX_ENV_VARS = 256; + +/** Values are not cached: transaction and procedure reads retain host semantics. */ +export const environment: Environment = new Proxy( + Object.freeze( + Object.assign(Object.create(null), { get: (key: string) => env_get(key) }) + ), + { + get(target, key) { + if (key === 'get') return target.get; + if (typeof key !== 'string') return undefined; + // The host rejects undeclared keys and missing required values. Optional + // named access uses undefined; the generic ABI accessor retains null. + return env_get(key) ?? undefined; + }, + } +); +export type { + Environment, + EnvironmentFor, + EnvironmentSchema, +} from '../lib/environment'; + +/** Produce metadata only. No environment value is embedded in the artifact. */ +export function environmentDeclarations( + schema: EnvironmentSchema +): EnvironmentDeclaration[] { + const entries = Object.entries(schema); + if (entries.length > MAX_ENV_VARS) + throw new TypeError('Too many environment declarations'); + const bytes = new TextEncoder(); + return entries.map(([name, definition]) => { + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || + bytes.encode(name).length > MAX_ENV_KEY_BYTES + ) { + throw new TypeError('Invalid environment declaration name'); + } + const optional = definition instanceof OptionBuilder; + const inner = optional ? definition.value : definition; + let constraint: EnvironmentConstraint; + if (inner instanceof StringBuilder) { + constraint = { tag: 'AnyString' }; + } else { + const type: AlgebraicType = inner?.algebraicType; + if (type?.tag !== 'Sum' || !('variants' in inner)) { + throw new TypeError( + `Environment '${name}' must be a string or simple enum` + ); + } + const values = type.value.variants.map(variant => { + if ( + variant.algebraicType.tag !== 'Product' || + variant.algebraicType.value.elements.length !== 0 + ) { + throw new TypeError( + `Environment '${name}' cannot use an enum payload` + ); + } + if (typeof variant.name !== 'string') + throw new TypeError( + `Environment '${name}' enum cases must have names` + ); + if (bytes.encode(variant.name).length > MAX_ENV_VALUE_BYTES) + throw new TypeError(`Environment '${name}' literal is too long`); + return variant.name; + }); + if (values.length === 0 || new Set(values).size !== values.length) + throw new TypeError( + `Environment '${name}' needs a nonempty literal union` + ); + constraint = + values.length === 1 + ? { tag: 'Literal', value: values[0]! } + : { tag: 'OneOf', value: values }; + } + return { name, constraint, optional }; + }); +} diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index 68d42a267f8..b8823577f4e 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,3 +1,4 @@ +import type { EnvironmentFor } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -219,6 +220,7 @@ export type HandlerAliasViews = : {}; export interface HandlerContext { + readonly env: EnvironmentFor; readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index ae084f672d4..3ac3e8f0fbb 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -35,4 +35,6 @@ export { export type { HandlerContext, HttpHandlerExport } from './http'; export { ScheduleAt } from '../lib/schedule_at'; +export type { Environment } from './environment'; + import './polyfills'; // Ensure polyfills are loaded diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 2dec68467c0..f5b8e2aa1e2 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,3 +1,4 @@ +import { environment, type EnvironmentFor } from './environment'; import { AlgebraicType, ProductType, @@ -108,6 +109,7 @@ export type ProcedureAliasViews = : {}; export interface ProcedureCtx { + readonly env: EnvironmentFor; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -226,6 +228,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + readonly env = environment as EnvironmentFor; #dispatches: SubmoduleDispatchInfo[]; #parentPrefix: string; #asViews: object | undefined; diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index cf579e87631..70c6eb81401 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,3 +1,4 @@ +import { environment, type EnvironmentFor } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; @@ -246,6 +247,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + readonly env = environment as EnvironmentFor; as: AliasViews; constructor( @@ -627,6 +629,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = viewFns![localId!]; const ctx: ViewCtx = freeze({ + env: environment, sender: new Identity(sender), db: dbView!, from: from!, @@ -677,6 +680,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = anonViewFns![localId!]; const ctx: AnonymousViewCtx = freeze({ + env: environment, db: dbView!, from: from!, }); @@ -773,6 +777,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { + readonly env = environment as EnvironmentFor; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index c399a66d31e..95688b7f3cd 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -1,3 +1,4 @@ +import { environmentDeclarations, type EnvironmentSchema } from './environment'; import { moduleHooks, type ModuleDefaultExport } from 'spacetime:sys@2.0'; import { CaseConversionPolicy, @@ -322,6 +323,9 @@ export class Schema implements ModuleDefaultExport { const rawDef = this.buildRawModuleDefV10(exports, { ignoreNonModuleExports: true, }); + if (this.#ctx.moduleDef.environment.length !== 0) { + throw new TypeError('Submodules cannot declare environment variables'); + } this.#ctx.resolveHttpRoutes(); return { rawDef, @@ -747,7 +751,11 @@ export type InferSchema> = /** * Module-level settings that can be passed to `schema()`. */ -export interface ModuleSettings { +export interface ModuleSettings< + E extends EnvironmentSchema = EnvironmentSchema, +> { + /** Declared strings installed only through publishing; omitted means empty. */ + env?: E; /** * The case conversion policy for this module. * Defaults to `SnakeCase` if not specified. @@ -825,11 +833,17 @@ function registerModuleExports( } } -export function schema>( +export function schema< + const H extends Record, + const E extends EnvironmentSchema = {}, +>( entries: H, - moduleSettings?: ModuleSettings -): Schema> { - const ctx = new SchemaInner>(ctx => { + moduleSettings?: ModuleSettings +): Schema & { env: E }> { + const ctx = new SchemaInner & { env: E }>(ctx => { + ctx.moduleDef.environment = environmentDeclarations( + moduleSettings?.env ?? {} + ); // Apply module settings. if (moduleSettings?.CASE_CONVERSION_POLICY != null) { ctx.setCaseConversionPolicy(moduleSettings.CASE_CONVERSION_POLICY); @@ -884,7 +898,10 @@ export function schema>( }); } } - return { tables: tableSchemas } as SchemaDefForEntries; + return { + tables: tableSchemas, + env: moduleSettings?.env ?? {}, + } as SchemaDefForEntries & { env: E }; }); return new Schema(ctx); diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index f0315867cb3..77a9dacaf4e 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -123,3 +123,8 @@ declare module 'spacetime:sys@2.0' { declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } + +declare module 'spacetime:sys@2.2' { + /** Null means missing; an empty string is a present value. */ + export function env_get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index c7cb9ca0b4d..f089de2f0c1 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,3 +1,4 @@ +import type { EnvironmentFor } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -81,11 +82,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; + env: EnvironmentFor; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; + env: EnvironmentFor; from: QueryBuilder; }>; diff --git a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts index 47cbf9d7039..257321cfe8e 100644 --- a/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts +++ b/crates/bindings-typescript/tests/__mocks__/spacetime-sys.ts @@ -91,3 +91,7 @@ export const procedure_http_request = ( export const procedure_start_mut_tx = (): bigint => 0n; export const procedure_commit_mut_tx = (): void => {}; export const procedure_abort_mut_tx = (): void => {}; + +export const env_get = (_key: string): string | null => { + throw new Error('mock environment read is not configured'); +}; diff --git a/crates/bindings-typescript/tests/environment.test.ts b/crates/bindings-typescript/tests/environment.test.ts new file mode 100644 index 00000000000..34fc8ae65c9 --- /dev/null +++ b/crates/bindings-typescript/tests/environment.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, expectTypeOf, it, vi } from 'vitest'; +import { schema } from '../src/server/schema'; +import { t } from '../src/lib/type_builders'; +import { + environment, + environmentDeclarations, +} from '../src/server/environment'; +import type { EnvironmentSchema } from '../src/lib/environment'; +import { env_get } from 'spacetime:sys@2.2'; + +vi.mock('spacetime:sys@2.2', async importOriginal => ({ + ...(await importOriginal()), + env_get: vi.fn(), +})); + +const declarations = { + FOOBAR: t.string(), + ENABLE_EMAIL: t.enum('EnableEmail', ['true', 'false']), + LOG_LEVEL: t.enum('LogLevel', ['debug', 'info', 'error']).optional(), + DEPLOYMENT_KIND: t.enum('DeploymentKind', ['production']), + get: t.string().optional(), +}; + +describe('declared database environment', () => { + it('emits canonical constraint metadata and an explicit empty section', () => { + expect(environmentDeclarations(declarations)).toEqual([ + { name: 'FOOBAR', constraint: { tag: 'AnyString' }, optional: false }, + { + name: 'ENABLE_EMAIL', + constraint: { tag: 'OneOf', value: ['true', 'false'] }, + optional: false, + }, + { + name: 'LOG_LEVEL', + constraint: { tag: 'OneOf', value: ['debug', 'info', 'error'] }, + optional: true, + }, + { + name: 'DEPLOYMENT_KIND', + constraint: { tag: 'Literal', value: 'production' }, + optional: false, + }, + { name: 'get', constraint: { tag: 'AnyString' }, optional: true }, + ]); + const defined = schema({}, { env: declarations }); + const section = defined + .buildRawModuleDefV10({}) + .sections.find(section => section.tag === 'Environment'); + expect(section).toEqual({ + tag: 'Environment', + value: environmentDeclarations(declarations), + }); + expect(schema({}).buildRawModuleDefV10({}).sections).toContainEqual({ + tag: 'Environment', + value: [], + }); + // Enum values outside env retain their existing tagged-sum interpretation. + expect(Reflect.get(declarations.ENABLE_EMAIL, 'true')).toEqual({ + tag: 'true', + }); + }); + + it('rejects unsupported constraints and submodule declarations before upload', () => { + const invalid = (value: unknown) => () => + environmentDeclarations(value as EnvironmentSchema); + expect(invalid({ BAD: t.u32() })).toThrow('string or simple enum'); + expect(invalid({ BAD: t.enum('Payload', { value: t.string() }) })).toThrow( + 'enum payload' + ); + expect(invalid({ BAD: t.enum('Empty', []) })).toThrow( + 'nonempty literal union' + ); + expect(invalid({ BAD: t.string().optional().optional() })).toThrow(); + expect(invalid({ 'bad-name': t.string() })).toThrow('name'); + expect(invalid({ BAD: t.enum('Long', ['x'.repeat(8193)]) })).toThrow( + 'too long' + ); + expect( + invalid( + Object.fromEntries( + Array.from({ length: 257 }, (_, i) => [`K${i}`, t.string()]) + ) + ) + ).toThrow('Too many'); + expect(() => + schema({ child: { default: schema({}, { env: declarations }) } }) + ).toThrow('Submodules'); + expect(() => + schema({ child: { default: schema({}, { env: {} }) } }) + ).not.toThrow(); + }); + + it('preserves generic get, optional absence, host errors and uncached named reads', () => { + const get = vi.mocked(env_get); + get.mockReset(); + get + .mockReturnValueOnce('first') + .mockReturnValueOnce('') + .mockReturnValueOnce(null) + .mockReturnValueOnce('declared get'); + const named = environment as typeof environment & { + readonly FOOBAR: string; + readonly LOG_LEVEL: string | undefined; + }; + expect(named.FOOBAR).toBe('first'); + expect(named.FOOBAR).toBe(''); + expect(named.LOG_LEVEL).toBeUndefined(); + expect(named.get('get')).toBe('declared get'); + get.mockReturnValueOnce(null); + expect(named.get('LOG_LEVEL')).toBeNull(); + get.mockImplementationOnce(() => { + throw new Error('undeclared host key'); + }); + expect(() => named.get('UNDECLARED')).toThrow('undeclared host key'); + expect(get.mock.calls.map(([name]) => name)).toEqual([ + 'FOOBAR', + 'FOOBAR', + 'LOG_LEVEL', + 'get', + 'LOG_LEVEL', + 'UNDECLARED', + ]); + }); +}); + +// These declarations are compiled by the focused typecheck. Callback bodies +// need not execute to assert their schema-specific context types. +const typed = schema({}, { env: declarations }); +typed.reducer(ctx => { + expectTypeOf(ctx.env.FOOBAR).toEqualTypeOf(); + expectTypeOf(ctx.env.ENABLE_EMAIL).toEqualTypeOf<'true' | 'false'>(); + expectTypeOf(ctx.env.LOG_LEVEL).toEqualTypeOf< + 'debug' | 'info' | 'error' | undefined + >(); + expectTypeOf(ctx.env.DEPLOYMENT_KIND).toEqualTypeOf<'production'>(); + expectTypeOf(ctx.env.get('FOOBAR')).toEqualTypeOf(); + expectTypeOf(ctx.env.get('get')).toEqualTypeOf(); + // @ts-expect-error Undeclared literal names have no checked accessor. + ctx.env.get('UNDECLARED'); + // @ts-expect-error No named access to undeclared keys. + void ctx.env.UNKNOWN; + // @ts-expect-error Environment access is read-only. + ctx.env.FOOBAR = 'changed'; +}); +function rejectPayloadType() { + // @ts-expect-error Payload enums cannot declare environment strings. + schema({}, { env: { BAD: t.enum('Payload', { value: t.string() }) } }); +} +void rejectPayloadType; diff --git a/crates/bindings-typescript/vitest.config.ts b/crates/bindings-typescript/vitest.config.ts index 7cfc858d47c..f532d437b74 100644 --- a/crates/bindings-typescript/vitest.config.ts +++ b/crates/bindings-typescript/vitest.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ alias: [ { find: 'spacetime:sys@2.0', replacement: sysMock }, { find: 'spacetime:sys@2.1', replacement: sysMock }, + { find: 'spacetime:sys@2.2', replacement: sysMock }, ], }, test: { diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index 4c8ea487c47..3638d35f9e2 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -98,6 +98,9 @@ pub use spacetimedb_bindings_macro::http_router as router; #[cfg(feature = "unstable")] #[non_exhaustive] pub struct HandlerContext { + /// Read-only access to this database's environment store. + pub env: crate::Environment, + /// The time at which the handler was started. pub timestamp: Timestamp, @@ -117,6 +120,7 @@ pub struct HandlerContext { impl HandlerContext { pub(crate) fn new(timestamp: Timestamp) -> Self { Self { + env: crate::Environment::default(), timestamp, http: HttpClient {}, #[cfg(feature = "rand08")] diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 2c7faa78c6b..190065c5e1f 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -919,10 +919,81 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Declare the complete publish-time environment schema and generate named accessors. +/// +/// Fields may be `String`, enums deriving [`EnvironmentValue`], or `Option` of +/// either, including type aliases. Values are supplied on every publish, never +/// in metadata. The macro generates an `EnvAccess` extension trait for a struct +/// named `Env`; import that trait if the declaration lives in another module. +/// The name `get` is reserved for generic checked access. +/// +/// ```no_run +/// #[derive(spacetimedb::EnvironmentValue)] +/// pub enum LogLevel { +/// #[env(value = "debug")] +/// Debug, +/// #[env(value = "info")] +/// Info, +/// } +/// #[spacetimedb::env] +/// pub struct Env { +/// pub API_KEY: String, +/// pub LOG_LEVEL: Option, +/// } +/// fn read(ctx: &spacetimedb::ReducerContext) { +/// let _: String = ctx.env.API_KEY(); +/// let _: Option = ctx.env.LOG_LEVEL(); +/// } +/// ``` +/// +/// Existing `#[env(values("a", "b"))]` field constraints remain supported for +/// `String` and `Option`; enum constraints come from their variants. +#[doc(inline)] +pub use spacetimedb_bindings_macro::env; + +/// Derive a typed environment value from an enum with unit variants. +/// +/// Each variant accepts its exact Rust name by default. Use +/// `#[env(value = "in progress")]` to map a variant to an arbitrary string, +/// including spaces, capitalization, Unicode or the empty string. Mappings must +/// be distinct, with 1 to 256 variants and at most 8192 UTF-8 bytes per string. +/// Generic enums and variants with payloads are not supported. +/// +/// The schema contains only allowed strings. A named environment accessor returns +/// this enum, or `Option` for an optional field; generic `env.get` still +/// returns `Option`. Missing required values and unmapped strings panic +/// with the key name only. Other derives and the enum's ordinary serialization +/// are unaffected. +#[doc(inline)] +pub use spacetimedb_bindings_macro::EnvironmentValue; + +/// Read-only access to this database's environment store. +/// +/// Reads use the current transaction. In a procedure outside a transaction, +/// each read uses a short snapshot; use `with_tx` to read related keys together. +/// Values are stored in plaintext and may be read by database collaborators. +#[derive(Clone, Copy, Debug, Default)] +pub struct Environment { + _private: (), +} + +impl Environment { + /// Return `None` for an absent declared optional key and `Some("")` for a present empty value. + /// Keys must be POSIX environment names of at most 256 bytes. + /// + /// # Panics + /// + /// Panics if the key is undeclared, invalid, or inaccessible in the current host call. + pub fn get(&self, key: &str) -> Option { + rt::env_get(key) + } +} + /// One of two possible types that can be passed as the first argument to a `#[view]`. /// The other is [`ViewContext`]. /// Use this type if the view does not depend on the caller's identity. pub struct AnonymousViewContext { + pub env: Environment, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -930,6 +1001,7 @@ pub struct AnonymousViewContext { impl Default for AnonymousViewContext { fn default() -> Self { Self { + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -939,6 +1011,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { + pub env: Environment, sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, @@ -948,6 +1021,7 @@ impl ViewContext { pub fn new(sender: Identity) -> Self { Self { sender, + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -978,6 +1052,8 @@ impl ViewContext { /// Implements the `DbContext` trait for accessing views into a database. #[non_exhaustive] pub struct ReducerContext { + /// Read-only access to the database environment in this transaction. + pub env: Environment, /// The `Identity` of the client that invoked the reducer. sender: Identity, @@ -1042,6 +1118,7 @@ impl ReducerContext { #[doc(hidden)] pub fn __dummy() -> Self { Self { + env: Environment::default(), db: Local {}, sender: Identity::__dummy(), timestamp: Timestamp::UNIX_EPOCH, @@ -1057,6 +1134,7 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { Self { + env: Environment::default(), db, sender, timestamp, @@ -1254,6 +1332,8 @@ fn with_tx(body: impl Fn(&TxContext) -> T, identity: Identity, connection_id: /// and exposes methods for running transactions and performing side-effecting operations. #[non_exhaustive] pub struct ProcedureContext { + /// Read-only access to the database environment. + pub env: Environment, /// The `Identity` of the client that invoked the procedure. sender: Identity, @@ -1285,6 +1365,7 @@ impl ProcedureContext { sender, timestamp, connection_id, + env: Environment::default(), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 7d456d73fde..4ebb26e0b63 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -917,6 +917,90 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { }) } +/// Implementation support for `#[env]` and `#[derive(EnvironmentValue)]`. +/// +/// The compiler resolves aliases before selecting metadata and accessors. Custom +/// implementations must keep their declared constraint and decoding in agreement. +/// The host independently validates every published string against that constraint. +#[doc(hidden)] +#[diagnostic::on_unimplemented( + message = "environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either" +)] +pub trait EnvironmentValue: Sized { + const OPTIONAL: bool; + + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint; + + /// Decode a checked host result. Errors must identify only the key, never its value. + fn from_environment(value: Option, key: &str) -> Self; + + fn get(environment: &crate::Environment, key: &str) -> Self { + Self::from_environment(environment.get(key), key) + } +} + +/// Required environment types supported by the optional-value implementation. +/// Derived enums and `String` implement this; `Option` deliberately does not. +#[doc(hidden)] +pub trait RequiredEnvironmentValue: EnvironmentValue {} + +impl EnvironmentValue for String { + const OPTIONAL: bool = false; + + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + spacetimedb_lib::environment::EnvironmentConstraint::AnyString + } + + fn from_environment(value: Option, key: &str) -> Self { + value.unwrap_or_else(|| panic!("required environment key is missing: {key}")) + } +} + +impl RequiredEnvironmentValue for String {} + +impl EnvironmentValue for Option { + const OPTIONAL: bool = true; + + fn constraint() -> spacetimedb_lib::environment::EnvironmentConstraint { + T::constraint() + } + + fn from_environment(value: Option, key: &str) -> Self { + value.map(|value| T::from_environment(Some(value), key)) + } +} + +mod string_environment_value_sealed { + pub trait Sealed {} + + impl Sealed for String {} + impl Sealed for Option {} +} + +/// Legacy field-level string constraints cannot override a typed enum's mapping. +#[doc(hidden)] +#[diagnostic::on_unimplemented( + message = "`#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = \"...\")]` instead" +)] +pub trait StringEnvironmentValue: EnvironmentValue + string_environment_value_sealed::Sealed { + fn with_constraint( + constraint: spacetimedb_lib::environment::EnvironmentConstraint, + ) -> spacetimedb_lib::environment::EnvironmentConstraint { + constraint + } +} + +impl StringEnvironmentValue for String {} +impl StringEnvironmentValue for Option {} + +/// Register declarative ENV metadata without reading any environment values. +#[doc(hidden)] +pub fn register_environment(declarations: fn() -> Vec) { + register_describer(move |module| { + module.inner.add_environment(declarations()); + }); +} + /// A builder for a module. #[derive(Default)] pub struct ModuleBuilder { @@ -983,6 +1067,7 @@ extern "C" fn __describe_module__(description: BytesSink) { } // Serialize the module to bsatn. + module.inner.ensure_environment(); let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); @@ -1319,6 +1404,13 @@ pub fn get_jwt(connection_id: ConnectionId) -> Option { Some(std::str::from_utf8(&buf).unwrap().to_string()) } +pub(crate) fn env_get(key: &str) -> Option { + let source = sys::env_get(key)?; + let mut buf = IterBuf::take(); + read_bytes_source_into(source, &mut buf); + Some(String::from_utf8(buf.to_vec()).expect("host environment values are UTF-8")) +} + /// Read `source` from the host fully into `buf`. pub(crate) fn read_bytes_source_into(source: BytesSource, buf: &mut Vec) { const INVALID: i16 = NO_SUCH_BYTES as i16; diff --git a/crates/bindings/tests/environment.rs b/crates/bindings/tests/environment.rs new file mode 100644 index 00000000000..b2b5181bc9d --- /dev/null +++ b/crates/bindings/tests/environment.rs @@ -0,0 +1,7 @@ +#[test] +fn environment_declaration_accessors_compile_with_exact_types() { + let tests = trybuild::TestCases::new(); + tests.pass("tests/pass/environment.rs"); + tests.compile_fail("tests/ui/environment_types.rs"); + tests.compile_fail("tests/ui/environment_enum.rs"); +} diff --git a/crates/bindings/tests/environment_enum_values.rs b/crates/bindings/tests/environment_enum_values.rs new file mode 100644 index 00000000000..ceecb647b5f --- /dev/null +++ b/crates/bindings/tests/environment_enum_values.rs @@ -0,0 +1,78 @@ +use spacetimedb::rt::EnvironmentValue as _; +use spacetimedb::spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use std::collections::BTreeMap; + +#[derive(Debug, PartialEq, Eq, spacetimedb::SpacetimeType, spacetimedb::EnvironmentValue)] +enum Mode { + #[env(value = "in progress")] + InProgress, + Ready, + #[env(value = "")] + Empty, + #[env(value = "hΓ©llo\0δΈ–η•Œ")] + Unicode, +} + +#[derive(Debug, PartialEq, Eq, spacetimedb::EnvironmentValue)] +enum Literal { + #[env(value = "only")] + Only, +} + +#[test] +fn typed_mappings_match_exact_schema_strings_and_optional_absence() { + let cases = [ + ("in progress", Mode::InProgress), + ("Ready", Mode::Ready), + ("", Mode::Empty), + ("hΓ©llo\0δΈ–η•Œ", Mode::Unicode), + ]; + assert_eq!( + Mode::constraint(), + EnvironmentConstraint::OneOf(cases.iter().map(|(s, _)| s.to_string()).collect()) + ); + assert_eq!(Option::::constraint(), Mode::constraint()); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "MODE".into(), + constraint: Mode::constraint(), + optional: false, + }]) + .unwrap(); + for (value, variant) in cases { + schema + .validate_values(&BTreeMap::from([("MODE".into(), value.into())])) + .unwrap(); + assert_eq!(Mode::from_environment(Some(value.into()), "MODE"), variant); + assert_eq!( + Option::::from_environment(Some(value.into()), "MODE"), + Some(variant) + ); + } + assert_eq!(Option::::from_environment(None, "MODE"), None); + assert_eq!(Literal::constraint(), EnvironmentConstraint::Literal("only".into())); + assert_eq!(Literal::from_environment(Some("only".into()), "VALUE"), Literal::Only); + for rejected in ["InProgress", "ready", "in progress ", "private-unmapped-value"] { + assert!(schema + .validate_values(&BTreeMap::from([("MODE".into(), rejected.into())])) + .is_err()); + } +} + +#[test] +fn decode_errors_report_the_key_without_the_supplied_value() { + for value in [None, Some("private-unmapped-value".into())] { + let error = std::panic::catch_unwind(|| Mode::from_environment(value, "MODE")).unwrap_err(); + let message = error.downcast_ref::().unwrap(); + assert!(message.contains("MODE")); + assert!(!message.contains("private-unmapped-value")); + assert!(!message.contains("in progress")); + } +} + +#[test] +fn environment_mapping_does_not_change_ordinary_enum_serialization() { + use spacetimedb::spacetimedb_lib::bsatn; + assert_eq!(bsatn::to_vec(&Mode::InProgress).unwrap(), vec![0]); + assert_eq!(bsatn::to_vec(&Mode::Ready).unwrap(), vec![1]); + assert_eq!(bsatn::from_slice::(&[2]).unwrap(), Mode::Empty); +} diff --git a/crates/bindings/tests/pass/environment.rs b/crates/bindings/tests/pass/environment.rs new file mode 100644 index 00000000000..136d631dc89 --- /dev/null +++ b/crates/bindings/tests/pass/environment.rs @@ -0,0 +1,50 @@ +#![deny(warnings)] + +use std::option::Option as Maybe; +use std::string::String as RenamedString; + +type RequiredAlias = RenamedString; +type OptionalAlias = Maybe; + +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Ready, +} +type ModeAlias = Mode; +type MaybeMode = Maybe; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + +const _: [(); 0] = [(); ::OPTIONAL as usize]; +const _: [(); 1] = [(); ::OPTIONAL as usize]; + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: RequiredAlias, + pub MODE: ModeAlias, + pub OPTIONAL_MODE: MaybeMode, + #[env(values("false", "true"))] + pub FLAG: String, + #[env(values(""))] + pub OPTIONAL: OptionalAlias, + pub get: Maybe, + pub r#type: RenamedString, +} + +fn reads(env: spacetimedb::Environment) { + let _: String = env.REQUIRED(); + let _: Mode = env.MODE(); + let _: Option = env.OPTIONAL_MODE(); + let _: String = env.FLAG(); + let _: Option = env.OPTIONAL(); + let _: Option = env.get("get"); + let _: String = env.r#type(); +} + +fn main() { + let _ = reads as fn(spacetimedb::Environment); +} diff --git a/crates/bindings/tests/ui/environment_enum.rs b/crates/bindings/tests/ui/environment_enum.rs new file mode 100644 index 00000000000..95a6387f5c4 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.rs @@ -0,0 +1,54 @@ +#[derive(spacetimedb::EnvironmentValue)] +struct NotEnum; + +#[derive(spacetimedb::EnvironmentValue)] +enum Empty {} + +#[derive(spacetimedb::EnvironmentValue)] +enum Generic { + Value(T), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Payload { + Value(String), +} + +#[derive(spacetimedb::EnvironmentValue)] +enum Duplicate { + #[env(value = "Same")] + First, + Same, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum DuplicateAttribute { + #[env(value = "x", value = "y")] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongAttribute { + #[env(values("x"))] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +enum WrongLiteral { + #[env(value = 1)] + Value, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + Ready, +} + +#[spacetimedb::env] +pub struct Env { + #[env(values("other"))] + pub MODE: Mode, + pub NESTED: Option>, +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_enum.stderr b/crates/bindings/tests/ui/environment_enum.stderr new file mode 100644 index 00000000000..f3f3d25ddc0 --- /dev/null +++ b/crates/bindings/tests/ui/environment_enum.stderr @@ -0,0 +1,85 @@ +error: EnvironmentValue requires an enum with unit variants + --> tests/ui/environment_enum.rs:2:8 + | +2 | struct NotEnum; + | ^^^^^^^ + +error: environment value enums require 1 to 256 variants + --> tests/ui/environment_enum.rs:5:6 + | +5 | enum Empty {} + | ^^^^^ + +error: environment value enums cannot be generic + --> tests/ui/environment_enum.rs:8:13 + | +8 | enum Generic { + | ^^^ + +error: environment value variants cannot have payloads + --> tests/ui/environment_enum.rs:14:10 + | +14 | Value(String), + | ^^^^^^^^ + +error: environment variants must map to distinct strings + --> tests/ui/environment_enum.rs:21:5 + | +21 | Same, + | ^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:26:24 + | +26 | #[env(value = "x", value = "y")] + | ^^^^^ + +error: expected one value = "literal" mapping + --> tests/ui/environment_enum.rs:32:11 + | +32 | #[env(values("x"))] + | ^^^^^^ + +error: expected string literal + --> tests/ui/environment_enum.rs:38:19 + | +38 | #[env(value = 1)] + | ^ + +error[E0277]: the trait bound `Option: RequiredEnvironmentValue` is not satisfied + --> tests/ui/environment_enum.rs:51:17 + | +51 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `Option` + | +help: the following other types implement trait `RequiredEnvironmentValue` + --> tests/ui/environment_enum.rs:42:10 + | +42 | #[derive(spacetimedb::EnvironmentValue)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Mode` + | + ::: src/rt.rs + | + | impl RequiredEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + = note: required for `Option>` to implement `EnvironmentValue` + = note: this error originates in the derive macro `spacetimedb::EnvironmentValue` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: `#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = "...")]` instead + --> tests/ui/environment_enum.rs:50:15 + | +50 | pub MODE: Mode, + | ^^^^ unsatisfied trait bound + | +help: the trait `StringEnvironmentValue` is not implemented for `Mode` + --> tests/ui/environment_enum.rs:43:1 + | +43 | pub enum Mode { + | ^^^^^^^^^^^^^ +help: the following other types implement trait `StringEnvironmentValue` + --> src/rt.rs + | + | impl StringEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` + | impl StringEnvironmentValue for Option {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `Option` diff --git a/crates/bindings/tests/ui/environment_types.rs b/crates/bindings/tests/ui/environment_types.rs new file mode 100644 index 00000000000..27693688e4f --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.rs @@ -0,0 +1,27 @@ +mod shadowed_string { + pub struct String; + + #[spacetimedb::env] + pub struct ShadowedString { + pub VALUE: String, + } +} + +mod shadowed_option { + pub struct Option(T); + + #[spacetimedb::env] + pub struct ShadowedOption { + pub VALUE: Option, + } +} + +#[spacetimedb::env] +pub struct Unsupported { + pub BOOL: bool, + pub NESTED: Option>, + // Even without a generated named accessor, metadata must check the type. + pub get: u32, +} + +fn main() {} diff --git a/crates/bindings/tests/ui/environment_types.stderr b/crates/bindings/tests/ui/environment_types.stderr new file mode 100644 index 00000000000..f884a635897 --- /dev/null +++ b/crates/bindings/tests/ui/environment_types.stderr @@ -0,0 +1,82 @@ +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:6:20 + | +6 | pub VALUE: String, + | ^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_string::String` + --> tests/ui/environment_types.rs:2:5 + | +2 | pub struct String; + | ^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:15:20 + | +15 | pub VALUE: Option, + | ^^^^^^^^^^^^^^ unsatisfied trait bound + | +help: the trait `EnvironmentValue` is not implemented for `shadowed_option::Option` + --> tests/ui/environment_types.rs:11:5 + | +11 | pub struct Option(T); + | ^^^^^^^^^^^^^^^^^^^^ +help: the following other types implement trait `EnvironmentValue` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:21:15 + | +21 | pub BOOL: bool, + | ^^^^ the trait `EnvironmentValue` is not implemented for `bool` + | +help: the following other types implement trait `EnvironmentValue` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` + +error[E0277]: the trait bound `std::option::Option: RequiredEnvironmentValue` is not satisfied + --> tests/ui/environment_types.rs:22:17 + | +22 | pub NESTED: Option>, + | ^^^^^^^^^^^^^^^^^^^^^^ the trait `RequiredEnvironmentValue` is not implemented for `std::option::Option` + | +help: the trait `RequiredEnvironmentValue` is implemented for `std::string::String` + --> src/rt.rs + | + | impl RequiredEnvironmentValue for String {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: required for `std::option::Option>` to implement `EnvironmentValue` + +error[E0277]: environment fields must be `String`, an enum deriving `EnvironmentValue`, or an `Option` of either + --> tests/ui/environment_types.rs:24:14 + | +24 | pub get: u32, + | ^^^ the trait `EnvironmentValue` is not implemented for `u32` + | +help: the following other types implement trait `EnvironmentValue` + --> src/rt.rs + | + | impl EnvironmentValue for String { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::string::String` +... + | impl EnvironmentValue for Option { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `std::option::Option` diff --git a/crates/bindings/tests/ui/http_handlers.stderr b/crates/bindings/tests/ui/http_handlers.stderr index 960b3cdeae9..c28936ac12e 100644 --- a/crates/bindings/tests/ui/http_handlers.stderr +++ b/crates/bindings/tests/ui/http_handlers.stderr @@ -175,7 +175,7 @@ error[E0609]: no field `db` on type `&mut HandlerContext` 53 | let _rows = ctx.db.test_table().iter(); | ^^ unknown field | - = note: available fields are: `timestamp`, `http` + = note: available fields are: `env`, `timestamp`, `http` error[E0308]: mismatched types --> tests/ui/http_handlers.rs:66:4 diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9411e6f5e5e..5357e8d067b 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -4,6 +4,7 @@ mod config; pub(crate) mod detect; mod edit_distance; mod errors; +mod schema_extract; pub mod spacetime_config; mod subcommands; mod tasks; @@ -22,6 +23,7 @@ pub use tasks::build; pub fn get_subcommands() -> Vec { vec![ publish::cli(), + env::cli(), delete::cli(), logs::cli(), call::cli(), @@ -57,6 +59,7 @@ pub async fn exec_subcommand( "describe" => describe::exec(config, args).await, "dev" => dev::exec(config, args).await, "publish" => publish::exec(config, args).await, + "env" => env::exec(config, args).await, "delete" => delete::exec(config, args).await, "logs" => logs::exec(config, args).await, "sql" => sql::exec(config, args).await, diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs new file mode 100644 index 00000000000..c00481ae40f --- /dev/null +++ b/crates/cli/src/schema_extract.rs @@ -0,0 +1,172 @@ +//! Local schema extraction shared by publish and generate. Each invocation +//! inspects a private copy of exact bounded input bytes, bounds its output and +//! lifetime, and owns the child through kill/wait on failure or cancellation. +use anyhow::{ensure, Context}; +use spacetimedb_lib::{sats::serde::SerdeWrapper, RawModuleDef}; +use spacetimedb_schema::def::ModuleDef; +use std::{ + path::{Path, PathBuf}, + process::Stdio, + time::Duration, +}; +use tokio::io::AsyncReadExt; + +fn extractor_path() -> anyhow::Result { + std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") + .map(PathBuf::from) + .map(Ok) + .unwrap_or_else(|| crate::util::resolve_sibling_binary("spacetimedb-standalone")) +} + +/// Keep generate's synchronous injectable function API. Its small dedicated +/// runtime works both inside and outside a caller's Tokio runtime; process +/// ownership and validation are identical to publish's async path. +pub(crate) fn from_path(path: &Path) -> anyhow::Result { + let bytes = read_program(path)?; + let host_type = match path.extension().and_then(|ext| ext.to_str()) { + Some("wasm") => "Wasm", + Some("js") => "Js", + _ => anyhow::bail!("Cannot determine module type from file extension"), + }; + inspect_blocking(extractor_path()?, bytes, host_type.into()) +} + +fn inspect_blocking(extractor: PathBuf, bytes: Vec, host_type: String) -> anyhow::Result { + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(inspect_with(extractor, bytes, host_type, INSPECT_TIMEOUT)) + }) + .join() + .map_err(|_| anyhow::anyhow!("Local module inspection thread failed"))? +} + +pub(crate) fn read_program(path: &std::path::Path) -> anyhow::Result> { + use std::io::Read; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, + "Module exceeds publish size limit" + ); + Ok(bytes) +} + +const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; +const INSPECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Inspect exactly the artifact bytes that will be uploaded. The private copy +/// prevents path replacement between inspection and upload, including --bin-path. +/// This invokes only local extraction, never a server or a saved CLI context. +pub(crate) async fn inspect(program: &[u8], host_type: &str) -> anyhow::Result { + let extractor = extractor_path()?; + inspect_with(extractor, program.to_vec(), host_type.to_owned(), INSPECT_TIMEOUT).await +} + +pub(crate) async fn inspect_with( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, +) -> anyhow::Result { + inspect_observed(extractor, program, host_type, deadline, Observation::default()).await +} + +#[derive(Default)] +struct Observation { + #[cfg(test)] + started: Option>, + #[cfg(test)] + reaped: Option>, +} +impl Observation { + fn started(&mut self, _pid: Option) { + #[cfg(test)] + if let Some(send) = self.started.take() { + let _ = send.send(_pid.expect("new child has a PID")); + } + } + fn reaped(&mut self, _status: std::process::ExitStatus) { + #[cfg(test)] + if let Some(send) = self.reaped.take() { + let _ = send.send(_status); + } + } +} + +async fn inspect_observed( + extractor: PathBuf, + program: Vec, + host_type: String, + deadline: Duration, + mut observation: Observation, +) -> anyhow::Result { + let (mut send, mut recv) = tokio::sync::oneshot::channel(); + // This owner retains the child and private file until actual reaping, even + // when its caller drops while extraction or stdout reading is in progress. + tokio::spawn(async move { + let result = async { + let dir = tempfile::tempdir().context("Cannot create private module inspection directory")?; + let module = dir.path().join("module"); + tokio::fs::write(&module, program) + .await + .context("Cannot prepare module inspection input")?; + let mut child = tokio::process::Command::new(extractor) + .arg("extract-schema") + .arg(&module) + .arg("--host-type") + .arg(host_type.to_ascii_lowercase()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .context("Cannot start local module schema inspection")?; + observation.started(child.id()); + let mut output = Vec::new(); + let mut stdout = child + .stdout + .take() + .context("Module inspection stdout unavailable")? + .take(MAX_SCHEMA_BYTES + 1); + let result = tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("Module inspection cancelled")), + result = tokio::time::timeout(deadline, async { + stdout.read_to_end(&mut output).await.context("Cannot read local module schema")?; + ensure!(output.len() as u64 <= MAX_SCHEMA_BYTES, "Local module schema exceeds output limit"); + let status = child.wait().await.context("Cannot reap local module inspector")?; + observation.reaped(status); + ensure!(status.success(), "Local module schema inspection failed"); + Ok(()) + }) => result.unwrap_or_else(|_| Err(anyhow::anyhow!("Local module schema inspection timed out"))), + }; + if result.is_err() { + // Queue termination, then retain ownership through positive reaping. + let _ = child.start_kill(); + let status = child + .wait() + .await + .context("Cannot reap failed local module inspector")?; + observation.reaped(status); + } + result?; + // Neither parser nor validation diagnostics may echo schema literals. + let SerdeWrapper::(raw) = serde_json::from_slice(&output) + .map_err(|_| anyhow::anyhow!("Local module inspector returned invalid schema data"))?; + let schema = + ModuleDef::try_from(raw).map_err(|_| anyhow::anyhow!("Local module schema validation failed"))?; + Ok(schema) + } + .await; + let _ = send.send(result); + }); + (&mut recv).await.context("Local module inspection owner failed")? +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs new file mode 100644 index 00000000000..d8c6daea04a --- /dev/null +++ b/crates/cli/src/schema_extract/tests.rs @@ -0,0 +1,143 @@ +use super::*; +use spacetimedb_lib::environment::{ + EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration, EnvironmentSchema, +}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +// These fixtures invoke only a locally generated executable in an owned tempdir. +// No CLI config, server credentials or user environment are imported. +#[cfg(unix)] +fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("inspector"); + std::fs::write(&path, format!("#!/bin/sh\n{script}\n")).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + (dir, path) +} + +#[cfg(unix)] +#[tokio::test] +async fn local_inspection_passes_exact_bytes_host_and_requires_success() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = js ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = exact-artifact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let result = inspect_with( + extractor, + b"exact-artifact".to_vec(), + "Js".into(), + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(result.environment(), &schema()); + drop(dir); + let (_dir, extractor) = inspector(&format!("printf '%s' '{}'; exit 9", json.replace('\'', "'\\''"))); + assert!( + inspect_with(extractor, b"anything".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .is_err() + ); +} + +#[cfg(unix)] +#[tokio::test(flavor = "current_thread")] +async fn synchronous_generate_adapter_uses_same_exact_byte_protocol_inside_a_runtime() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + let raw = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + }); + let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); + let (_dir, extractor) = inspector(&format!( + "[ \"$1\" = extract-schema ] && [ \"$3\" = --host-type ] && [ \"$4\" = wasm ] || exit 2\n[ \"$(/bin/cat \"$2\")\" = generate-exact ] || exit 3\nprintf '%s' '{}'", json.replace('\'', "'\\''") + )); + let module = inspect_blocking(extractor, b"generate-exact".to_vec(), "Wasm".into()).unwrap(); + assert_eq!(module.environment(), &schema()); +} + +#[cfg(unix)] +#[tokio::test] +async fn invalid_and_oversize_inspector_output_never_becomes_diagnostics() { + for script in [ + "printf 'generated-inspector-secret'", + "exec /usr/bin/head -c 16777217 /dev/zero", + ] { + let (_dir, extractor) = inspector(script); + let error = inspect_with(extractor, b"input".to_vec(), "Wasm".into(), Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(!format!("{error:#}").contains("generated-inspector-secret")); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn timeout_and_caller_drop_produce_positive_wait_receipts() { + for cancel in [false, true] { + let (_dir, extractor) = inspector("while :; do :; done"); + let (started, started_receipt) = tokio::sync::oneshot::channel(); + let (reaped, reaped_receipt) = tokio::sync::oneshot::channel(); + let operation = tokio::spawn(inspect_observed( + extractor, + b"input".to_vec(), + "Wasm".into(), + if cancel { + Duration::from_secs(30) + } else { + Duration::from_millis(300) + }, + Observation { + started: Some(started), + reaped: Some(reaped), + }, + )); + assert!( + tokio::time::timeout(Duration::from_secs(5), started_receipt) + .await + .unwrap() + .unwrap() + > 0 + ); + if cancel { + operation.abort(); + let _ = operation.await; + } else { + assert!(operation.await.unwrap().is_err()); + } + let status = tokio::time::timeout(Duration::from_secs(5), reaped_receipt) + .await + .unwrap() + .unwrap(); + assert!(!status.success()); + } +} diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..32d3030082a 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use clap::{ArgMatches, Command}; use path_clean::PathClean; @@ -183,6 +185,14 @@ impl SpacetimeConfig { let mut fields = self.additional_fields.clone(); if let Some(parent) = parent_fields { for (key, value) in parent { + if key == "env" + && let Some(child) = fields.get_mut(key) + { + let mut combined = value.clone(); + environment::overlay(&mut combined, child); + *child = combined; + continue; + } if fields.contains_key(key) { continue; } @@ -310,6 +320,9 @@ impl CommandSchemaBuilder { // Check that all the defined keys exist in clap for key in &self.keys { + if key.config_only { + continue; + } if !clap_arg_names.contains(key.clap_arg_name()) { return Err(CommandConfigError::InvalidClapReference { config_name: key.config_name().to_string(), @@ -343,6 +356,9 @@ impl CommandSchemaBuilder { let mut config_to_alias_map = HashMap::new(); for key in &self.keys { + if key.config_only { + continue; + } let config_name = key.config_name().to_string(); let clap_name = key.clap_arg_name().to_string(); @@ -398,6 +414,13 @@ impl CommandSchema { matches: &ArgMatches, config_name: &str, ) -> Result, CommandConfigError> { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return Ok(None); + } // Check clap with mapped name (if from_clap was used, use that name, otherwise use config name) let clap_name = self .config_to_clap @@ -428,6 +451,13 @@ impl CommandSchema { /// Check if a value was provided via CLI (not from config). /// Only returns true if the user explicitly provided the value, not if it came from a default. pub fn is_from_cli(&self, matches: &ArgMatches, config_name: &str) -> bool { + if self + .keys + .iter() + .any(|key| key.config_name() == config_name && key.config_only) + { + return false; + } // Check clap with mapped name let clap_name = self .config_to_clap @@ -567,6 +597,8 @@ impl CommandSchema { pub struct Key { /// The key name in the config file (e.g., "module-path") config_name: String, + /// This field has no value-bearing CLI argument (even if a same-named selector exists). + config_only: bool, /// The corresponding clap argument name (e.g., "project-path"), if different clap_name: Option, /// Alias for a clap argument, useful for example if we have to deprecate a clap @@ -585,6 +617,7 @@ impl Key { pub fn new(name: impl Into) -> Self { Self { config_name: name.into(), + config_only: false, clap_name: None, clap_alias: None, module_specific: false, @@ -593,6 +626,12 @@ impl Key { } } + /// Read this key exclusively from project configuration. + pub fn config_only(mut self) -> Self { + self.config_only = true; + self + } + /// Map this config key to a different clap argument name. When fetching values /// the key that is defined should be used. /// Example: Key::new("module-path").from_clap("project-path") @@ -813,8 +852,10 @@ impl SpacetimeConfig { let content = std::fs::read_to_string(path).with_context(|| format!("Failed to read config file: {}", path.display()))?; - let config: Self = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; + let config: Self = environment::decode_config(value) + .with_context(|| format!("Invalid configuration structure in {}", path.display()))?; Ok(config) } @@ -911,8 +952,8 @@ fn load_json_value(path: &Path) -> anyhow::Result> { // comments and formatting since json5 crate doesn't support serialization. remove_source_config_from_text(path, &content); - let value: serde_json::Value = json5::from_str(&content) - .map_err(|e| anyhow::anyhow!("Failed to parse config file {}: {}", path.display(), e))?; + let value = + environment::parse(&content).with_context(|| format!("Failed to parse config file {}", path.display()))?; Ok(Some(value)) } @@ -1000,6 +1041,12 @@ fn overlay_json(base: &mut serde_json::Value, mut overlay: serde_json::Value, so base_obj.insert(key.clone(), other); } } + } else if key == "env" { + if let Some(base_env) = base_obj.get_mut(key) { + environment::overlay(base_env, value); + } else { + base_obj.insert(key.clone(), value_owned); + } } else { base_obj.insert(key.clone(), value_owned); } @@ -1068,7 +1115,7 @@ pub fn find_and_load_with_env_from(env: Option<&str>, start_dir: PathBuf) -> any } } - let config: SpacetimeConfig = serde_json::from_value(merged).context("Failed to deserialize merged config")?; + let config = environment::decode_config(merged)?; Ok(Some(LoadedConfig { config, diff --git a/crates/cli/src/spacetime_config/environment.rs b/crates/cli/src/spacetime_config/environment.rs new file mode 100644 index 00000000000..4fea7a548c7 --- /dev/null +++ b/crates/cli/src/spacetime_config/environment.rs @@ -0,0 +1,332 @@ +//! Preserve JSON numeric values before the JSON5 deserializer can round them. +//! For example, converting `9007199254740993` through `f64` yields +//! `9007199254740992`, changing an environment value before it reaches the module. +//! This also permits existing comments, unquoted names and trailing commas. +use serde_json::Value; + +pub(super) fn parse(content: &str) -> anyhow::Result { + let bytes = content.as_bytes(); + let mut numbers = Vec::new(); + let mut replacements = Vec::new(); + let mut strings: Vec = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let start = i; + match bytes[i] { + b'\'' | b'"' => { + let quote = bytes[i]; + i += 1; + while i < bytes.len() { + if bytes[i] == b'\\' { + i = (i + 2).min(bytes.len()); + } else if bytes[i] == quote { + i += 1; + break; + } else { + i += 1; + } + } + } + b'/' if bytes.get(i + 1) == Some(&b'/') => { + i += 2; + while i < bytes.len() { + let ch = content[i..].chars().next().unwrap(); + if matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}') { + break; + } + i += ch.len_utf8(); + } + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + i = (i + 2).min(bytes.len()); + } + b'{' | b'}' | b'[' | b']' | b':' | b',' => i += 1, + _ if is_space(content[i..].chars().next().unwrap()) => i += content[i..].chars().next().unwrap().len_utf8(), + _ => { + while i < bytes.len() + && !is_space(content[i..].chars().next().unwrap()) + && !matches!(bytes[i], b'{' | b'}' | b'[' | b']' | b':' | b',' | b'/' | b'\'' | b'"') + { + i += content[i..].chars().next().unwrap().len_utf8(); + } + if i == start { + i += content[i..].chars().next().unwrap().len_utf8(); + } + let token = &content[start..i]; + if (token.starts_with(|c: char| c.is_ascii_digit() || matches!(c, '-' | '+' | '.')) + || matches!(token, "Infinity" | "NaN")) + && !content[i..].trim_start_matches(is_space).starts_with(':') + { + replacements.push((start, i)); + numbers.push(token); + continue; + } + } + } + if matches!(bytes[start], b'\'' | b'"') { + strings.push(json5::from_str(&content[start..i]).map_err(|_| anyhow::anyhow!("Invalid JSON5 string"))?); + } + } + // Check decoded strings as well: Unicode escapes must not manufacture an + // internal marker and cause a string to be interpreted as a number. + let mut prefix = "__spacetime_numeric_".to_owned(); + while content.contains(&prefix) || strings.iter().any(|s| s.contains(&prefix)) { + prefix.push('_'); + } + let mut text = String::with_capacity(content.len()); + let mut previous = 0; + for (index, (start, end)) in replacements.into_iter().enumerate() { + text.push_str(&content[previous..start]); + text.push_str(&format!("\"{prefix}{index}\"")); + previous = end; + } + text.push_str(&content[previous..]); + // Parser diagnostics may quote the source line, which can contain secrets. + let mut value: Value = json5::from_str(&text).map_err(|_| anyhow::anyhow!("Invalid JSON5 configuration"))?; + restore(&mut value, &prefix, &numbers, None)?; + Ok(value) +} + +/// Deserialize through JSON text so Serde's flattened-field buffer does not +/// receive visit_u128 from Value's deserializer. That buffer cannot represent +/// u128, while the arbitrary-precision JSON parser preserves its decimal token. +/// Diagnostics discard values while retaining a bounded, ordinary unknown field +/// name, which is useful for correcting misspelled configuration options. +pub(super) fn decode_config(value: Value) -> anyhow::Result { + let encoded = serde_json::to_vec(&value).map_err(|_| anyhow::anyhow!("Invalid configuration structure"))?; + serde_json::from_slice(&encoded).map_err(|error| { + let diagnostic = error.to_string(); + if let Some((field, suffix)) = diagnostic + .strip_prefix("unknown field `") + .and_then(|message| message.split_once('`')) + && suffix.starts_with(", expected ") + && !field.is_empty() + && field.len() <= 64 + && field + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return anyhow::anyhow!("unknown field `{field}`"); + } + anyhow::anyhow!("Invalid configuration structure") + }) +} + +fn is_space(ch: char) -> bool { + ch.is_whitespace() || ch == '\u{feff}' +} + +fn restore(value: &mut Value, prefix: &str, numbers: &[&str], env_key: Option<&str>) -> anyhow::Result<()> { + match value { + Value::String(s) => { + if let Some(index) = s.strip_prefix(prefix).and_then(|s| s.parse::().ok()) { + let token = numbers[index]; + *value = if let Some(env_key) = env_key { + Value::Number(token.parse().map_err(|_| { + anyhow::anyhow!( + "Environment key {:?}: numeric config input must use JSON number syntax", + env_key + ) + })?) + } else { + // Preserve existing JSON5 conveniences outside env. JSON numbers retain + // arbitrary precision throughout layering and config serialization. + token + .parse() + .map(Value::Number) + .or_else(|_| json5::from_str(token)) + .map_err(|_| anyhow::anyhow!("Invalid numeric configuration input"))? + }; + } + } + Value::Object(object) => { + for (key, value) in object { + if key == "env" && env_key.is_none() { + if let Value::Object(env) = value { + for (name, value) in env { + restore(value, prefix, numbers, Some(name))?; + } + } else { + restore(value, prefix, numbers, Some("env"))?; + } + } else { + restore(value, prefix, numbers, env_key)?; + } + } + } + Value::Array(values) => { + for value in values { + restore(value, prefix, numbers, env_key)?; + } + } + _ => {} + } + Ok(()) +} + +/// Merge only object-valued env maps. Invalid higher precedence input remains +/// invalid instead of silently falling back to the lower layer. +pub(super) fn overlay(base: &mut Value, higher: &Value) { + if let (Some(base), Some(higher)) = (base.as_object_mut(), higher.as_object()) { + base.extend(higher.clone()); + } else { + *base = higher.clone(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}; + #[test] + fn numeric_input_is_lossless_with_json5_comments_and_strings() { + let value = parse( + r#"{ // comment 111 + env: { HUGE: 9007199254740993123456789, DECIMAL: 0.1234567890123456789012345, + EXP: 1.000000000000000000001e+300, STRING: '12 // 99', BOOL: false, }, + 'server': 'http://127.0.0.1:9', /* 222 */ 'num-replicas': 3, + }"#, + ) + .unwrap(); + assert_eq!(value["env"]["HUGE"].to_string(), "9007199254740993123456789"); + assert_eq!(value["env"]["DECIMAL"].to_string(), "0.1234567890123456789012345"); + assert_eq!(value["env"]["EXP"].to_string(), "1.000000000000000000001e+300"); + assert_eq!(value["env"]["STRING"], "12 // 99"); + assert_eq!(value["num-replicas"], 3); + let unicode_lines = parse("{ // comment\u{2028} env: { NUMBER: 9007199254740993123456789\u{feff}} }").unwrap(); + assert_eq!(unicode_lines["env"]["NUMBER"].to_string(), "9007199254740993123456789"); + } + #[test] + fn numeric_extensions_and_parse_errors_do_not_quote_secret_input() { + for input in ["NaN", "Infinity", "-Infinity", "0xFF", "+2", ".5"] { + assert!(parse(&format!("{{env: {{KEY: {input}}}}}")).is_err()); + } + let error = parse("{env: {KEY: 'generated-secret-sentinel', broken }").unwrap_err(); + assert!(!format!("{error:#}").contains("generated-secret-sentinel")); + let value = parse("{env: {A: '__spacetime_numeric_0', B: 2}}").unwrap(); + assert_eq!(value["env"]["A"], "__spacetime_numeric_0"); + assert_eq!(value["env"]["B"], 2); + let escaped = parse(r#"{env: {A: '\u005f_spacetime_numeric_999', B: 2}}"#).unwrap(); + assert_eq!(escaped["env"]["A"], "__spacetime_numeric_999"); + } + #[test] + fn four_layers_and_parent_child_merge_individual_keys() { + let dir = tempfile::tempdir().unwrap(); + for (file, json) in [ + ( + "spacetime.json", + r#"{database:'parent', env:{A:'base',B:1},children:[{database:'child',env:{B:2,C:'child'}}]}"#, + ), + ( + "spacetime.local.json", + r#"{env:{A:'local'},children:[{env:{D:'local-child'}}]}"#, + ), + ( + "spacetime.prod.json", + r#"{env:{A:'prod',E:true},children:[{env:{B:3}}]}"#, + ), + ( + "spacetime.prod.local.json", + r#"{env:{A:'prod-local'},children:[{env:{}}]}"#, + ), + ] { + std::fs::write(dir.path().join(file), json).unwrap(); + } + let config = find_and_load_with_env_from(Some("prod"), dir.path().to_owned()) + .unwrap() + .unwrap(); + let targets = config.config.collect_all_targets_with_inheritance(); + assert_eq!( + targets[0].fields["env"], + serde_json::json!({"A":"prod-local","B":1,"E":true}) + ); + assert_eq!( + targets[1].fields["env"], + serde_json::json!({"A":"prod-local","B":3,"C":"child","D":"local-child","E":true}) + ); + // Direct loads use the same lossless parser. + let base = SpacetimeConfig::load(&dir.path().join("spacetime.json")).unwrap(); + assert_eq!(base.additional_fields["env"]["B"], 1); + } + #[test] + fn loaded_flattened_configuration_preserves_full_precision_and_redacts_structure_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + for number in [ + "9007199254740993123456789", + "0.1234567890123456789012345", + "1.000000000000000000001e+300", + ] { + std::fs::write( + &path, + format!(r#"{{"database":"owned","env":{{"NUMBER":{number}}},"children":[{{"database":"child"}}]}}"#), + ) + .unwrap(); + let direct = SpacetimeConfig::load(&path).unwrap(); + let layered = find_and_load_with_env_from(None, dir.path().to_owned()) + .unwrap() + .unwrap() + .config; + for config in [direct, layered] { + for target in config.collect_all_targets_with_inheritance() { + assert_eq!(target.fields["env"]["NUMBER"].to_string(), number); + } + } + } + std::fs::write( + &path, + r#"{"children":"private-structure-sentinel","env":{"NUMBER":9007199254740993123456789}}"#, + ) + .unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let error = format!("{error:#}"); + assert!(!error.contains("private-structure-sentinel")); + assert!(!error.contains("9007199254740993123456789")); + } + } + + #[test] + fn invalid_higher_layer_is_not_an_empty_map_or_fallback() { + let mut base = serde_json::json!({"A":"base"}); + overlay(&mut base, &Value::Null); + assert!(base.is_null()); + let config: SpacetimeConfig = + serde_json::from_value(serde_json::json!({"env":{"A":1},"children":[{"env":null}]})).unwrap(); + assert!(config.collect_all_targets_with_inheritance()[1].fields["env"].is_null()); + } + + #[test] + fn unknown_configuration_fields_are_actionable_without_exposing_values() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("spacetime.json"); + let secret = "private-configuration-value-sentinel"; + let value = serde_json::json!({"dev": {"run_command": secret}, "env": {"TOKEN": secret}}); + std::fs::write(&path, serde_json::to_vec(&value).unwrap()).unwrap(); + for error in [ + SpacetimeConfig::load(&path).unwrap_err(), + find_and_load_with_env_from(None, dir.path().to_owned()).err().unwrap(), + ] { + let diagnostic = format!("{error:#}"); + assert!(diagnostic.contains("unknown field `run_command`")); + assert!(!diagnostic.contains(secret)); + } + + for field in [ + "control\ncharacters".to_owned(), + "x".repeat(65), + "quote`injection".to_owned(), + ] { + let error = decode_config(serde_json::json!({"dev": {field: secret}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } + let error = decode_config(serde_json::json!({"dev": {"run": [secret]}})).unwrap_err(); + assert_eq!(error.to_string(), "Invalid configuration structure"); + } +} diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs new file mode 100644 index 00000000000..01a1f25afe9 --- /dev/null +++ b/crates/cli/src/subcommands/env.rs @@ -0,0 +1,321 @@ +//! Read-only conveniences over the private st_env SQL table. +use anyhow::{ensure, Context}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_lib::environment::{validate_key, MAX_ENV_KEY_BYTES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS}; + +use super::{ + db_arg_resolution::{load_config_db_targets, resolve_database_arg}, + sql, +}; +use crate::{api::ClientApi, common_args, Config}; + +pub fn cli() -> Command { + let target = |command: Command| { + command + .arg( + Arg::new("database") + .index(1) + .required(true) + .help("The database name, identity, or configured target"), + ) + .arg(common_args::server()) + .arg(common_args::anonymous()) + .arg(common_args::yes()) + .arg(common_args::confirmed()) + .arg( + Arg::new("no_config") + .long("no-config") + .action(ArgAction::SetTrue) + .help("Ignore project configuration when resolving the database target"), + ) + }; + Command::new("env") + .about("Inspect published database environment variables") + .subcommand_required(true) + .subcommand(target( + Command::new("get").about("Read one published environment value").arg( + Arg::new("key") + .index(2) + .required(true) + .help("The declared environment key to read"), + ), + )) + .subcommand(target( + Command::new("list").about("List published environment keys and values"), + )) +} + +#[derive(Clone)] +enum Query { + List, + Get(String), +} +impl Query { + fn sql(&self) -> anyhow::Result { + match self { + Self::List => Ok("SELECT key, value FROM st_env".into()), + Self::Get(key) => { + // POSIX names cannot contain quotes or SQL syntax. + validate_key(key).map_err(|_| anyhow::anyhow!("Invalid environment key name"))?; + Ok(format!("SELECT value FROM st_env WHERE key = '{key}'")) + } + } + } +} + +pub async fn exec(config: Config, args: &ArgMatches) -> anyhow::Result<()> { + let (command, args) = args.subcommand().context("Expected env get or list")?; + let query = match command { + "list" => Query::List, + "get" => Query::Get( + args.get_one::("key") + .context("Expected environment key")? + .clone(), + ), + _ => anyhow::bail!("Environment values can only be changed by publishing"), + }; + query.sql()?; + let targets = load_config_db_targets(args.get_flag("no_config"))?; + let database = resolve_database_arg( + args.get_one::("database").map(String::as_str), + targets.as_deref(), + "spacetime env get/list ", + )?; + let con = sql::parse_req(config, args, &database.database, database.server.as_deref()).await?; + let mut request = ClientApi::new(con).sql(); + if let Some(confirmed) = args.get_one::("confirmed") { + request = request.query(&[("confirmed", confirmed)]); + } + print!("{}", fetch(request, query).await?); + Ok(()) +} + +async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result { + use futures::StreamExt; + let response = request + .timeout(std::time::Duration::from_secs(30)) + .body(query.sql()?) + .send() + .await?; + ensure!( + response.status().is_success(), + "Environment read failed with HTTP {}", + response.status() + ); + let mut body = Vec::new(); + let limit = MAX_ENV_VARS * (MAX_ENV_KEY_BYTES + MAX_ENV_VALUE_BYTES) * 6 + 64 * 1024; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + ensure!( + body.len().saturating_add(chunk.len()) <= limit, + "Environment read response exceeds limit" + ); + body.extend_from_slice(&chunk); + } + render(&body, &query) +} + +fn render(body: &[u8], query: &Query) -> anyhow::Result { + // Validate the requested projection before rendering any response values. + let results: Vec>> = + serde_json::from_slice(body).map_err(|_| anyhow::anyhow!("Invalid environment read response"))?; + ensure!(results.len() == 1, "Invalid environment read result count"); + let result = &results[0]; + let expected: &[&str] = match query { + Query::List => &["key", "value"], + Query::Get(_) => &["value"], + }; + ensure!( + result.schema.elements.len() == expected.len() + && result.schema.elements.iter().zip(expected).all(|(column, name)| { + column.name.as_deref() == Some(*name) && column.algebraic_type == spacetimedb_lib::AlgebraicType::String + }), + "Invalid environment read projection" + ); + ensure!(result.rows.len() <= MAX_ENV_VARS, "Environment key count exceeds limit"); + for row in &result.rows { + ensure!(row.len() == expected.len(), "Invalid environment read row"); + if matches!(query, Query::List) { + validate_key(&row[0]).context("Invalid environment key in response")?; + } + ensure!( + row.last().unwrap().len() <= MAX_ENV_VALUE_BYTES, + "Environment read value exceeds limit" + ); + } + match query { + Query::List => { + let mut rows: Vec<_> = result.rows.iter().collect(); + rows.sort_unstable_by(|a, b| a[0].cmp(&b[0])); + let rows = rows.into_iter().map(|row| { + Ok::<_, std::convert::Infallible>(spacetimedb_lib::sats::product![row[0].as_str(), row[1].as_str()]) + }); + let table = sql::build_table( + spacetimedb_lib::sats::satn::PsqlClient::SpacetimeDB, + &result.schema, + rows, + )?; + Ok(format!("{table}\n")) + } + Query::Get(_) => { + ensure!(result.rows.len() == 1, "Environment key is absent"); + Ok(format!("{}\n", result.rows[0][0])) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn body(columns: &[&'static str], rows: Vec>) -> Vec { + serde_json::to_vec(&[spacetimedb_client_api_messages::http::SqlStmtResult { + schema: spacetimedb_lib::sats::ProductType::from_iter( + columns + .iter() + .map(|column| (*column, spacetimedb_lib::AlgebraicType::String)), + ), + rows, + total_duration_micros: 0, + stats: Default::default(), + }]) + .unwrap() + } + #[test] + fn read_only_commands_and_safe_queries() { + for command in ["set", "del", "delete", "update"] { + assert!(cli().try_get_matches_from(["env", command, "db", "KEY"]).is_err()); + } + let matches = cli() + .try_get_matches_from([ + "env", + "get", + "db", + "KEY", + "--server", + "http://127.0.0.1:9", + "--no-config", + ]) + .unwrap(); + let get = matches.subcommand_matches("get").unwrap(); + assert_eq!(get.get_one::("database").unwrap(), "db"); + assert_eq!(get.get_one::("key").unwrap(), "KEY"); + assert_eq!(Query::List.sql().unwrap(), "SELECT key, value FROM st_env"); + assert_eq!( + Query::Get("KEY".into()).sql().unwrap(), + "SELECT value FROM st_env WHERE key = 'KEY'" + ); + assert!(Query::Get("x';DELETE FROM st_env;--".into()).sql().is_err()); + } + #[test] + fn list_renders_sorted_keys_and_values_and_rejects_unexpected_columns() { + assert_eq!( + render( + &body(&["key", "value"], vec![vec!["Z", "last"], vec!["A", "first"]]), + &Query::List + ) + .unwrap() + .lines() + .enumerate() + .filter(|(index, _)| *index != 1) + .map(|(_, line)| line.split('|').map(str::trim).collect::>()) + .collect::>(), + [["key", "value"], ["\"A\"", "\"first\""], ["\"Z\"", "\"last\""]] + ); + let err = render(&body(&["value"], vec![vec!["generated-secret-sentinel"]]), &Query::List).unwrap_err(); + assert!(!format!("{err:#}").contains("generated-secret-sentinel")); + assert_eq!( + render(&body(&["value"], vec![vec![""]]), &Query::Get("A".into())).unwrap(), + "\n" + ); + assert!(render(&body(&["value"], vec![]), &Query::Get("A".into())).is_err()); + } + #[tokio::test] + async fn actual_loopback_queries_and_error_redaction() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for (query, status, response, expected) in [ + ( + Query::List, + "200 OK", + body(&["key", "value"], vec![vec!["KEY", "generated-list-sentinel"]]), + Some("generated-list-sentinel"), + ), + ( + Query::Get("KEY".into()), + "200 OK", + body(&["value"], vec![vec!["generated-read-sentinel"]]), + Some("generated-read-sentinel\n"), + ), + ( + Query::Get("KEY".into()), + "403 Forbidden", + b"generated-error-secret".to_vec(), + None, + ), + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let expected_sql = query.sql().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut input = Vec::new(); + let (head, length) = loop { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + assert!(input.len() <= 16384); + if let Some(end) = input.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = std::str::from_utf8(&input[..end]).unwrap(); + assert!(headers.starts_with("POST /v1/database/owned/sql HTTP/1.1\r\n")); + let length: usize = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length: ") + .map(str::to_owned) + }) + .unwrap() + .parse() + .unwrap(); + break (end + 4, length); + } + }; + while input.len() < head + length { + let mut chunk = [0; 1024]; + let n = stream.read(&mut chunk).await.unwrap(); + assert!(n > 0); + input.extend_from_slice(&chunk[..n]); + } + assert_eq!(&input[head..head + length], expected_sql.as_bytes()); + let headers = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", response.len()); + stream.write_all(headers.as_bytes()).await.unwrap(); + stream.write_all(&response).await.unwrap(); + stream.shutdown().await.unwrap(); + }); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + let result = fetch( + client.post(format!("http://{address}/v1/database/owned/sql")), + query.clone(), + ) + .await; + match expected { + Some(expected) if matches!(query, Query::List) => { + let output = result.unwrap(); + assert!(output.contains("key") && output.contains("value")); + assert!(output.contains("KEY") && output.contains(expected)); + } + Some(expected) => assert_eq!(result.unwrap(), expected), + None => assert!(!format!("{:#}", result.unwrap_err()).contains("generated-error-secret")), + } + tokio::time::timeout(std::time::Duration::from_secs(5), server) + .await + .unwrap() + .unwrap(); + } + } +} diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index e312bf65ea9..14b39212458 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -10,11 +10,10 @@ use spacetimedb_codegen::{ UnrealCpp, AUTO_GENERATED_PREFIX, }; use spacetimedb_lib::de::serde::DeserializeWrapper; -use spacetimedb_lib::{sats, RawModuleDef}; +use spacetimedb_lib::RawModuleDef; use spacetimedb_schema; use spacetimedb_schema::def::ModuleDef; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use crate::common_args::parse_optional_dotnet_version; use crate::spacetime_config::{ @@ -22,7 +21,7 @@ use crate::spacetime_config::{ }; use crate::tasks::csharp::dotnet_format; use crate::tasks::rust::rustfmt; -use crate::util::{resolve_sibling_binary, y_or_n}; +use crate::util::y_or_n; use crate::Config; use crate::{build, common_args}; use clap::builder::PossibleValue; @@ -750,18 +749,7 @@ impl Language { pub type ExtractDescriptions = fn(&Path) -> anyhow::Result; pub fn extract_descriptions(wasm_file: &Path) -> anyhow::Result { - let bin_path = std::env::var_os("SPACETIMEDB_SCHEMA_EXTRACTOR") - .map(PathBuf::from) - .map(Ok) - .unwrap_or_else(|| resolve_sibling_binary("spacetimedb-standalone"))?; - let child = Command::new(&bin_path) - .arg("extract-schema") - .arg(wasm_file) - .stdout(Stdio::piped()) - .spawn() - .with_context(|| format!("failed to spawn {}", bin_path.display()))?; - let sats::serde::SerdeWrapper::(module) = serde_json::from_reader(child.stdout.unwrap())?; - Ok(module.try_into()?) + crate::schema_extract::from_path(wasm_file) } #[cfg(test)] diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index af0d2e364c3..9c659a5880f 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -5,6 +5,7 @@ pub mod delete; pub mod describe; pub mod dev; pub mod dns; +pub mod env; pub mod generate; pub mod init; pub mod list; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..5162dbeca2c 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -1,3 +1,7 @@ +mod environment; +#[cfg(test)] +mod wire_tests; + use anyhow::{ensure, Context}; use clap::Arg; use clap::ArgAction::{self, Set, SetTrue}; @@ -6,8 +10,8 @@ use reqwest::{StatusCode, Url}; use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, PublishResult}; use spacetimedb_client_api_messages::name::{DatabaseNameError, PrePublishResult, PrettyPrintStyle, PublishOp}; use std::collections::HashMap; +use std::env; use std::path::PathBuf; -use std::{env, fs}; use crate::common_args::parse_optional_dotnet_version; use crate::common_args::ClearMode; @@ -87,6 +91,7 @@ pub fn build_publish_schema(command: &clap::Command) -> Result, + environment: std::collections::BTreeMap, +) -> anyhow::Result<(&'static str, Vec)> { + if module.environment_declared() { + let body = spacetimedb_client_api_messages::publish::PublishRequest { + module: bytes, + environment, + } + .encode()?; + Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) + } else { + anyhow::ensure!(environment.is_empty(), "Module does not declare environment keys"); + // Preserve older servers for ordinary modules. Explicit empty ENV + // declarations still use the envelope, expressing replacement intent. + Ok(("application/octet-stream", bytes)) + } } fn confirm_and_clear( @@ -564,7 +590,14 @@ async fn execute_publish_configs<'a>( ) .await? }; - let program_bytes = fs::read(path_to_program)?; + let program_bytes = environment::read_program(&path_to_program)?; + let module_schema = environment::inspect(&program_bytes, host_type).await?; + let environment = environment::resolve( + module_schema.environment(), + command_config.get_config_value("env"), + |key| std::env::var_os(key), + )?; + print!("{}", environment.display()); let server_address = { let url = Url::parse(&database_host)?; @@ -583,7 +616,10 @@ async fn execute_publish_configs<'a>( database_host ); - let client = reqwest::Client::new(); + // The body contains secrets. Never replay it to a redirect destination. + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; // If a name was given, ensure to percent-encode it. // We also use PUT with a name or identity, and POST otherwise. let mut builder = if let Some(name_or_identity) = name_or_identity { @@ -634,8 +670,17 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let res = builder.body(program_bytes).send().await?; - let response: PublishResult = res.json_or_error().await?; + let (content_type, payload) = publication_body(&module_schema, program_bytes, environment.values)?; + let res = builder + .header(reqwest::header::CONTENT_TYPE, content_type) + .body(payload) + .send() + .await?; + anyhow::ensure!(res.status().is_success(), "Publish failed with HTTP {}", res.status()); + let response: PublishResult = res + .json() + .await + .map_err(|_| anyhow::anyhow!("Invalid publish response"))?; match response { PublishResult::Success { domain, diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs new file mode 100644 index 00000000000..4427ae52a65 --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -0,0 +1,81 @@ +//! Resolve a complete, declared environment without consulting stored values. +use std::collections::BTreeMap; +use std::ffi::OsString; + +pub(super) use crate::schema_extract::{inspect, read_program}; +use anyhow::{ensure, Context}; +use serde_json::Value; +use spacetimedb_lib::environment::EnvironmentSchema; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Source { + Config, + Shell, +} +impl std::fmt::Display for Source { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Config => "config", + Self::Shell => "shell", + }) + } +} + +// Deliberately no Debug: values are credentials, not diagnostics. +pub(super) struct Resolved { + pub values: BTreeMap, + pub sources: BTreeMap, +} +impl Resolved { + pub fn display(&self) -> String { + use std::fmt::Write; + let mut output = String::new(); + for (name, source) in &self.sources { + let _ = writeln!(output, "Environment {name} ({source})"); + } + output + } +} + +pub(super) fn resolve( + schema: &EnvironmentSchema, + config: Option<&Value>, + mut shell: impl FnMut(&str) -> Option, +) -> anyhow::Result { + let mut resolved = Resolved { + values: BTreeMap::new(), + sources: BTreeMap::new(), + }; + if let Some(config) = config { + let config = config.as_object().context("Environment config must be an object")?; + for (name, value) in config { + ensure!( + schema.get(name).is_some(), + "Environment key {name:?}: key is not declared" + ); + let value = match value { + Value::String(value) => value.clone(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + _ => anyhow::bail!("Environment key {name:?}: config input must be a string, boolean or JSON number"), + }; + resolved.values.insert(name.clone(), value); + resolved.sources.insert(name.clone(), Source::Config); + } + } + // Lookup only the new artifact's declared names, never enumerate ambient values. + for declaration in schema.declarations() { + if let Some(value) = shell(&declaration.name) { + let value = value + .into_string() + .map_err(|_| anyhow::anyhow!("Environment key {:?}: shell input must be UTF-8", declaration.name))?; + resolved.values.insert(declaration.name.clone(), value); + resolved.sources.insert(declaration.name.clone(), Source::Shell); + } + } + schema.validate_values(&resolved.values)?; + Ok(resolved) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs new file mode 100644 index 00000000000..7ed2bca6f2e --- /dev/null +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -0,0 +1,180 @@ +use super::*; +use spacetimedb_lib::environment::{EnvironmentConstraint as Constraint, EnvironmentDeclaration as Declaration}; +use std::{path::PathBuf, time::Duration}; + +fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + Declaration { + name: "A".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "B".into(), + constraint: Constraint::OneOf(vec!["true".into(), "false".into()]), + optional: false, + }, + Declaration { + name: "C".into(), + constraint: Constraint::AnyString, + optional: false, + }, + Declaration { + name: "OPTIONAL".into(), + constraint: Constraint::AnyString, + optional: true, + }, + ]) + .unwrap() +} + +#[test] +fn declared_shell_overrides_are_complete_and_redacted() { + let mut checked = Vec::new(); + let resolved = resolve( + &schema(), + Some(&serde_json::json!({"A":"config-sentinel","B":false})), + |name| { + checked.push(name.to_owned()); + match name { + "C" => Some("shell-sentinel".into()), + "A" => Some("".into()), + _ => None, + } + }, + ) + .unwrap(); + assert_eq!( + resolved.values, + BTreeMap::from([ + ("A".into(), "".into()), + ("B".into(), "false".into()), + ("C".into(), "shell-sentinel".into()) + ]) + ); + assert_eq!(checked, vec!["A", "B", "C", "OPTIONAL"]); + assert_eq!( + resolved.display(), + "Environment A (shell)\nEnvironment B (config)\nEnvironment C (shell)\n" + ); + assert!(!resolved.display().contains("sentinel")); + // No declaration means no ambient lookup, including PATH or credentials. + let empty = resolve(&EnvironmentSchema::default(), None, |_| panic!("ambient access")).unwrap(); + assert!(empty.values.is_empty()); +} + +#[test] +fn missing_required_never_reuses_old_values_and_optional_disappears() { + let config = serde_json::json!({"A":"first","B":true,"C":"first","OPTIONAL":"old"}); + let first = resolve(&schema(), Some(&config), |_| None).unwrap(); + assert!(first.values.contains_key("OPTIONAL")); + let second = resolve( + &schema(), + Some(&serde_json::json!({"A":"next","B":false,"C":"next"})), + |_| None, + ) + .unwrap(); + assert!(!second.values.contains_key("OPTIONAL")); + let error = resolve(&schema(), Some(&serde_json::json!({"A":"first","B":true})), |_| None) + .err() + .unwrap(); + assert!(error.to_string().contains('C')); +} + +#[test] +fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { + for input in [Value::Null, serde_json::json!([]), serde_json::json!({})] { + let config = serde_json::json!({"A":input,"B":true,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |_| None).err().unwrap(); + assert!(!format!("{error:#}").contains("private-sentinel")); + } + let config = serde_json::json!({"A":"private-sentinel","B":false,"C":"private-sentinel"}); + let error = resolve(&schema(), Some(&config), |name| { + (name == "B").then(|| "invalid-shell-secret".into()) + }) + .err() + .unwrap(); + let error = format!("{error:#}"); + assert!(error.contains('B')); + assert!(!error.contains("invalid-shell-secret") && !error.contains("private-sentinel")); + let error = resolve(&schema(), Some(&serde_json::json!({"UNDECLARED":"secret"})), |_| { + panic!("must fail first") + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UNDECLARED")); +} + +#[test] +fn env_layer_selector_is_not_a_value_source() { + let command = super::super::cli(); + let args = command + .clone() + .try_get_matches_from(["publish", "db", "--env", "prod"]) + .unwrap(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let config = crate::spacetime_config::CommandConfig::new( + &schema, + std::collections::HashMap::from([("env".into(), serde_json::json!({"A":"from-config"}))]), + &args, + ) + .unwrap(); + assert_eq!(config.get_config_value("env").unwrap()["A"], "from-config"); + assert!(!config.is_from_cli("env")); +} + +#[test] +fn number_boolean_and_empty_string_conversion_has_no_float_rounding() { + let schema = EnvironmentSchema::new( + ["NUMBER", "BOOL", "EMPTY"] + .map(|name| Declaration { + name: name.into(), + constraint: Constraint::AnyString, + optional: false, + }) + .to_vec(), + ) + .unwrap(); + let config = serde_json::from_str(r#"{"NUMBER":9007199254740993123456789,"BOOL":false,"EMPTY":""}"#).unwrap(); + let resolved = resolve(&schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values["NUMBER"], "9007199254740993123456789"); + assert_eq!(resolved.values["BOOL"], "false"); + assert_eq!(resolved.values["EMPTY"], ""); +} + +#[cfg(unix)] +#[test] +fn non_utf8_declared_shell_value_is_rejected_without_bytes() { + use std::os::unix::ffi::OsStringExt; + let error = resolve(&schema(), None, |name| { + (name == "A").then(|| OsString::from_vec(vec![0xff, 0xfe])) + }) + .err() + .unwrap(); + assert!(error.to_string().contains("UTF-8")); +} + +#[tokio::test] +#[ignore = "requires explicit locally built ENV-aware standalone and declared Wasm fixture paths"] +async fn actual_precompiled_declarations_are_inspected_without_server_or_values() { + let extractor = + PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_EXTRACTOR").expect("explicit inspector path")); + let module = PathBuf::from(std::env::var_os("SPACETIMEDB_ENV_CLI_TEST_MODULE").expect("explicit module path")); + assert!(extractor.is_absolute() && module.is_absolute()); + let program = read_program(&module).unwrap(); + let inspected = crate::schema_extract::inspect_with(extractor, program, "Wasm".into(), Duration::from_secs(60)) + .await + .unwrap(); + let schema = inspected.environment(); + assert!(inspected.environment_declared()); + assert!(!schema.get("REQUIRED").unwrap().optional); + assert_eq!( + schema.get("MODE").unwrap().constraint, + Constraint::OneOf(vec!["other".into(), "ready".into()]) + ); + let config = serde_json::json!({"REQUIRED":"generated-local-inspection-sentinel","MODE":"ready"}); + let resolved = resolve(schema, Some(&config), |_| None).unwrap(); + assert_eq!(resolved.values.len(), 2); + assert!(!resolved.display().contains("generated-local-inspection-sentinel")); + assert!(resolve(schema, None, |_| None).is_err()); +} diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs new file mode 100644 index 00000000000..70781417800 --- /dev/null +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -0,0 +1,56 @@ +use super::*; +use spacetimedb_lib::{ + db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}, + RawModuleDef, +}; +use spacetimedb_schema::def::ModuleDef; +use std::collections::BTreeMap; + +fn schema(declared: bool) -> ModuleDef { + let mut sections = vec![RawModuleDefV10Section::Typespace(Default::default())]; + if declared { + sections.push(RawModuleDefV10Section::Environment(vec![])); + } + ModuleDef::try_from(RawModuleDef::V10(RawModuleDefV10 { sections })).unwrap() +} + +#[test] +fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { + let bytes = b"exact selected module bytes\0\xff".to_vec(); + let (kind, body) = publication_body(&schema(false), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, "application/octet-stream"); + assert_eq!(body, bytes); + let (kind, body) = publication_body(&schema(true), bytes.clone(), BTreeMap::new()).unwrap(); + assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); + let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); + assert_eq!(envelope.module, bytes); + assert!(envelope.environment.is_empty()); + let error = publication_body( + &schema(false), + bytes, + BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), + ) + .unwrap_err(); + assert!(!format!("{error:#}").contains("secret-sentinel")); +} + +#[test] +fn short_help_is_concise_and_long_help_explains_environment_replacement() { + let short = cli().render_help().to_string(); + let long = cli() + .render_long_help() + .to_string() + .split_whitespace() + .collect::>() + .join(" "); + assert!(!short.contains("Every publish replaces")); + assert!(short.contains("spacetime help publish")); + for text in [ + "Every publish replaces the complete declared environment", + "including empty strings", + "Optional values omitted", + "--env selects config file layers", + ] { + assert!(long.contains(text), "missing long-help guidance: {text}"); + } +} diff --git a/crates/cli/src/subcommands/sql.rs b/crates/cli/src/subcommands/sql.rs index d0a75e17d76..5b782d7106b 100644 --- a/crates/cli/src/subcommands/sql.rs +++ b/crates/cli/src/subcommands/sql.rs @@ -287,7 +287,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } /// Generates a [`tabled::Table`] from a schema and rows, using the style of a psql table. -fn build_table( +pub(super) fn build_table( client: PsqlClient, schema: &ProductType, rows: impl Iterator>, diff --git a/crates/client-api-messages/src/lib.rs b/crates/client-api-messages/src/lib.rs index bf2f1dc9ca5..67b58de659f 100644 --- a/crates/client-api-messages/src/lib.rs +++ b/crates/client-api-messages/src/lib.rs @@ -4,3 +4,5 @@ pub mod energy; pub mod http; pub mod name; pub mod websocket; + +pub mod publish; diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs new file mode 100644 index 00000000000..fea29732139 --- /dev/null +++ b/crates/client-api-messages/src/publish.rs @@ -0,0 +1,117 @@ +//! Complete publish input. Environment values travel only in the request body. +use serde::{Deserialize, Deserializer, Serialize}; +use serde_with::{base64::Base64, serde_as}; +use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; +use std::collections::BTreeMap; + +pub const CONTENT_TYPE: &str = "application/vnd.spacetimedb.publish+json"; +pub const MAX_MODULE_BYTES: usize = 128 * 1024 * 1024; +/// Includes base64 module expansion and worst-case JSON escaping of configuration. +pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; + +/// Values deliberately have no Debug representation. Omission is an empty map, +/// including for a publish of an unchanged module. +#[serde_as] +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + #[serde_as(as = "Base64")] + pub module: Vec, + #[serde(default, deserialize_with = "deserialize_environment")] + pub environment: BTreeMap, +} + +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum PublishRequestError { + #[error("invalid publish request body")] + Invalid, + #[error("publish request exceeds size limit")] + TooLarge, +} + +impl PublishRequest { + pub fn decode(body: &[u8]) -> Result { + if body.len() > MAX_REQUEST_BYTES { + return Err(PublishRequestError::TooLarge); + } + // Never expose serde's error text: it can quote a supplied secret. + let request: Self = serde_json::from_slice(body).map_err(|_| PublishRequestError::Invalid)?; + request.validate()?; + Ok(request) + } + + pub fn encode(&self) -> Result, PublishRequestError> { + self.validate()?; + serde_json::to_vec(self).map_err(|_| PublishRequestError::Invalid) + } + + fn validate(&self) -> Result<(), PublishRequestError> { + if self.module.len() > MAX_MODULE_BYTES || self.environment.len() > MAX_ENV_VARS { + return Err(PublishRequestError::TooLarge); + } + for (key, value) in &self.environment { + validate_key(key).map_err(|_| PublishRequestError::Invalid)?; + validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; + } + Ok(()) + } +} + +fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = BTreeMap; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a complete map of environment strings") + } + fn visit_map>(self, mut map: A) -> Result { + use serde::de::Error; + let mut values = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { + return Err(A::Error::custom("invalid environment keys")); + } + let value = map.next_value::()?; + if validate_value(&value).is_err() { + return Err(A::Error::custom("environment value exceeds size limit")); + } + values.insert(key, value); + } + Ok(values) + } + } + de.deserialize_map(Visitor) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn roundtrip_and_omission_preserve_complete_string_input() { + let request = PublishRequest { + module: vec![0, 1, 255], + environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "ι›ͺ\0false".into())]), + }; + let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); + assert_eq!(decoded.module, request.module); + assert_eq!(decoded.environment, request.environment); + assert!(PublishRequest::decode(br#"{"module":""}"#) + .unwrap() + .environment + .is_empty()); + } + #[test] + fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { + for body in [ + r#"{"module":"","environment":{"KEY":true}}"#, + r#"{"module":"","environment":{"KEY":null}}"#, + r#"{"module":"","environment":{"KEY":"first","KEY":"secret-marker"}}"#, + r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#, + r#"{"module":"secret-marker"}"#, + r#"{"module":"","unknown":"secret-marker"}"#, + ] { + let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + } +} diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index e8999e1e617..7d3175e751b 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -146,7 +146,8 @@ impl Host { .await .map_err(|_| (StatusCode::NOT_FOUND, "module not found".to_string()))?; - tracing::debug!(sql = body); + // Environment SQL contains values; routine request logs must omit them. + tracing::debug!(sql_bytes = body.len(), "executing SQL"); let mut header = vec![]; let sql_start = std::time::Instant::now(); let sql_span = tracing::trace_span!("execute_sql", total_duration = tracing::field::Empty,); @@ -164,8 +165,8 @@ impl Host { ) .await .map_err(|e| { - // TODO: Review log level after user SQL errors can be distinguished from internal database failures. - log::warn!("{e}"); + // Parser diagnostics can quote values. Return them only to the caller. + log::debug!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; @@ -205,6 +206,26 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + pub async fn update_with_environment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_environment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + environment, + ) + .await + } } /// Parameters for publishing a database. /// @@ -214,6 +235,8 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, + /// Complete publish input, never persisted in the public Database record. + pub environment: std::collections::BTreeMap, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -231,6 +254,7 @@ pub struct DatabaseDef { pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, + pub environment: std::collections::BTreeMap, pub num_replicas: Option, pub host_type: Option, } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index f170f9b290e..5d98a065976 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,3 +1,6 @@ +mod publish_environment; +use publish_environment::{ModuleBody, PublishBody}; + use std::borrow::Cow; use std::future::Future; use std::num::NonZeroU8; @@ -835,7 +838,10 @@ pub async fn reset( host_type, }): Query, Extension(auth): Extension, - program_bytes: Option, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -856,6 +862,7 @@ pub async fn reset( DatabaseResetDef { database_identity, program_bytes, + environment, num_replicas, host_type: Some(host_type), }, @@ -933,7 +940,10 @@ pub async fn publish( update_confirmation_timeout: confirmation_timeout, }): Query, Extension(auth): Extension, - program_bytes: Bytes, + PublishBody { + program_bytes, + environment, + }: PublishBody, ) -> axum::response::Result> { // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. @@ -963,13 +973,17 @@ pub async fn publish( host_type, }), Extension(auth), - Some(program_bytes), + PublishBody { + program_bytes, + environment, + }, ) .await; } } } + let program_bytes = program_bytes.unwrap_or_default(); let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; let maybe_parent_database_identity = match parent.as_ref() { None => None, @@ -1033,6 +1047,7 @@ pub async fn publish( DatabaseDef { database_identity, program_bytes, + environment, num_replicas, host_type, parent, @@ -1217,7 +1232,7 @@ pub async fn pre_publish Extension(ResolvedDatabase(database)): Extension, Query(PrePublishQueryParams { style, host_type }): Query, Extension(auth): Extension, - program_bytes: Bytes, + ModuleBody(program_bytes): ModuleBody, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -1236,6 +1251,7 @@ pub async fn pre_publish DatabaseDef { database_identity, program_bytes, + environment: Default::default(), num_replicas: None, host_type, parent: None, diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs new file mode 100644 index 00000000000..0287d226688 --- /dev/null +++ b/crates/client-api/src/routes/database/publish_environment.rs @@ -0,0 +1,139 @@ +//! Bounded publish extraction. Neither errors nor Debug output retain configuration values. +use axum::body::{to_bytes, Bytes}; +use axum::extract::{FromRequest, Request}; +use axum::response::{IntoResponse, Response}; +use http::{header, StatusCode}; +use spacetimedb_client_api_messages::publish::{PublishRequest, CONTENT_TYPE, MAX_MODULE_BYTES, MAX_REQUEST_BYTES}; +use std::collections::BTreeMap; + +pub struct PublishBody { + pub program_bytes: Option, + pub environment: BTreeMap, +} + +async fn bounded_body(request: Request, limit: usize) -> Result { + if request + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|len| len > limit as u64) + { + return Err((StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()); + } + to_bytes(request.into_body(), limit) + .await + .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()) +} + +#[async_trait::async_trait] +impl FromRequest for PublishBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let envelope = request + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .unwrap_or_default() + .trim() + .eq_ignore_ascii_case(CONTENT_TYPE) + }); + let bytes = bounded_body(request, if envelope { MAX_REQUEST_BYTES } else { MAX_MODULE_BYTES }).await?; + if envelope { + let request = PublishRequest::decode(&bytes) + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; + Ok(Self { + program_bytes: (!request.module.is_empty()).then_some(request.module.into()), + environment: request.environment, + }) + } else { + // An absent reset body retains the program, but never retains env values. + Ok(Self { + program_bytes: (!bytes.is_empty()).then_some(bytes), + environment: BTreeMap::new(), + }) + } + } +} + +pub struct ModuleBody(pub Bytes); + +#[async_trait::async_trait] +impl FromRequest for ModuleBody { + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + bounded_body(request, MAX_MODULE_BYTES).await.map(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + + #[tokio::test] + async fn legacy_envelope_and_empty_reset_have_complete_input_semantics() { + let legacy = PublishBody::from_request(Request::new(Body::from("module")), &()) + .await + .unwrap(); + assert_eq!(legacy.program_bytes.unwrap(), "module"); + assert!(legacy.environment.is_empty()); + let empty = PublishBody::from_request(Request::new(Body::empty()), &()) + .await + .unwrap(); + assert!(empty.program_bytes.is_none()); + assert!(empty.environment.is_empty()); + let input = PublishRequest { + module: vec![1, 2, 3], + environment: BTreeMap::from([("TOKEN".into(), "ι›ͺ\0".into())]), + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(input.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert_eq!(extracted.environment, input.environment); + assert_eq!(extracted.program_bytes.unwrap(), input.module); + let reset = PublishRequest { + module: vec![], + environment: input.environment, + }; + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(reset.encode().unwrap())) + .unwrap(); + let extracted = PublishBody::from_request(request, &()).await.unwrap(); + assert!(extracted.program_bytes.is_none()); + assert_eq!(extracted.environment, reset.environment); + } + + #[tokio::test] + async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { + let stream = futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from_static(b"12345")), + Ok(Bytes::from_static(b"67890")), + ]); + let error = bounded_body(Request::new(Body::from_stream(stream)), 8) + .await + .unwrap_err(); + assert_eq!(error.into_response().status(), StatusCode::PAYLOAD_TOO_LARGE); + let request = Request::builder() + .header(header::CONTENT_TYPE, CONTENT_TYPE) + .body(Body::from(r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#)) + .unwrap(); + let error = PublishBody::from_request(request, &()) + .await + .err() + .unwrap() + .into_response(); + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(error.into_body(), 1024).await.unwrap(); + assert!(!String::from_utf8_lossy(&body).contains("secret-marker")); + } +} diff --git a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap index ab899e7f663..21fe014ee8d 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_csharp.snap @@ -69,6 +69,84 @@ namespace SpacetimeDB } } ''' +"Procedures/ReadEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteProcedures : RemoteBase + { + public void ReadEnvironment(string key, ProcedureCallback callback) + { + // Convert the clean callback to the wrapper callback + InternalReadEnvironment(key, (ctx, result) => { + if (result.IsSuccess && result.Value != null) + { + callback(ctx, ProcedureCallbackResult.Success(result.Value.Value)); + } + else + { + callback(ctx, ProcedureCallbackResult.Failure(result.Error!)); + } + }); + } + + private void InternalReadEnvironment(string key, ProcedureCallback callback) + { + conn.InternalCallProcedure(new Procedure.ReadEnvironmentArgs(key), callback); + } + + } + + public abstract partial class Procedure + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironment + { + [DataMember(Name = "Value")] + public string? Value; + + public ReadEnvironment(string? Value) + { + this.Value = Value; + } + + public ReadEnvironment() + { + } + } + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ReadEnvironmentArgs : Procedure, IProcedureArgs + { + [DataMember(Name = "key")] + public string Key; + + public ReadEnvironmentArgs(string Key) + { + this.Key = Key; + } + + public ReadEnvironmentArgs() + { + this.Key = ""; + } + + string IProcedureArgs.ProcedureName => "read_environment"; + } + + } +} +''' "Procedures/ReturnValue.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -683,6 +761,82 @@ namespace SpacetimeDB } } ''' +"Reducers/ExpectEnvironment.g.cs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void ExpectEnvironmentHandler(ReducerEventContext ctx, string key, string? expected); + public event ExpectEnvironmentHandler? OnExpectEnvironment; + + public void ExpectEnvironment(string key, string? expected) + { + conn.InternalCallReducer(new Reducer.ExpectEnvironment(key, expected)); + } + + public bool InvokeExpectEnvironment(ReducerEventContext ctx, Reducer.ExpectEnvironment args) + { + if (OnExpectEnvironment == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch(ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnExpectEnvironment( + ctx, + args.Key, + args.Expected + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class ExpectEnvironment : Reducer, IReducerArgs + { + [DataMember(Name = "key")] + public string Key; + [DataMember(Name = "expected")] + public string? Expected; + + public ExpectEnvironment( + string Key, + string? Expected + ) + { + this.Key = Key; + this.Expected = Expected; + } + + public ExpectEnvironment() + { + this.Key = ""; + } + + string IReducerArgs.ReducerName => "expect_environment"; + } + } +} +''' "Reducers/ListOverAge.g.cs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -1681,6 +1835,7 @@ namespace SpacetimeDB Reducer.AssertCallerIdentityIsModuleIdentity args => Reducers.InvokeAssertCallerIdentityIsModuleIdentity(eventContext, args), Reducer.DeletePlayer args => Reducers.InvokeDeletePlayer(eventContext, args), Reducer.DeletePlayersByName args => Reducers.InvokeDeletePlayersByName(eventContext, args), + Reducer.ExpectEnvironment args => Reducers.InvokeExpectEnvironment(eventContext, args), Reducer.ListOverAge args => Reducers.InvokeListOverAge(eventContext, args), Reducer.LogModuleIdentity args => Reducers.InvokeLogModuleIdentity(eventContext, args), Reducer.QueryPrivate args => Reducers.InvokeQueryPrivate(eventContext, args), diff --git a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap index 1cda8c8d75b..af115ee46b5 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_rust.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_rust.snap @@ -488,6 +488,88 @@ impl delete_players_by_name for super::RemoteReducers { } } +''' +"expect_environment_reducer.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub(super) struct ExpectEnvironmentArgs { + pub key: String, + pub expected: Option::, +} + +impl From for super::Reducer { + fn from(args: ExpectEnvironmentArgs) -> Self { + Self::ExpectEnvironment { + key: args.key, + expected: args.expected, +} +} +} + +impl __sdk::InModule for ExpectEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the reducer `expect_environment`. +/// +/// Implemented for [`super::RemoteReducers`]. +pub trait expect_environment { + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and this method provides no way to listen for its completion status. + /// /// Use [`expect_environment:expect_environment_then`] to run a callback after the reducer completes. + fn expect_environment(&self, key: String, +expected: Option::, +) -> __sdk::Result<()> { + self.expect_environment_then(key, expected, |_, _| {}) + } + + /// Request that the remote module invoke the reducer `expect_environment` to run as soon as possible, + /// registering `callback` to run when we are notified that the reducer completed. + /// + /// This method returns immediately, and errors only if we are unable to send the request. + /// The reducer will run asynchronously in the future, + /// and its status can be observed with the `callback`. + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()>; +} + +impl expect_environment for super::RemoteReducers { + fn expect_environment_then( + &self, + key: String, +expected: Option::, + + callback: impl FnOnce(&super::ReducerEventContext, Result, __sdk::InternalError>) + + Send + + 'static, + ) -> __sdk::Result<()> { + self.imp.invoke_reducer_with_callback(ExpectEnvironmentArgs { key, expected, }, callback) + } +} + ''' "foobar_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE @@ -1116,6 +1198,7 @@ pub mod add_private_reducer; pub mod assert_caller_identity_is_module_identity_reducer; pub mod delete_player_reducer; pub mod delete_players_by_name_reducer; +pub mod expect_environment_reducer; pub mod list_over_age_reducer; pub mod log_module_identity_reducer; pub mod query_private_reducer; @@ -1129,6 +1212,7 @@ pub mod test_d_table; pub mod test_f_table; pub mod my_player_table; pub mod get_my_schema_via_http_procedure; +pub mod read_environment_procedure; pub mod return_value_procedure; pub mod sleep_one_second_procedure; pub mod with_tx_procedure; @@ -1163,6 +1247,7 @@ pub use add_private_reducer::add_private; pub use assert_caller_identity_is_module_identity_reducer::assert_caller_identity_is_module_identity; pub use delete_player_reducer::delete_player; pub use delete_players_by_name_reducer::delete_players_by_name; +pub use expect_environment_reducer::expect_environment; pub use list_over_age_reducer::list_over_age; pub use log_module_identity_reducer::log_module_identity; pub use query_private_reducer::query_private; @@ -1170,6 +1255,7 @@ pub use say_hello_reducer::say_hello; pub use test_reducer::test; pub use test_btree_index_args_reducer::test_btree_index_args; pub use get_my_schema_via_http_procedure::get_my_schema_via_http; +pub use read_environment_procedure::read_environment; pub use return_value_procedure::return_value; pub use sleep_one_second_procedure::sleep_one_second; pub use with_tx_procedure::with_tx; @@ -1198,6 +1284,10 @@ pub enum Reducer { } , DeletePlayersByName { name: String, +} , + ExpectEnvironment { + key: String, + expected: Option::, } , ListOverAge { age: u8, @@ -1228,6 +1318,7 @@ impl __sdk::Reducer for Reducer { Reducer::AssertCallerIdentityIsModuleIdentity => "assert_caller_identity_is_module_identity", Reducer::DeletePlayer { .. } => "delete_player", Reducer::DeletePlayersByName { .. } => "delete_players_by_name", + Reducer::ExpectEnvironment { .. } => "expect_environment", Reducer::ListOverAge { .. } => "list_over_age", Reducer::LogModuleIdentity => "log_module_identity", Reducer::QueryPrivate => "query_private", @@ -1268,6 +1359,13 @@ Reducer::DeletePlayer{ name, } => __sats::bsatn::to_vec(&delete_players_by_name_reducer::DeletePlayersByNameArgs { name: name.clone(), +}), + Reducer::ExpectEnvironment{ + key, + expected, +} => __sats::bsatn::to_vec(&expect_environment_reducer::ExpectEnvironmentArgs { + key: key.clone(), + expected: expected.clone(), }), Reducer::ListOverAge{ age, @@ -3296,6 +3394,64 @@ impl query_private for super::RemoteReducers { } } +''' +"read_environment_procedure.rs" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{ + self as __sdk, + __lib, + __sats, + __ws, +}; + + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] + struct ReadEnvironmentArgs { + pub key: String, +} + + +impl __sdk::InModule for ReadEnvironmentArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `read_environment`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait read_environment { + fn read_environment(&self, key: String, +) { + self.read_environment_then(key, |_, _| {}); + } + + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ); +} + +impl read_environment for super::RemoteProcedures { + fn read_environment_then( + &self, + key: String, + + __callback: impl FnOnce(&super::ProcedureEventContext, Result, __sdk::InternalError>) + Send + 'static, + ) { + self.imp.invoke_procedure_with_callback::<_, Option::>( + "read_environment", + ReadEnvironmentArgs { key, }, + __callback, + ); + } +} + ''' "remove_table_type.rs" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE diff --git a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap index 9ca1e9926a5..addaeab2185 100644 --- a/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap +++ b/crates/codegen/tests/snapshots/codegen__codegen_typescript.snap @@ -103,6 +103,24 @@ export default { name: __t.string(), }; ''' +"expect_environment_reducer.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + key: __t.string(), + expected: __t.option(__t.string()), +}; +''' "get_my_schema_via_http_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -162,6 +180,7 @@ import AddPrivateReducer from "./add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "./assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "./delete_player_reducer"; import DeletePlayersByNameReducer from "./delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "./expect_environment_reducer"; import ListOverAgeReducer from "./list_over_age_reducer"; import LogModuleIdentityReducer from "./log_module_identity_reducer"; import QueryPrivateReducer from "./query_private_reducer"; @@ -171,6 +190,7 @@ import TestBtreeIndexArgsReducer from "./test_btree_index_args_reducer"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "./get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "./read_environment_procedure"; import * as ReturnValueProcedure from "./return_value_procedure"; import * as SleepOneSecondProcedure from "./sleep_one_second_procedure"; import * as WithTxProcedure from "./with_tx_procedure"; @@ -270,6 +290,7 @@ const reducersSchema = __reducers( __reducerSchema("assert_caller_identity_is_module_identity", AssertCallerIdentityIsModuleIdentityReducer), __reducerSchema("delete_player", DeletePlayerReducer), __reducerSchema("delete_players_by_name", DeletePlayersByNameReducer), + __reducerSchema("expect_environment", ExpectEnvironmentReducer), __reducerSchema("list_over_age", ListOverAgeReducer), __reducerSchema("log_module_identity", LogModuleIdentityReducer), __reducerSchema("query_private", QueryPrivateReducer), @@ -281,6 +302,7 @@ const reducersSchema = __reducers( /** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ const proceduresSchema = __procedures( __procedureSchema("get_my_schema_via_http", GetMySchemaViaHttpProcedure.params, GetMySchemaViaHttpProcedure.returnType), + __procedureSchema("read_environment", ReadEnvironmentProcedure.params, ReadEnvironmentProcedure.returnType), __procedureSchema("return_value", ReturnValueProcedure.params, ReturnValueProcedure.returnType), __procedureSchema("sleep_one_second", SleepOneSecondProcedure.params, SleepOneSecondProcedure.returnType), __procedureSchema("with_tx", WithTxProcedure.params, WithTxProcedure.returnType), @@ -531,6 +553,23 @@ import { export default {}; ''' +"read_environment_procedure.ts" = ''' +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const params = { + key: __t.string(), +}; +export const returnType = __t.option(__t.string())''' "return_value_procedure.ts" = ''' // THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE // WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. @@ -827,12 +866,15 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all procedure arg schemas import * as GetMySchemaViaHttpProcedure from "../get_my_schema_via_http_procedure"; +import * as ReadEnvironmentProcedure from "../read_environment_procedure"; import * as ReturnValueProcedure from "../return_value_procedure"; import * as SleepOneSecondProcedure from "../sleep_one_second_procedure"; import * as WithTxProcedure from "../with_tx_procedure"; export type GetMySchemaViaHttpArgs = __Infer; export type GetMySchemaViaHttpResult = __Infer; +export type ReadEnvironmentArgs = __Infer; +export type ReadEnvironmentResult = __Infer; export type ReturnValueArgs = __Infer; export type ReturnValueResult = __Infer; export type SleepOneSecondArgs = __Infer; @@ -856,6 +898,7 @@ import AddPrivateReducer from "../add_private_reducer"; import AssertCallerIdentityIsModuleIdentityReducer from "../assert_caller_identity_is_module_identity_reducer"; import DeletePlayerReducer from "../delete_player_reducer"; import DeletePlayersByNameReducer from "../delete_players_by_name_reducer"; +import ExpectEnvironmentReducer from "../expect_environment_reducer"; import ListOverAgeReducer from "../list_over_age_reducer"; import LogModuleIdentityReducer from "../log_module_identity_reducer"; import QueryPrivateReducer from "../query_private_reducer"; @@ -869,6 +912,7 @@ export type AddPrivateParams = __Infer; export type AssertCallerIdentityIsModuleIdentityParams = __Infer; export type DeletePlayerParams = __Infer; export type DeletePlayersByNameParams = __Infer; +export type ExpectEnvironmentParams = __Infer; export type ListOverAgeParams = __Infer; export type LogModuleIdentityParams = __Infer; export type QueryPrivateParams = __Infer; diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs new file mode 100644 index 00000000000..10718637d13 --- /dev/null +++ b/crates/core/src/db/environment.rs @@ -0,0 +1,166 @@ +//! Dedicated access to the private environment store. +//! +//! Publishing replaces the complete environment inside the program transaction. +//! Helpers never acquire a second transaction or expose individual mutation APIs. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use spacetimedb_lib::environment::{ + validate_key, EnvironmentSchema, EnvironmentSchemaError, EnvironmentValidationError, +}; +use spacetimedb_sats::AlgebraicValue; +use std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error(transparent)] + Validation(#[from] EnvironmentValidationError), + #[error(transparent)] + Schema(#[from] EnvironmentSchemaError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Read from exactly the caller's snapshot, preserving missing versus empty. +pub fn get(state: &impl StateView, key: &str) -> Result, EnvironmentError> { + validate_key(key)?; + state + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| Ok(StEnvRow::try_from(row)?.value)) + .transpose() +} + +pub fn snapshot(state: &impl StateView) -> Result, EnvironmentError> { + state + .iter(ST_ENV_ID)? + .map(|row| { + let row = StEnvRow::try_from(row)?; + Ok((row.key, row.value)) + }) + .collect() +} + +/// Apply a complete publish configuration in the caller's program transaction. +/// Validate every input before modifying any row, even if the caller chooses to +/// recover from a validation error and commit other transaction work. +pub fn replace( + db: &RelationalDB, + tx: &mut MutTx, + schema: &EnvironmentSchema, + values: &BTreeMap, +) -> Result<(), EnvironmentError> { + schema.validate_values(values)?; + let previous = snapshot(tx)?; + for (key, value) in &previous { + if values.get(key) != Some(value) { + delete(db, tx, key)?; + } + } + for (key, value) in values { + if previous.get(key) != Some(value) { + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.clone(), + value: value.clone(), + }, + )?; + } + } + Ok(()) +} + +fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { + validate_key(key)?; + let pointer = tx + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| row.pointer()); + if let Some(pointer) = pointer { + db.delete(tx, ST_ENV_ID, [pointer]); + return Ok(true); + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn schema() -> EnvironmentSchema { + EnvironmentSchema::new(vec![ + EnvironmentDeclaration { + name: "REQUIRED".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }, + EnvironmentDeclaration { + name: "OPTIONAL".into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap() + } + + #[test] + fn replacement_preserves_empty_and_nul_and_removes_omitted_values() { + let db = TestDB::in_memory().unwrap(); + let initial = BTreeMap::from([("REQUIRED".into(), "".into()), ("OPTIONAL".into(), "a\0b".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &initial)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); + let next = BTreeMap::from([("REQUIRED".into(), "new".into())]); + db.with_auto_commit(Workload::ForTests, |tx| replace(&db, tx, &schema(), &next)) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), next)); + db.with_auto_commit(Workload::ForTests, |tx| { + replace(&db, tx, &EnvironmentSchema::default(), &BTreeMap::new()) + }) + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| assert!(snapshot(tx).unwrap().is_empty())); + } + + #[test] + fn invalid_complete_input_does_not_reuse_stored_values_or_mutate() { + let db = TestDB::in_memory().unwrap(); + let initial = BTreeMap::from([("REQUIRED".into(), "old".into()), ("OPTIONAL".into(), "keep".into())]); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + replace(&db, tx, &schema(), &initial)?; + for invalid in [ + BTreeMap::new(), + BTreeMap::from([ + ("REQUIRED".into(), "new".into()), + ("UNKNOWN".into(), "secret-marker".into()), + ]), + BTreeMap::from([("REQUIRED".into(), "x".repeat(8193))]), + ] { + assert!(replace(&db, tx, &schema(), &invalid).is_err()); + assert_eq!(snapshot(tx)?, initial); + } + Ok(()) + }) + .unwrap(); + let failed_publish = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + replace( + &db, + tx, + &schema(), + &BTreeMap::from([("REQUIRED".into(), "updated".into())]), + )?; + // Simulate a later failure in the same publish transaction. + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(failed_publish.is_err()); + db.with_read_only(Workload::ForTests, |tx| assert_eq!(snapshot(tx).unwrap(), initial)); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 6b1d2f6700b..a7117db5b71 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -1,3 +1,5 @@ +pub mod environment; + pub mod persistence { pub use spacetimedb_engine::persistence::*; } diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index efee537dbbe..0f8d5621b3f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -30,6 +30,10 @@ impl From for DBError { #[derive(Error, Debug)] pub enum NodesError { + #[error("invalid environment variable name")] + InvalidEnvironmentKey, + #[error("too many outstanding byte sources for environment read")] + EnvironmentSourceLimit, #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index ff017b733c8..38a198599b6 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -25,7 +25,7 @@ use crate::worker_metrics::{ record_module_host_init_attempt, record_module_host_init_failure, record_module_host_unexpected_exit, ModuleHostInitFailureCause, WORKER_METRICS, }; -use anyhow::{anyhow, bail, Context}; +use anyhow::{bail, Context}; use async_trait::async_trait; use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; @@ -92,6 +92,14 @@ where pub type ProgramStorage = Arc; +/// Private complete configuration for a not-yet-initialized database generation. +/// Implementations must verify the exact persisted database identity, program and +/// bootstrap generation. This source is never consulted during ordinary reopen. +#[async_trait] +pub trait InitialEnvironmentSource: Send + Sync { + async fn load(&self, database: &Database) -> anyhow::Result>; +} + /// A launched module host plus any pending controldb program-bootstrap completion work. pub struct ModuleHostWithBootstrap { pub module: ModuleHost, @@ -187,6 +195,7 @@ pub struct HostController { default_config: db::Config, /// The [`ProgramStorage`] to query when instantiating a module. program_storage: ProgramStorage, + initial_environment_source: Option>, /// The [`EnergyMonitor`] used by this controller. energy_monitor: Arc, /// The [`MemoryObserver`] used by this controller. @@ -357,6 +366,7 @@ impl HostController { hosts: <_>::default(), default_config, program_storage, + initial_environment_source: None, energy_monitor, memory_observer, persistence, @@ -368,6 +378,12 @@ impl HostController { } } + /// Install the private bootstrap input source before this controller is shared. + pub fn with_initial_environment_source(mut self, source: Arc) -> Self { + self.initial_environment_source = Some(source); + self + } + /// Replace the [`ProgramStorage`] used by this controller. pub fn set_program_storage(&mut self, ps: ProgramStorage) { self.program_storage = ps; @@ -514,6 +530,16 @@ impl HostController { /// This is not necessary during hotswap publishes, /// as the automigration planner and executor accomplish the same validity checks. pub async fn check_module_validity(&self, database: Database, program: Program) -> anyhow::Result> { + self.check_module_validity_with_environment(database, program, Default::default()) + .await + } + + pub async fn check_module_validity_with_environment( + &self, + database: Database, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result> { let (program, launched) = Host::try_init_in_memory_to_check( &self.runtimes, self.page_pool.clone(), @@ -529,12 +555,20 @@ impl HostController { ) .await?; - let InitDatabaseResult { reducer, .. } = launched.module_host.init_database(program).await?; + let result = launched + .module_host + .init_database_with_environment(program, environment) + .await; + let info = launched.module_host.info.clone(); + // Validation never starts scheduled work. Release its receiver before + // waiting for scheduler closure, including when initialization failed. + drop(launched.scheduler_starter); + launched.module_host.exit().await; + let InitDatabaseResult { reducer, .. } = result?; if let Some(call_result) = reducer { Result::from(call_result)?; } - - Ok(launched.module_host.info) + Ok(info) } /// Update the [`ModuleHost`] identified by `replica_id` to the given @@ -553,6 +587,27 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_environment( + database, + host_type, + replica_id, + program_bytes, + policy, + Default::default(), + ) + .await + } + + #[tracing::instrument(level = "trace", skip_all, err)] + pub async fn update_module_host_with_environment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -605,12 +660,14 @@ impl HostController { this.energy_monitor.clone(), this.unregister_fn(replica_id, database_identity), this.db_cores.take(), + environment, ) - .await?; + .await; + // Rejected publication leaves the existing host usable. Restore it + // before propagating validation or migration failure to the caller. *guard = Some(host); - - Ok::<_, anyhow::Error>(update_result) + update_result }) .await??; @@ -657,6 +714,25 @@ impl HostController { /// and deregister it from the controller. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { + let start = Instant::now(); + if tokio::time::timeout(timeout, self.exit_module_host_and_join(replica_id)) + .await + .is_err() + { + warn!( + "replica={replica_id} shutdown timed out after {}s", + start.elapsed().as_secs_f32() + ); + } + Ok(()) + } + + /// Wait for actual module and database closure, without treating an elapsed + /// request deadline as completion. The caller must retain this future and + /// exclude new launch admission until it returns, including if its own + /// request waiter is cancelled. + #[tracing::instrument(level = "trace", skip_all)] + pub async fn exit_module_host_and_join(&self, replica_id: u64) -> Result<(), anyhow::Error> { let Some(lock) = self.hosts.lock().remove(&replica_id) else { return Ok(()); }; @@ -677,35 +753,24 @@ impl HostController { }); defer!(warn_blocked.abort()); - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); + let mut guard = lock.write_owned().await; + let Some(host) = guard.take() else { + return Ok(()); + }; + let module = host.module.borrow().clone(); + let info = module.info(); - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); + let database_identity = info.database_identity; + let table_names = info.module_def.tables().map(|t| t.name.deref()); - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); - } + // Ensure we clear the metrics even if the future is cancelled. + defer!(remove_database_gauges(&database_identity, table_names)); + info!("replica={replica_id} database={database_identity} exiting module"); + module.exit().await; + info!("replica={replica_id} database={database_identity} exiting database"); + module.relational_db().shutdown().await; + info!("replica={replica_id} database={database_identity} module host exited"); Ok(()) } @@ -1030,32 +1095,25 @@ fn repair_stale_view_backing_tables_on_launch(launched: &LaunchedModule) -> anyh /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Otherwise publish the complete environment with the module, including when +/// its program hash is unchanged. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let addr = db.database_identity(); - match stored_program_hash(db)? { - None => Err(anyhow!("database `{addr}` not yet initialized")), - Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) - } - } + let Some(stored) = stored_program_hash(db)? else { + bail!("database `{addr}` not yet initialized"); + }; + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + // Even an unchanged program publishes a complete replacement environment. + module + .update_database_with_environment(program, old_module_info, policy, environment) + .await } /// Encapsulates a database, associated module, and auxiliary state. @@ -1194,6 +1252,14 @@ impl Host { } }; let bootstrap_generation = database.bootstrap_generation; + let initial_environment = if program_needs_init { + match &host_controller.initial_environment_source { + Some(source) => source.load(&database).await?, + None => Default::default(), + } + } else { + Default::default() + }; let mut bootstrap_completion = Some(BootstrapCompletion::durable(bootstrap_generation)); let relational_db = Arc::new(db); @@ -1284,7 +1350,10 @@ impl Host { }; if program_needs_init { - let InitDatabaseResult { reducer, tx_offset } = launched.module_host.init_database(program).await?; + let InitDatabaseResult { reducer, tx_offset } = launched + .module_host + .init_database_with_environment(program, initial_environment) + .await?; if let Some(call_result) = reducer { validate_init_reducer_call_result(call_result)?; } @@ -1414,6 +1483,7 @@ impl Host { energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, core: AllocatedJobCore, + environment: std::collections::BTreeMap, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1432,8 +1502,25 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = match update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + environment, + ) + .await + { + Ok(result) => result, + Err(error) => { + // This candidate was never installed or scheduled. Close its + // receiver first so cleanup cannot wait on an unstarted actor. + drop(scheduler_starter); + module.exit().await; + return Err(error); + } + }; // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. @@ -1470,7 +1557,10 @@ impl Host { let old_module = old_watcher.borrow().clone(); old_module.exit().await; } - _ => {} + _ => { + drop(scheduler_starter); + module.exit().await; + } } Ok(update_result) @@ -1678,6 +1768,42 @@ where mod tests { use super::*; + #[tokio::test] + async fn positive_close_stays_pending_past_a_waiter_deadline_until_host_ownership_is_released() { + use crate::db::persistence::LocalPersistenceProvider; + use spacetimedb_paths::FromPathUnchecked; + let temp = tempfile::tempdir().unwrap(); + let directory = Arc::new(ServerDataDir::from_path_unchecked(temp.path().to_owned())); + let controller = HostController::new( + directory.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + HostRuntimeConfig::new(WasmConfig::default(), V8Config::default(), ModuleHttpConfig::default()), + Arc::new(|_| std::future::ready(anyhow::Ok(None))), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(directory)), + JobCores::without_pinned_cores(), + ); + let cell = Arc::new(AsyncRwLock::new(None)); + controller.hosts.lock().insert(17, cell.clone()); + let accepted_reader = cell.clone().read_owned().await; + let mut close = tokio::spawn(async move { controller.exit_module_host_and_join(17).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut close) + .await + .is_err()); + // A deadline on the observer did not complete or discard the owned close. + assert!(!close.is_finished()); + drop(accepted_reader); + tokio::time::timeout(Duration::from_secs(5), close) + .await + .unwrap() + .unwrap() + .unwrap(); + } + fn reducer_call_result(outcome: ReducerOutcome) -> ReducerCallResult { ReducerCallResult { outcome, diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 9bf807ebf7a..ac9885b83a1 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -17,6 +17,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOrRange, MutTxId}; +use spacetimedb_datastore::system_tables::{is_module_restricted_index, is_module_restricted_table}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_metrics::utils::IntGaugeExt; @@ -49,6 +50,9 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Bound by the host after validating this instance's module metadata. + environment_module: Option<(spacetimedb_lib::Hash, Arc)>, + environment_call_active: bool, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -235,6 +239,8 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + environment_module: None, + environment_call_active: false, in_anon_tx: false, procedure_last_tx_offset: None, } @@ -251,6 +257,7 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.environment_call_active = true; } /// Returns the name of the most recent reducer to be run in this environment, @@ -259,9 +266,29 @@ impl InstanceEnv { self.func_name.as_deref() } - /// Swap in a temporary function type, returning the previous one. - pub fn swap_func_type(&mut self, func_type: FuncCallType) -> FuncCallType { - mem::replace(&mut self.func_type, func_type) + pub(crate) fn bind_environment_module( + &mut self, + hash: spacetimedb_lib::Hash, + def: Arc, + ) { + self.environment_module = Some((hash, def)); + } + + pub(crate) fn finish_funcall(&mut self) { + self.environment_call_active = false; + } + + /// Nested host-dispatched view refreshes must use the view's namespace, + /// then restore the enclosing procedure's namespace and dependency tracking. + pub(crate) fn swap_func_context( + &mut self, + name: Option, + func_type: FuncCallType, + ) -> (Option, FuncCallType) { + ( + mem::replace(&mut self.func_name, name), + mem::replace(&mut self.func_type, func_type), + ) } fn get_tx(&self) -> Result + '_, GetTxError> { @@ -281,6 +308,60 @@ impl InstanceEnv { self.replica_ctx.relational_db() } + /// Read configuration using the schema of this exact module instance. + /// Missing optional reads also register a view dependency on `st_env`. + pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { + use spacetimedb_datastore::system_tables::ST_ENV_ID; + spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; + if !self.environment_call_active || self.func_name.as_ref().is_none_or(|name| name.is_namespaced()) { + return Err(DBError::Other(anyhow::anyhow!( + "environment access requires a host-dispatched root module function" + )) + .into()); + } + if let Ok(mut tx) = self.get_tx() { + tx.record_table_scan(&self.func_type, ST_ENV_ID); + return self.read_declared_environment(&*tx, key); + } + if !matches!(self.func_type, FuncCallType::Procedure) { + return Err(NodesError::NotInTransaction); + } + self.relational_db() + .with_read_only(Workload::Internal, |tx| self.read_declared_environment(tx, key)) + } + + fn read_declared_environment(&self, state: &impl StateView, key: &str) -> Result, NodesError> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let fail = |message| NodesError::from(DBError::Other(anyhow::anyhow!("{message}"))); + let (hash, module) = self + .environment_module + .as_ref() + .ok_or_else(|| fail("environment schema is not available"))?; + let declaration = module + .environment() + .get(key) + .ok_or_else(|| fail("environment key is not declared"))?; + // Check inside this same snapshot. A suspended old procedure must never + // combine its declarations with values installed for a different module. + let row = state + .iter(ST_MODULE_ID) + .map_err(DBError::from)? + .next() + .ok_or_else(|| fail("database program is not initialized"))?; + let current_hash = spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash) + .map_err(DBError::from)?; + if current_hash != *hash { + return Err(fail("module was replaced while this function was running")); + } + let value = crate::db::environment::get(state, key).map_err(|error| DBError::Other(error.into()))?; + if value.is_none() && !declaration.optional { + return Err(fail( + "required environment value is missing from the published configuration", + )); + } + Ok(value) + } + pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { let tx = &mut *self.get_tx()?; Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) @@ -355,9 +436,19 @@ impl InstanceEnv { count } + /// Environment values are reachable only through their dedicated host interface. + fn require_module_table(table_id: TableId) -> Result<(), NodesError> { + if is_module_restricted_table(table_id) { + Err(NodesError::TableNotFound) + } else { + Ok(()) + } + } + pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, insert_flags) = stdb .insert(tx, table_id, buffer) @@ -436,6 +527,7 @@ impl InstanceEnv { pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, update_flags) = stdb .update(tx, table_id, index_id, buffer) @@ -479,6 +571,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, _, iter) = stdb.index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = iter.map(|row_ref| row_ref.pointer()).collect::>(); @@ -499,6 +592,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, iter) = stdb.index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = match iter { IndexScanPointOrRange::Point(_, iter) => iter.map(|row_ref| row_ref.pointer()).collect(), @@ -540,6 +634,7 @@ impl InstanceEnv { pub fn datastore_delete_all_by_eq_bsatn(&self, table_id: TableId, relation: &[u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Track the number of bytes coming from the caller tx.metrics.bytes_scanned += relation.len(); @@ -563,6 +658,7 @@ impl InstanceEnv { pub fn clear(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let rows_deleted = stdb.clear_table(tx, table_id).map_err(NodesError::from)?; @@ -584,6 +680,7 @@ impl InstanceEnv { // Query the table id from the name. stdb.table_id_from_name_mut(tx, table_name)? + .filter(|id| !is_module_restricted_table(*id)) .ok_or(NodesError::TableNotFound) } @@ -598,6 +695,7 @@ impl InstanceEnv { // Query the index id from the name. stdb.index_id_from_name_mut(tx, index_name)? + .filter(|id| !is_module_restricted_index(*id)) .ok_or(NodesError::IndexNotFound) } @@ -609,6 +707,7 @@ impl InstanceEnv { pub fn datastore_table_row_count(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Query the row count for id. stdb.table_row_count_mut(tx, table_id) @@ -625,6 +724,7 @@ impl InstanceEnv { table_id: TableId, ) -> Result>, NodesError> { let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Open the iterator. let iter = self.relational_db().iter_mut(tx, table_id)?; @@ -652,6 +752,7 @@ impl InstanceEnv { // Open index iterator let (table_id, point, iter) = self.relational_db().index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (chunks, rows_scanned, bytes_scanned) = ChunkedWriter::collect_iter(pool, iter); @@ -682,6 +783,7 @@ impl InstanceEnv { let (table_id, iter) = self.relational_db() .index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (point, (chunks, rows_scanned, bytes_scanned)) = match iter { @@ -1446,6 +1548,219 @@ mod test { Ok(db) } + fn bind_test_environment(env: &mut InstanceEnv) -> Result { + use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_environment( + [("A", false), ("MISSING", true)] + .into_iter() + .map(|(name, optional)| EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional, + }) + .collect(), + ); + let module: spacetimedb_schema::def::ModuleDef = builder.finish().try_into()?; + let program = spacetimedb_datastore::traits::Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"environment-unit-test".as_slice(), + ); + env.bind_environment_module(program.hash, Arc::new(module)); + Ok(program) + } + + #[test] + fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; + use spacetimedb_primitives::ViewId; + use std::collections::BTreeMap; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("root".into())?), + Timestamp::now(), + FuncCallType::Reducer, + ); + assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); + let mut tx = begin_mut_tx(&db); + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into())]), + )?; + env.tx.set_raw(tx); + assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + assert!(env.env_get("UNDECLARED").is_err()); + let view = ViewCallInfo::anonymous(ViewId(88)); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("view".into())?), + Timestamp::now(), + FuncCallType::View(view.clone()), + ); + assert_eq!(env.env_get("MISSING")?, None); + let tx = env.tx.take()?; + db.commit_tx(tx)?; + let mut tx = begin_mut_tx(&db); + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "uncommitted".into()), ("MISSING".into(), "".into())]), + )?; + assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + assert_eq!(env.env_get("MISSING")?, None); + assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); + Ok(()) + } + + #[test] + fn environment_rejects_submodules_finished_calls_and_replaced_programs() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::traits::Program; + use std::collections::BTreeMap; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + let mut tx = begin_mut_tx(&db); + db.update_program(&mut tx, program)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "old-value".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + let previous = env.swap_func_context( + Some(NamespacedIdentifier::from_segments(vec![ + spacetimedb_schema::identifier::Identifier::new("child".into())?, + spacetimedb_schema::identifier::Identifier::new("view".into())?, + ])), + FuncCallType::Procedure, + ); + assert!(env.env_get("A").is_err()); + env.swap_func_context(previous.0, previous.1); + assert_eq!(env.env_get("A")?.as_deref(), Some("old-value")); + env.finish_funcall(); + assert!(env.env_get("A").is_err()); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + let mut tx = begin_mut_tx(&db); + let newer = Program::from_bytes( + spacetimedb_datastore::system_tables::ModuleKind::WASM, + b"new-code".as_slice(), + ); + db.update_program(&mut tx, newer)?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("A".into(), "new-secret".into())]), + )?; + db.commit_tx(tx)?; + assert!(env.env_get("A").is_err()); + env.start_mutable_tx()?; + assert!(env.env_get("A").is_err()); + let tx = env.take_mutable_tx_for_commit()?; + env.rollback_procedure_tx(tx); + Ok(()) + } + + #[test] + fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::ST_ENV_ID; + let db = relational_db()?; + let (env, _runtime) = instance_env(db.clone())?; + let mut slot = env.tx.clone(); + let protected = [(ST_ENV_ID, "st_env", to_vec("TOKEN")?)]; + let tx = begin_mut_tx(&db); + let (tx, result) = slot.set(tx, || -> Result<()> { + for (table, name, point) in &protected { + // Host lookup remains available, independently of module lookup. + let (index, index_name) = { + let tx = env.get_tx()?; + let schema = db.schema_for_table_mut(&tx, *table)?; + let index = &schema.indexes[0]; + (index.index_id, index.index_name.to_string()) + }; + assert!(matches!(env.table_id_from_name(name), Err(NodesError::TableNotFound))); + assert!(matches!( + env.index_id_from_name(&index_name), + Err(NodesError::IndexNotFound) + )); + assert!(matches!(env.insert(*table, &mut []), Err(NodesError::TableNotFound))); + assert!(matches!( + env.update(*table, index, &mut []), + Err(NodesError::TableNotFound) + )); + assert!(matches!(env.clear(*table), Err(NodesError::TableNotFound))); + assert!(matches!( + env.datastore_table_row_count(*table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_table_scan_bsatn_chunks(&mut ChunkPool::default(), *table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_all_by_eq_bsatn(*table, &[]), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_index_scan_point_bsatn_chunks(&mut ChunkPool::default(), index, point), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_point_bsatn(index, point), + Err(NodesError::TableNotFound) + )); + let bound = to_vec(&Bound::::Unbounded)?; + assert!(matches!( + env.datastore_index_scan_range_bsatn_chunks( + &mut ChunkPool::default(), + index, + &[], + 0.into(), + &bound, + &bound + ), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_range_bsatn(index, &[], 0.into(), &bound, &bound), + Err(NodesError::TableNotFound) + )); + } + Ok(()) + }); + let _ = db.rollback_mut_tx(tx); + result + } + /// Generate a `ProductValue` for use in [create_table_with_index] fn product_row(i: usize) -> ProductValue { let str = i.to_string(); diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 31aa62ec6bc..d0a1f23c560 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -26,8 +26,8 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, - HostController, HostRuntimeConfig, MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, - ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, + HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, + ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, @@ -191,6 +191,7 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 25eb09e6382..495f6a61fa9 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -608,15 +608,23 @@ pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> (anyhow::Result, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + program, + environment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, program: Program, + environment: std::collections::BTreeMap, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> anyhow::Result<(InitDatabaseResult, bool)> { log::debug!("init database"); @@ -674,6 +682,7 @@ fn init_database_inner( .with_context(|| format!("failed to create row-level security for table `{table_id}`: `{sql}`",))?; } + crate::db::environment::replace(stdb, tx, module_def.environment(), &environment)?; stdb.set_initialized(tx, program)?; anyhow::Ok(()) @@ -3181,12 +3190,20 @@ impl ModuleHost { } pub async fn init_database(&self, program: Program) -> Result { + self.init_database_with_environment(program, Default::default()).await + } + + pub async fn init_database_with_environment( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> Result { call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, environment), + |(program, environment), inst| inst.init_database(program, environment), + |(program, environment), inst| inst.init_database(program, environment).await, )? .map_err(InitDatabaseError::Other) } @@ -3196,13 +3213,24 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_environment(program, old_module_info, policy, Default::default()) + .await + } + + pub async fn update_database_with_environment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, environment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 6f357962bbb..90a83691fd8 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -400,6 +400,7 @@ impl JsInstanceEnv { /// This resets all of the state associated to a single function call, /// and returns instrumentation records. fn finish_funcall(&mut self) -> ExecutionTimings { + self.instance_env.finish_funcall(); let total_duration = self.reducer_start().elapsed(); let func_name = self.log_record_function().unwrap_or("").to_owned(); @@ -424,7 +425,9 @@ impl JsInstanceEnv { } } - fn set_module_def(&mut self, module_def: Arc) { + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def); } @@ -475,11 +478,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + environment, }) .await } @@ -535,8 +540,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.request(InitDatabaseRequest { program, environment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -620,6 +629,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, } => "update_database", anyhow::Result, UpdateDatabase } @@ -662,6 +672,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + environment: std::collections::BTreeMap, } => "init_database", anyhow::Result, InitDatabase } @@ -804,6 +815,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -864,6 +876,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>, program: Program, + environment: std::collections::BTreeMap, }, } @@ -1398,8 +1411,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + environment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, environment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1510,16 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + environment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, bool) = + init_database(replica_ctx, &info.module_def, program, environment, call_reducer); + (res, trapped) + }), } } @@ -1668,7 +1684,10 @@ where return; } Ok(Ok((crf, module_common))) => { - env_on_isolate_unwrap(scope).set_module_def(module_common.info().module_def.clone()); + env_on_isolate_unwrap(scope).set_module_def( + module_common.info().module_def.clone(), + module_common.info().module_hash, + ); if let Some(result_tx) = startup_result_tx.take() && result_tx.send(Ok(module_common.clone())).is_err() @@ -1872,8 +1891,8 @@ impl WasmInstance for V8Instance<'_, '_, '_> { self.scope.get_slot::().unwrap().instance_env.tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - env_on_isolate_unwrap(self.scope).set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + env_on_isolate_unwrap(self.scope).set_module_def(module_def, module_hash); } fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult { diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 52b78463def..e00cf40d1f0 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -865,9 +865,9 @@ fn call_view( fn_ptr: ViewFnPtr, sender: Option, ) -> SysCallResult { - let prev_func_type = get_env(scope)? + let (prev_func_name, prev_func_type) = get_env(scope)? .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let result = { let args = crate::host::ArgsTuple::nullary(); @@ -900,7 +900,9 @@ fn call_view( } }; - get_env(scope)?.instance_env.swap_func_type(prev_func_type); + get_env(scope)? + .instance_env + .swap_func_context(prev_func_name, prev_func_type); result.map_err(|err| match err { ErrorOrException::Err(err) => TypeError(format!( diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index a09e7cbba0c..467e7f26562 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,6 +62,7 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index f49d2260549..3e25145fc74 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,24 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.2", (with_sys_result, AbiCall::EnvGet, env_get),) +} + +fn env_get<'s>( + scope: &mut PinScope<'s, '_>, + args: FunctionCallbackArguments<'s>, +) -> SysCallResult> { + let key: String = deserialize_js(scope, args.get(0))?; + match get_env(scope)?.instance_env.env_get(&key)? { + Some(value) => Ok(value + .into_string(scope) + .map_err(|_| RangeError("environment value could not be represented").throw(scope))? + .into()), + None => Ok(v8::null(scope).into()), + } +} + /// Registers a function in `module` /// where the function has `name` and does `body`. fn register_module_fun( diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7fd18b5e1..f5ab623fec3 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -362,6 +362,8 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::DecodeRow(_) => errno::BSATN_DECODE_ERROR, NodesError::DecodeValue(_) => errno::BSATN_DECODE_ERROR, NodesError::TableNotFound => errno::NO_SUCH_TABLE, + NodesError::InvalidEnvironmentKey => errno::HOST_CALL_FAILURE, + NodesError::EnvironmentSourceLimit => errno::NO_SPACE, NodesError::IndexNotFound => errno::NO_SUCH_INDEX, NodesError::IndexNotUnique => errno::INDEX_NOT_UNIQUE, NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, @@ -442,6 +444,7 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, + "spacetime_10.6"::env_get, } $link_async! { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index aee984fe3a5..7abc0aaf186 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,6 +37,7 @@ use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo, ViewInstanceArgs}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::ExecutionParams; @@ -85,7 +86,7 @@ pub trait WasmInstance { fn tx_slot(&self) -> TxSlot; - fn set_module_def(&mut self, module_def: Arc); + fn set_module_def(&mut self, module_def: Arc, module_hash: Hash); fn call_reducer(&mut self, op: ReducerOp<'_>, budget: FunctionBudget) -> ReducerExecuteResult; @@ -190,6 +191,15 @@ pub(crate) fn run_query_for_view( // Validate shape and disallow views-on-views. for plan in &plans { + // This SQL originates in module code, not an authenticated external + // query. Check every source, including non-returned join inputs, before + // any plan executes. The checked env accessor is the only module path. + ensure!( + !plan + .table_ids() + .any(spacetimedb_datastore::system_tables::is_module_restricted_table), + "module SQL views cannot read a module-restricted table" + ); let Some(source_schema) = plan.return_table() else { bail!("query does not return plain table rows"); }; @@ -431,7 +441,7 @@ impl WasmModuleHostActor { impl WasmModuleHostActor { fn make_from_instance(&self, mut instance: T::Instance) -> WasmModuleInstance { let common = InstanceCommon::new(&self.common); - instance.set_module_def(common.info().module_def.clone()); + instance.set_module_def(common.info().module_def.clone(), common.info().module_hash); WasmModuleInstance { instance, common, @@ -489,9 +499,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, environment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -545,11 +556,15 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result { + pub fn init_database( + &mut self, + program: Program, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { let module_def = &self.common.info.clone().module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx_offset(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database(replica_ctx, module_def, program, environment, call_reducer); self.trapped = trapped; res } @@ -621,6 +636,42 @@ impl WasmModuleInstance { } } +/// Client disconnection and materialized-view refresh are independent effects. +/// A breaking migration does not remove surviving cached view instances. +struct UpdateEffects { + refresh_views: bool, + disconnect_clients: bool, +} + +impl UpdateEffects { + fn after_migration(result: crate::db::update::UpdateResult, tx: &MutTxId) -> Self { + use crate::db::update::UpdateResult; + Self { + refresh_views: matches!(result, UpdateResult::EvaluateSubscribedViews) + || tx.views_for_refresh().next().is_some(), + disconnect_clients: matches!(result, UpdateResult::RequiresClientDisconnect), + } + } + + fn committed( + &self, + tx_offset: TransactionOffset, + durable_offset: Option, + ) -> UpdateDatabaseResult { + if self.disconnect_clients { + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + } else { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + } + } +} + pub struct InstanceCommon { info: Arc, energy_monitor: Arc, @@ -649,6 +700,7 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + environment: std::collections::BTreeMap, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); @@ -674,7 +726,22 @@ impl InstanceCommon { let program_hash = program.hash; let host_type = HostType::from(program.kind); let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; + let row = tx + .iter(ST_MODULE_ID)? + .next() + .context("database program is not initialized")?; + let current_hash = + spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash)?; + anyhow::ensure!( + current_hash == old_module_info.module_hash, + "database program changed before publication" + ); + crate::db::environment::replace(stdb, tx, self.info.module_def.environment(), &environment)?; + stdb.update_program(tx, program)?; + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -717,45 +784,31 @@ impl InstanceCommon { }; let durable_offset = stdb.durable_tx_offset(); - let res: UpdateDatabaseResult = match res { - crate::db::update::UpdateResult::Success => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - crate::db::update::UpdateResult::EvaluateSubscribedViews => { - let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; - tx = out.tx; - if trapped || out.outcome != ViewOutcome::Success { - let msg = match trapped { - true => "Trapped while evaluating views during database update".to_string(), - false => format!( - "Views evaluation did not complete successfully during database update: {:?}", - out.outcome - ), - }; - - let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); - stdb.report_mut_tx_metrics(reducer, tx_metrics, None); - UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) - } else { - let tx_offset = - succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); - UpdateDatabaseResult::UpdatePerformed { - tx_offset, - durable_offset, - } - } - } - crate::db::update::UpdateResult::RequiresClientDisconnect => { - let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); - UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { - tx_offset, - durable_offset, - } + let effects = UpdateEffects::after_migration(res, &tx); + let res = if effects.refresh_views { + // Resolve surviving materializations through the new module, + // even when this migration also requires client disconnection. + let (out, _, trapped) = self.evaluate_subscribed_views(tx, inst)?; + tx = out.tx; + if trapped || out.outcome != ViewOutcome::Success { + let msg = match trapped { + true => "Trapped while evaluating views during database update".to_string(), + false => format!( + "Views evaluation did not complete successfully during database update: {:?}", + out.outcome + ), + }; + + let (_, tx_metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, tx_metrics, None); + UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!(msg)) + } else { + let tx_offset = succeed(self.info.clone(), out.execution_budget_used, out.total_duration, tx); + effects.committed(tx_offset, durable_offset) } + } else { + let tx_offset = succeed(self.info.clone(), FunctionBudget::ZERO, Duration::ZERO, tx); + effects.committed(tx_offset, durable_offset) }; Ok(res) @@ -778,32 +831,22 @@ impl InstanceCommon { params: CallProcedureParams, inst: &mut I, ) -> (CallProcedureReturn, bool) { + // Resolve authority and type refs from the same canonical flattened ID. + // A ProcedureDef's local name alone does not identify its host scope. + let (op, procedure_def, owning_def) = + ProcedureOp::for_module(&self.info.module_def, ¶ms).expect("validated procedure id should resolve"); + let procedure_name = op.name.clone(); let CallProcedureParams { timestamp, caller_identity, - caller_connection_id, timer, - procedure_id, - args, + .. } = params; - // We've already validated by this point that the procedure exists, - // so it's fine to use the panicking `procedure_by_id`. - let procedure_def = self.info.module_def.procedure_by_id(procedure_id); - let procedure_name = &procedure_def.name; - // TODO(observability): Add tracing spans, energy, metrics? // These will require further thinking once we implement procedure suspend/resume, // and so are not worth doing yet. - let op = ProcedureOp { - id: procedure_id, - name: procedure_name.clone().into(), - caller_identity, - caller_connection_id, - timestamp, - arg_bytes: args.get_bsatn().clone(), - }; let energy_fingerprint = FunctionFingerprint { module_hash: self.info.module_hash, module_identity: self.info.owner_identity, @@ -833,7 +876,7 @@ impl InstanceCommon { WORKER_METRICS .wasm_instance_errors - .with_label_values(&self.info.database_identity, &self.info.module_hash, procedure_name) + .with_label_values(&self.info.database_identity, &self.info.module_hash, &procedure_name) .inc(); // TODO(procedure-energy): @@ -846,7 +889,7 @@ impl InstanceCommon { } Ok(return_val) => { let return_type = &procedure_def.return_type; - let seed = spacetimedb_sats::WithTypespace::new(self.info.module_def.typespace(), return_type); + let seed = spacetimedb_sats::WithTypespace::new(owning_def.typespace(), return_type); seed.deserialize(bsatn::Deserializer::new(&mut &return_val[..])) .map_err(|err| ProcedureCallError::InternalError(format!("{err}"))) .map(|return_val| ProcedureCallResult { @@ -1962,6 +2005,27 @@ pub struct ProcedureOp { pub arg_bytes: Bytes, } +impl ProcedureOp { + fn for_module<'a>( + module: &'a ModuleDef, + params: &CallProcedureParams, + ) -> Option<(Self, &'a spacetimedb_schema::def::ProcedureDef, &'a ModuleDef)> { + let (name, def, owning) = module.get_procedure_by_id_with_module(params.procedure_id)?; + Some(( + Self { + id: params.procedure_id, + name, + caller_identity: params.caller_identity, + caller_connection_id: params.caller_connection_id, + timestamp: params.timestamp, + arg_bytes: params.args.get_bsatn().clone(), + }, + def, + owning, + )) + } +} + impl InstanceOp for ProcedureOp { fn name(&self) -> &NamespacedIdentifier { &self.name @@ -2006,6 +2070,242 @@ mod tests { use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; + #[test] + fn breaking_migration_preserves_disconnect_and_refreshes_surviving_environment_view() -> anyhow::Result<()> { + use super::UpdateEffects; + use crate::db::{environment, update}; + use crate::host::UpdateDatabaseResult; + use spacetimedb_datastore::execution_context::Workload; + use spacetimedb_datastore::locking_tx_datastore::FuncCallType; + use spacetimedb_datastore::system_tables::{ModuleKind, ST_ENV_ID}; + use spacetimedb_datastore::traits::Program; + use spacetimedb_lib::db::raw_def::{ + v10::{RawModuleDefV10Builder, RawModuleDefV10Section}, + v9::TableAccess, + }; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + use spacetimedb_lib::identity::AuthCtx; + use spacetimedb_schema::auto_migrate::ponder_migrate; + use std::collections::BTreeMap; + + struct TestLogger; + impl update::UpdateLogger for TestLogger { + fn info(&self, _: &str) {} + } + fn module(include_obsolete_table: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let row = builder.add_algebraic_type( + [], + "EnvironmentViewRow", + AlgebraicType::Product(ProductType::from_iter([("value", AlgebraicType::String)])), + true, + ); + builder.add_view( + "environment_view", + 0, + true, + true, + ProductType::unit(), + AlgebraicType::array(AlgebraicType::Ref(row)), + ); + if include_obsolete_table { + builder + .build_table_with_new_type("obsolete", ProductType::from_iter([("id", AlgebraicType::U64)]), true) + .with_access(TableAccess::Public) + .finish(); + } + let mut raw = builder.finish(); + raw.sections + .push(RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])); + raw.try_into().expect("valid ENV view module") + } + + let db = TestDB::in_memory()?; + let old = module(true); + let new = module(false); + let before = BTreeMap::from([("TOKEN".into(), "before".into())]); + let after = BTreeMap::from([("TOKEN".into(), "after".into())]); + let mut tx = begin_mut_tx(&db); + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"old-program".as_slice()), + )?; + for table in old.tables() { + update::create_table_from_def(&db, &mut tx, &old, table)?; + } + let (view_id, _) = db.create_view(&mut tx, &old, old.view("environment_view").unwrap())?; + let call = ViewCallInfo::anonymous(view_id); + // Ordinary SQL materialization has no live subscriber to disconnect. + tx.update_view_timestamp(call.clone(), ViewInstanceArgs::Anonymous)?; + tx.record_table_scan(&FuncCallType::View(call.clone()), ST_ENV_ID); + environment::replace(&db, &mut tx, old.environment(), &before)?; + db.commit_tx(tx)?; + + let mut tx = begin_mut_tx(&db); + environment::replace(&db, &mut tx, new.environment(), &after)?; + db.update_program( + &mut tx, + Program::from_bytes(ModuleKind::WASM, b"new-program".as_slice()), + )?; + let plan = ponder_migrate(&old, &new)?; + assert!( + plan.breaks_client(), + "removing the unrelated table must require disconnection" + ); + let result = update::update_database(&db, &mut tx, AuthCtx::for_testing(), plan, &TestLogger)?; + assert!(matches!(result, update::UpdateResult::RequiresClientDisconnect)); + assert!(tx.views_for_refresh().any(|dirty| *dirty == call)); + let effects = UpdateEffects::after_migration(result, &tx); + assert!(effects.refresh_views); + assert!(effects.disconnect_clients); + let calls = collect_subscribed_view_calls(&tx, &new, Identity::ZERO)?; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].view_id, view_id); + assert_eq!(&*calls[0].view_name, "environment_view"); + assert!(tx.active_subscribers_for_view(view_id).is_empty()); + let (_send, receive) = tokio::sync::oneshot::channel(); + assert!(matches!( + effects.committed(receive, None), + UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + )); + + // The failed-view branch can roll back the same complete migration tx. + let _ = db.rollback_mut_tx(tx); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!(environment::snapshot(tx).unwrap(), before); + }); + let tx = begin_mut_tx(&db); + assert!(db.table_id_from_name_mut(&tx, "obsolete")?.is_some()); + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn module_sql_views_cannot_read_environment_directly_or_through_a_join() -> anyhow::Result<()> { + use super::run_query_for_view; + use crate::db::environment; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use spacetimedb_primitives::ViewId; + use spacetimedb_sats::product; + use std::collections::BTreeMap; + + let db = TestDB::in_memory()?; + let visible = db.create_table_for_test( + "visible", + &[("key", AlgebraicType::String), ("value", AlgebraicType::String)], + &[0.into()], + )?; + let mut tx = begin_mut_tx(&db); + tx.insert_via_serialize_bsatn(visible, &product!("TOKEN", "ordinary-value"))?; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }])?; + environment::replace( + &db, + &mut tx, + &schema, + &BTreeMap::from([("TOKEN".into(), "private-value".into())]), + )?; + let row_type = ProductType::from_iter([("key", AlgebraicType::String), ("value", AlgebraicType::String)]); + let call = ViewCallInfo::anonymous(ViewId(99)); + let run = |tx: &mut _, query| run_query_for_view(tx, query, &row_type, &call, db.database_identity()); + assert_eq!( + run(&mut tx, "SELECT * FROM visible")?, + vec![product!("TOKEN", "ordinary-value")] + ); + for query in [ + "SELECT * FROM st_env", + "SELECT e.* FROM st_env AS e", + "SELECT v.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + "SELECT e.* FROM visible AS v JOIN st_env AS e ON v.key = e.key", + ] { + let error = run(&mut tx, query).expect_err("module SQL must use the checked environment accessor"); + assert!( + error.to_string().contains("module-restricted table"), + "unexpected query failure for {query}: {error:#}" + ); + assert!(!error.to_string().contains("private-value")); + } + let _ = db.rollback_mut_tx(tx); + Ok(()) + } + + #[test] + fn procedure_operations_resolve_root_and_nested_host_scope_and_typespace() { + use super::{CallProcedureParams, InstanceOp, ProcedureOp}; + use crate::host::ArgsTuple; + use spacetimedb_lib::db::raw_def::v10::{ + RawModuleDefV10, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10, + }; + use spacetimedb_lib::de::DeserializeSeed; + use spacetimedb_lib::Timestamp; + use spacetimedb_primitives::ProcedureId; + use spacetimedb_sats::{bsatn, AlgebraicValue, ProductValue, WithTypespace}; + + fn module(value_type: AlgebraicType) -> RawModuleDefV10 { + let mut builder = RawModuleDefV10Builder::new(); + let result_type = builder.add_algebraic_type( + [], + "ResultRow", + AlgebraicType::Product(ProductType::from_iter([("value", value_type)])), + true, + ); + builder.add_procedure("read_env", ProductType::unit(), AlgebraicType::Ref(result_type)); + builder.finish() + } + + let mut child = module(AlgebraicType::String); + child + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "nested".into(), + module: module(AlgebraicType::Bool), + }])); + let mut root = module(AlgebraicType::U64); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".into(), + module: child, + }])); + let module: ModuleDef = root.try_into().expect("valid nested module"); + for (index, expected_name) in ["read_env", "lib.read_env", "lib.nested.read_env"] + .into_iter() + .enumerate() + { + let params = CallProcedureParams::from_system( + Timestamp::UNIX_EPOCH, + Identity::ZERO, + ProcedureId::from(index), + ArgsTuple::nullary(), + ); + let (op, def, owning) = ProcedureOp::for_module(&module, ¶ms).unwrap(); + assert_eq!(&**op.name(), expected_name); + assert_eq!(op.name().is_namespaced(), index != 0); + assert_eq!(op.id, params.procedure_id); + assert_eq!(&*def.name, "read_env", "declaration names remain local"); + if index == 2 { + let expected = AlgebraicValue::Product(ProductValue::from_iter([AlgebraicValue::Bool(true)])); + let bytes = bsatn::to_vec(&expected).unwrap(); + let decoded = WithTypespace::new(owning.typespace(), &def.return_type) + .deserialize(bsatn::Deserializer::new(&mut &bytes[..])) + .unwrap(); + assert_eq!( + decoded, expected, + "nested type refs must not use the root U64 typespace" + ); + } + } + assert!(module + .get_procedure_by_id_with_module(ProcedureId::from(3usize)) + .is_none()); + } + fn module_def_for_view(name: &str, is_anonymous: bool) -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); let name = RawIdentifier::new(name); diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index da4a2f3987f..0c22ee54694 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -31,6 +31,10 @@ use std::sync::Arc; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; +/// Env reads may retain at most 2 MiB of value bytes outside the Wasm heap. +/// Other outstanding byte sources count against this interface's handle limit. +const MAX_OUTSTANDING_ENV_SOURCES: usize = 256; + /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -253,7 +257,14 @@ impl WasmInstanceEnv { // This allows the module to avoid allocating and make a system call in those cases. if bytes.is_empty() { Ok(BytesSourceId::INVALID) - } else if bytes.len() > u32::MAX as usize { + } else { + self.create_present_bytes_source(bytes) + } + } + + /// Allocate a valid source even for an empty value when zero means absence. + fn create_present_bytes_source(&mut self, bytes: bytes::Bytes) -> RtResult { + if bytes.len() > u32::MAX as usize { // There's no inherent reason we need to error here, // other than that it makes it impossible to report the length in `bytes_source_remaining_length` // and that all of our usage of `BytesSource`s as of writing (pgoldman 2025-09-26) @@ -287,7 +298,9 @@ impl WasmInstanceEnv { self.mem = Some(mem); } - pub fn set_module_def(&mut self, module_def: Arc) { + pub fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.instance_env + .bind_environment_module(module_hash, module_def.clone()); self.module_def = Some(module_def) } @@ -368,6 +381,7 @@ impl WasmInstanceEnv { /// /// This resets the call times and clears the arguments source and error sink. pub fn finish_funcall(&mut self, result_sink: u32) -> (ExecutionTimings, Vec) { + self.instance_env.finish_funcall(); // For the moment, // we only explicitly clear the source/sink buffers and the "syscall" times. // TODO: should we be clearing `iters` and/or `timing_spans`? @@ -1572,6 +1586,34 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + /// Finds the JWT payload associated with `connection_id`. /// A `[ByteSourceId]` for the payload will be written to `target_ptr`. /// If nothing is found for the connection, `[ByteSourceId::INVALID]` (zero) is written to `target_ptr`. @@ -1850,10 +1892,10 @@ impl WasmInstanceEnv { fn_ptr: ViewFnPtr, sender: Option, ) -> anyhow::Result { - let prev_func_type = caller + let (prev_func_name, prev_func_type) = caller .data_mut() .instance_env - .swap_func_type(FuncCallType::View(view_call.clone())); + .swap_func_context(Some(view_name.clone()), FuncCallType::View(view_call.clone())); let mut nested_result_sink = None; let call_result = (|| -> anyhow::Result { @@ -1885,7 +1927,10 @@ impl WasmInstanceEnv { Ok(code) })(); - caller.data_mut().instance_env.swap_func_type(prev_func_type); + caller + .data_mut() + .instance_env + .swap_func_context(prev_func_name, prev_func_type); let result_bytes = { let env = caller.data_mut(); diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index e34367ee0de..be8dd719582 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 5); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 6); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) @@ -625,8 +625,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { self.store.data().instance_env().tx.clone() } - fn set_module_def(&mut self, module_def: Arc) { - self.store.data_mut().set_module_def(module_def); + fn set_module_def(&mut self, module_def: Arc, module_hash: spacetimedb_lib::Hash) { + self.store.data_mut().set_module_def(module_def, module_hash); } #[tracing::instrument(level = "trace", skip_all)] diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index ddd57d780c5..1f2c587b762 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -88,7 +88,23 @@ fn run_inner( // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth) + let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + // Check mutation authority while the automatic rollback guard owns + // the transaction, including rejected administrative statements. + if matches!(&stmt, Statement::DML(_)) && !auth.has_write_access() { + return Err(anyhow!( + "Caller {} is not authorized to run SQL mutations", + auth.caller() + )); + } + if let Statement::DML(dml) = &stmt + && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID + { + return Err(anyhow!( + "Database environment variables can only be changed by publishing" + )); + } + Ok(stmt) })?; let mut metrics = ExecutionMetrics::default(); @@ -142,13 +158,10 @@ fn run_inner( )) } Statement::DML(stmt) => { - // An extra layer of auth is required for DML - if !auth.has_write_access() { - return Err(anyhow!("Caller {} is not authorized to run SQL DML statements", auth.caller()).into()); - } - - // Evaluate the mutation - let (mut tx, _) = db.with_auto_rollback(tx, |tx| execute_dml_stmt(&auth, stmt, tx, &mut metrics))?; + let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + execute_dml_stmt(&auth, stmt, tx, &mut metrics)?; + Ok(()) + })?; // Update transaction metrics tx.metrics.merge(metrics); @@ -243,6 +256,63 @@ pub(crate) mod tests { use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_schema::table_name::TableName; + #[test] + fn environment_sql_is_read_only_including_for_owner() { + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use spacetimedb_lib::identity::SqlPermission; + use std::collections::BTreeMap; + let db = TestDB::in_memory().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let owner = AuthCtx::for_current(Identity::ZERO); + let viewer = AuthCtx::with_permissions( + Identity::ONE, + Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), + ); + let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let value = "secret-marker"; + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + crate::db::environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), value.into())])) + }) + .unwrap(); + let execute = |statement: &str, auth: AuthCtx| { + runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) + }; + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", viewer.clone()) + .unwrap() + .rows, + vec![product![value]] + ); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [owner.clone(), viewer, outsider] { + for statement in [ + "SET env.TOKEN = 'forbidden'", + "DELETE env.TOKEN", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!( + execute(statement, auth.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + } + // Rejected writes must release their transactions and preserve data. + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'TOKEN'", owner) + .unwrap() + .rows, + vec![product![value]] + ); + } + /// Short-cut for simplify test execution pub(crate) fn run_for_testing(db: &Arc, sql_text: &str) -> Result, DBError> { let (subs, runtime) = ModuleSubscriptions::for_test_new_runtime(db.clone()); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 0ce629524c4..3f61903e645 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -29,9 +29,9 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, ST_TABLE_ACCESSOR_ID, - ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, - ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, + ST_ENV_ID, ST_ENV_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, + ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, }; use anyhow::anyhow; @@ -357,6 +357,7 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + self.create_table(ST_ENV_ID, schemas[ST_ENV_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 7293a81fba0..1082e0ae9c6 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1092,6 +1092,7 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ST_ENV_ID, ST_ENV_NAME}; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1558,6 +1559,7 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, + TableRow { id: ST_ENV_ID.into(), name: ST_ENV_NAME, ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1655,6 +1657,8 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 0, name: "table_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 1, name: "col_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1687,6 +1691,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1732,6 +1737,7 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 23, table_id: ST_INDEX_ACCESSOR_ID.into(), unique_columns: col(1), constraint_name: "st_index_accessor_accessor_name_key", }, ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, + ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col_list![0], constraint_name: "st_env_key_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2165,6 +2171,7 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 2635093cc2d..61dcd11be9b 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -207,7 +207,7 @@ pub enum SystemTable { st_event_table = ST_EVENT_TABLE_ID.0 as _, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 21] { [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -230,6 +230,7 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + st_env_schema(), ] } @@ -278,6 +279,7 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; +pub(crate) const ST_ENV_IDX: usize = 20; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -313,6 +315,9 @@ macro_rules! st_fields_enum { } } +mod environment; +pub use environment::*; + // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, @@ -670,6 +675,8 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + environment::register_table(&mut builder); + let result = builder .finish() .try_into() @@ -695,6 +702,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + validate_system_table::(&result, ST_ENV_NAME); result } @@ -743,6 +751,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.insert("st_env_key_key", ConstraintId(26)); m }; } @@ -781,6 +790,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.insert("st_env_key_idx_btree", IndexId(30)); m }; } @@ -970,6 +980,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), + ST_ENV_ID => Some(st_env_schema()), _ => None, } } diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs new file mode 100644 index 00000000000..d79f676edb8 --- /dev/null +++ b/crates/datastore/src/system_tables/environment.rs @@ -0,0 +1,42 @@ +//! Private database environment state. Values follow ordinary table durability. +use super::*; +pub const ST_ENV_ID: TableId = TableId(21); +pub const ST_ENV_NAME: &str = "st_env"; +st_fields_enum!(enum StEnvFields { "key", Key = 0, "value", Value = 1, }); +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StEnvRow { + pub key: String, + pub value: String, +} +impl TryFrom> for StEnvRow { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } +} +impl From for ProductValue { + fn from(row: StEnvRow) -> Self { + to_product_value(&row) + } +} +pub(super) fn register_table(builder: &mut RawModuleDefV9Builder) { + let ty = builder.add_type::(); + builder + .build_table(ST_ENV_NAME, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); +} +pub(crate) fn st_env_schema() -> TableSchema { + st_schema(ST_ENV_NAME, ST_ENV_ID) +} +/// Module code must use env_get even when it guesses numeric identifiers. +pub fn is_module_restricted_table(table: TableId) -> bool { + table == ST_ENV_ID +} +pub fn is_module_restricted_index(index: IndexId) -> bool { + index == IndexId(30) +} diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index b21d8470b84..9aec59ffaa8 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -101,6 +101,9 @@ pub enum RawModuleDefV10Section { /// Submodules, keyed by the namespace they are registered under. Submodules(Vec), + + /// Declared publish-only configuration. Even an empty section requires ENV support. + Environment(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -715,6 +718,27 @@ impl RawModuleDefV10Builder { Default::default() } + /// Declare a complete environment schema, including an explicit empty schema. + /// Repeated calls remain repeated sections so host validation rejects ambiguity. + pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { + self.module + .sections + .push(RawModuleDefV10Section::Environment(declarations)); + self + } + + /// New ENV-aware bindings declare an empty schema when no declaration is registered. + pub fn ensure_environment(&mut self) { + if !self + .module + .sections + .iter() + .any(|section| matches!(section, RawModuleDefV10Section::Environment(_))) + { + self.add_environment(Vec::new()); + } + } + /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { let idx = self diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs new file mode 100644 index 00000000000..8b6f6db38d8 --- /dev/null +++ b/crates/lib/src/environment.rs @@ -0,0 +1,383 @@ +//! Limits shared by the database environment store and its clients. + +pub const MAX_ENV_KEY_BYTES: usize = 256; +pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; +pub const MAX_ENV_VARS: usize = 256; +/// Total key and literal bytes in a declaration schema, independent of runtime values. +pub const MAX_ENV_SCHEMA_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_ENV_UNION_ENTRIES: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnvironmentValidationError { + InvalidKey, + ValueTooLarge, + TooManyVariables, +} + +impl std::fmt::Display for EnvironmentValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::InvalidKey => "invalid POSIX environment variable name (maximum 256 bytes)", + Self::ValueTooLarge => "environment value exceeds 8192 UTF-8 bytes", + Self::TooManyVariables => "environment store exceeds 256 variables", + }) + } +} + +impl std::error::Error for EnvironmentValidationError {} + +pub fn validate_key(key: &str) -> Result<(), EnvironmentValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > MAX_ENV_KEY_BYTES + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(EnvironmentValidationError::InvalidKey); + } + Ok(()) +} + +/// NUL is representable in the database. Container launch separately rejects it. +pub fn validate_value(value: &str) -> Result<(), EnvironmentValidationError> { + if value.len() > MAX_ENV_VALUE_BYTES { + return Err(EnvironmentValidationError::ValueTooLarge); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_utf8_byte_limits_and_posix_keys_without_container_policy() { + for key in ["", "1FIRST", "A=B", "A\0B", "Γ©", "A-B"] { + assert_eq!(validate_key(key), Err(EnvironmentValidationError::InvalidKey)); + } + assert!(validate_key(&"A".repeat(256)).is_ok()); + assert!(validate_key(&"A".repeat(257)).is_err()); + assert!(validate_key("SPACETIMEDB_USER_DATA").is_ok()); + assert!(validate_value("\0").is_ok()); + assert!(validate_value(&"Γ©".repeat(4096)).is_ok()); + assert!(validate_value(&"Γ©".repeat(4097)).is_err()); + } +} + +/// Host-validated string constraints. Values remain strings in every module SDK. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub enum EnvironmentConstraint { + AnyString, + Literal(String), + OneOf(Vec), +} + +/// Declaration metadata, never an environment value supplied during publishing. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, crate::SpacetimeType)] +#[sats(crate = crate)] +pub struct EnvironmentDeclaration { + pub name: String, + pub constraint: EnvironmentConstraint, + pub optional: bool, +} + +/// An environment schema whose declarations have passed host validation. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct EnvironmentSchema { + declarations: std::collections::BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EnvironmentSchemaErrorKind { + InvalidName, + TooManyDeclarations, + DuplicateDeclaration, + EmptyUnion, + TooManyUnionEntries, + SchemaTooLarge, + LiteralTooLarge, + Undeclared, + MissingRequired, + ValueTooLarge, + ConstraintMismatch, +} + +/// Errors identify a key and rule, and never contain a supplied or allowed value. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct EnvironmentSchemaError { + pub key: Option, + pub kind: EnvironmentSchemaErrorKind, +} + +impl std::fmt::Display for EnvironmentSchemaError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(key) = &self.key { + write!(f, "environment key {key:?}: ")?; + } + f.write_str(match self.kind { + EnvironmentSchemaErrorKind::InvalidName => "invalid name", + EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", + EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", + EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", + EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", + EnvironmentSchemaErrorKind::SchemaTooLarge => "declaration schema exceeds size limit", + EnvironmentSchemaErrorKind::LiteralTooLarge => "declared literal exceeds value size limit", + EnvironmentSchemaErrorKind::Undeclared => "key is not declared", + EnvironmentSchemaErrorKind::MissingRequired => "required value is missing", + EnvironmentSchemaErrorKind::ValueTooLarge => "value exceeds size limit", + EnvironmentSchemaErrorKind::ConstraintMismatch => "value does not satisfy its declared string constraint", + }) + } +} + +impl std::error::Error for EnvironmentSchemaError {} + +impl EnvironmentSchema { + fn validate_metadata(declarations: &[EnvironmentDeclaration]) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + if declarations.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError { + key: None, + kind: Kind::TooManyDeclarations, + }); + } + let mut bytes = 0usize; + for declaration in declarations { + // Never retain or format unvalidated key bytes in diagnostics. + validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind, + }; + bytes += declaration.name.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + let literals = match &declaration.constraint { + EnvironmentConstraint::AnyString => &[][..], + EnvironmentConstraint::Literal(value) => std::slice::from_ref(value), + EnvironmentConstraint::OneOf(values) => { + if values.is_empty() { + return Err(error(Kind::EmptyUnion)); + } + if values.len() > MAX_ENV_UNION_ENTRIES { + return Err(error(Kind::TooManyUnionEntries)); + } + values.as_slice() + } + }; + for value in literals { + validate_value(value).map_err(|_| error(Kind::LiteralTooLarge))?; + bytes += value.len(); + if bytes > MAX_ENV_SCHEMA_BYTES { + return Err(error(Kind::SchemaTooLarge)); + } + } + } + Ok(()) + } + + pub fn new(declarations: Vec) -> Result { + Self::validate_metadata(&declarations)?; + let mut schema = Self::default(); + for mut declaration in declarations { + if let EnvironmentConstraint::OneOf(values) = &mut declaration.constraint { + values.sort_unstable(); + values.dedup(); + } + if schema.declarations.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name), + kind: EnvironmentSchemaErrorKind::DuplicateDeclaration, + }); + } + schema.declarations.insert(declaration.name.clone(), declaration); + } + Ok(schema) + } + + /// Check bounds before cloning raw untrusted metadata into the validated schema. + pub fn from_declarations(declarations: &[EnvironmentDeclaration]) -> Result { + Self::validate_metadata(declarations)?; + Self::new(declarations.to_vec()) + } + + pub fn get(&self, name: &str) -> Option<&EnvironmentDeclaration> { + self.declarations.get(name) + } + + pub fn declarations(&self) -> impl ExactSizeIterator { + self.declarations.values() + } + + pub fn into_declarations(self) -> Vec { + self.declarations.into_values().collect() + } + + pub fn is_empty(&self) -> bool { + self.declarations.is_empty() + } + + /// Validate a complete publish input. Existing stored values are not inputs. + pub fn validate_values( + &self, + values: &std::collections::BTreeMap, + ) -> Result<(), EnvironmentSchemaError> { + use EnvironmentSchemaErrorKind as Kind; + for (name, value) in values { + validate_key(name).map_err(|_| EnvironmentSchemaError { + key: None, + kind: Kind::InvalidName, + })?; + let error = |kind| EnvironmentSchemaError { + key: Some(name.clone()), + kind, + }; + let declaration = self.get(name).ok_or_else(|| error(Kind::Undeclared))?; + validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + let matches = match &declaration.constraint { + EnvironmentConstraint::AnyString => true, + EnvironmentConstraint::Literal(expected) => value == expected, + EnvironmentConstraint::OneOf(allowed) => allowed.binary_search(value).is_ok(), + }; + if !matches { + return Err(error(Kind::ConstraintMismatch)); + } + } + for declaration in self.declarations() { + if !declaration.optional && !values.contains_key(&declaration.name) { + return Err(EnvironmentSchemaError { + key: Some(declaration.name.clone()), + kind: Kind::MissingRequired, + }); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod schema_tests { + use super::*; + use std::collections::BTreeMap; + + fn declaration(name: &str, constraint: EnvironmentConstraint, optional: bool) -> EnvironmentDeclaration { + EnvironmentDeclaration { + name: name.into(), + constraint, + optional, + } + } + + #[test] + fn complete_inputs_preserve_optional_empty_and_exact_string_constraints() { + let schema = EnvironmentSchema::new(vec![ + declaration("REQUIRED", EnvironmentConstraint::AnyString, false), + declaration( + "MODE", + EnvironmentConstraint::OneOf(vec!["false".into(), "true".into(), "false".into()]), + false, + ), + declaration("OPTIONAL", EnvironmentConstraint::Literal("".into()), true), + ]) + .unwrap(); + let mut values = BTreeMap::from([("REQUIRED".into(), "\0ι›ͺ".into()), ("MODE".into(), "false".into())]); + schema.validate_values(&values).unwrap(); + values.insert("OPTIONAL".into(), "".into()); + schema.validate_values(&values).unwrap(); + values.insert("MODE".into(), "False".into()); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::ConstraintMismatch + ); + values.remove("MODE"); + assert_eq!( + schema.validate_values(&values).unwrap_err().kind, + EnvironmentSchemaErrorKind::MissingRequired + ); + values.insert("UNDECLARED".into(), "secret-marker".into()); + let error = schema.validate_values(&values).unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::Undeclared); + assert!(!format!("{error:?}: {error}").contains("secret-marker")); + } + + #[test] + fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { + let optional = declaration("A", EnvironmentConstraint::AnyString, true); + assert_eq!( + EnvironmentSchema::new(vec![optional.clone(), optional]) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::DuplicateDeclaration + ); + assert_eq!( + EnvironmentSchema::new( + (0..=MAX_ENV_VARS) + .map(|i| declaration(&format!("K{i}"), EnvironmentConstraint::AnyString, true)) + .collect() + ) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::TooManyDeclarations + ); + for (name, constraint, expected) in [ + ( + "A-B", + EnvironmentConstraint::AnyString, + EnvironmentSchemaErrorKind::InvalidName, + ), + ( + "A", + EnvironmentConstraint::OneOf(vec![]), + EnvironmentSchemaErrorKind::EmptyUnion, + ), + ( + "A", + EnvironmentConstraint::Literal("x".repeat(MAX_ENV_VALUE_BYTES + 1)), + EnvironmentSchemaErrorKind::LiteralTooLarge, + ), + ] { + assert_eq!( + EnvironmentSchema::new(vec![declaration(name, constraint, true)]) + .unwrap_err() + .kind, + expected + ); + } + assert!(EnvironmentSchema::default().validate_values(&BTreeMap::new()).is_ok()); + assert_eq!( + EnvironmentSchema::default() + .validate_values(&BTreeMap::from([("A".into(), "".into())])) + .unwrap_err() + .kind, + EnvironmentSchemaErrorKind::Undeclared + ); + } + + #[test] + fn raw_metadata_is_bounded_before_copying_or_formatting_untrusted_keys() { + let invalid = format!("private-marker\n{}", "x".repeat(100_000)); + let error = + EnvironmentSchema::new(vec![declaration(&invalid, EnvironmentConstraint::AnyString, true)]).unwrap_err(); + assert_eq!(error.key, None); + assert!(!format!("{error:?}: {error}").contains("private-marker")); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["".into(); MAX_ENV_UNION_ENTRIES + 1]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::TooManyUnionEntries); + let error = EnvironmentSchema::new(vec![declaration( + "A", + EnvironmentConstraint::OneOf(vec!["x".repeat(MAX_ENV_VALUE_BYTES); MAX_ENV_UNION_ENTRIES]), + true, + )]) + .unwrap_err(); + assert_eq!(error.kind, EnvironmentSchemaErrorKind::SchemaTooLarge); + } +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index aedded78ad1..55d5621a16d 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -13,6 +13,7 @@ use std::collections::{btree_map, BTreeMap}; pub mod connection_id; pub mod db; mod direct_index_key; +pub mod environment; pub mod error; mod filterable_value; pub mod http; diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 521bcb35aa9..e324d93b0ea 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -18,6 +18,7 @@ use std::collections::BTreeMap; use std::fmt::{self, Debug, Write}; use std::hash::Hash; +use std::sync::LazyLock; use crate::error::{IdentifierError, ValidationErrors}; use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; @@ -44,6 +45,7 @@ use spacetimedb_lib::db::raw_def::v9::{ RawUniqueConstraintDataV9, RawViewDefV9, TableAccess, TableType, }; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; +use spacetimedb_lib::environment::EnvironmentSchema; use spacetimedb_lib::{ProductType, RawModuleDef}; use spacetimedb_primitives::{ ColId, ColList, ColOrCols, ColSet, HttpHandlerId, ProcedureId, ReducerId, TableId, ViewFnPtr, @@ -179,6 +181,9 @@ pub struct ModuleDef { /// Submodules, keyed by the namespace they are registered under. submodules: IndexMap, + + /// `None` means undeclared; an explicitly empty declaration is `Some(empty)`. + environment: Option, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -190,6 +195,20 @@ pub enum RawModuleDefVersion { } impl ModuleDef { + /// The validated root environment schema. Legacy modules have an empty schema. + pub fn environment(&self) -> &EnvironmentSchema { + static EMPTY: LazyLock = LazyLock::new(EnvironmentSchema::default); + match &self.environment { + Some(schema) => schema, + None => &EMPTY, + } + } + + /// Whether the raw module explicitly required environment support. + pub fn environment_declared(&self) -> bool { + self.environment.is_some() + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -823,15 +842,27 @@ impl ModuleDef { /// Look up a procuedure by its id, returning `None` if it doesn't exist. pub fn get_procedure_by_id(&self, id: ProcedureId) -> Option<&ProcedureDef> { + self.get_procedure_by_id_with_module(id).map(|(_, def, _)| def) + } + + /// Resolve a flattened wire ID to its host-qualified name, definition and + /// owning module. Procedure definitions store local names and type refs. + pub fn get_procedure_by_id_with_module( + &self, + id: ProcedureId, + ) -> Option<(NamespacedIdentifier, &ProcedureDef, &ModuleDef)> { let idx = id.idx(); if idx < self.procedures.len() { - return self.procedures.get_index(idx).map(|(_, def)| def); + return self + .procedures + .get_index(idx) + .map(|(_, def)| (self.path.join(def.name.clone()), def, self)); } let mut offset = self.procedures.len(); for submodule in self.submodules.values() { let count = submodule.procedure_count(); if idx < offset + count { - return submodule.get_procedure_by_id(ProcedureId::from(idx - offset)); + return submodule.get_procedure_by_id_with_module(ProcedureId::from(idx - offset)); } offset += count; } @@ -986,6 +1017,7 @@ impl From for RawModuleDefV9 { http_routes: _, raw_module_def_version: _, submodules: _, + environment: _, } = val; // Extract column defaults from tables before consuming tables @@ -1046,9 +1078,13 @@ impl From for RawModuleDefV10 { http_routes, raw_module_def_version: _, submodules, + environment, } = val; let mut sections = Vec::new(); + if let Some(environment) = environment { + sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); + } let mut explicit_names = ExplicitNames::default(); sections.push(RawModuleDefV10Section::Typespace(typespace)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index d6303f3a81c..084f6a1aac5 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -78,6 +78,7 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let environment = validate_environment(&def); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -297,8 +298,8 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules) = - (tables_types_reducers_procedures_views, submodules) + let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules, environment) = + (tables_types_reducers_procedures_views, submodules, environment) .combine_errors() .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; @@ -322,6 +323,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_routes, raw_module_def_version: RawModuleDefVersion::V10, submodules, + environment, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -334,6 +336,22 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } +fn validate_environment(def: &RawModuleDefV10) -> Result> { + let mut sections = def.sections.iter().filter_map(|section| match section { + RawModuleDefV10Section::Environment(declarations) => Some(declarations), + _ => None, + }); + let Some(declarations) = sections.next() else { + return Ok(None); + }; + if sections.next().is_some() { + return Err(ValidationError::RepeatedEnvironmentDeclaration.into()); + } + let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) + .map_err(|error| ValidationError::Environment { error })?; + Ok(Some(schema)) +} + /// Validate that each submodule's namespace is a valid identifier of at most 63 characters, /// that no two submodules share the same namespace, and that no submodule declares lifecycle /// reducers (lifecycle reducers are only permitted in the root module). @@ -365,6 +383,11 @@ fn validate_submodules(submodules: Vec) -> Result { + if !def.environment().is_empty() { + errors.push(ValidationError::EnvironmentInSubmodule { + namespace: submodule.namespace.clone(), + }); + } for (lifecycle, opt_id) in def.lifecycle_reducers_map() { if opt_id.is_some() { errors.push(ValidationError::LifecycleInSubmodule { @@ -2796,3 +2819,80 @@ mod tests { }); } } + +#[cfg(test)] +mod environment_tests { + use super::*; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + + fn declared(name: &str) -> RawModuleDefV10 { + RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + name: name.into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }])], + } + } + + #[test] + fn environment_schema_round_trip_preserves_explicit_empty_and_exact_keys() { + let legacy = validate(RawModuleDefV10::default()).unwrap(); + assert!(legacy.environment().is_empty()); + assert!(!legacy.environment_declared()); + let raw: RawModuleDefV10 = legacy.into(); + assert!(!validate(raw).unwrap().environment_declared()); + let explicit = validate(RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }) + .unwrap(); + assert!(explicit.environment().is_empty()); + assert!(explicit.environment_declared()); + let raw: RawModuleDefV10 = explicit.into(); + assert!(validate(raw).unwrap().environment_declared()); + let module = validate(declared("Mixed_CASE")).unwrap(); + assert!(module.environment().get("Mixed_CASE").is_some()); + assert!(module.environment().get("mixed_case").is_none()); + let raw: RawModuleDefV10 = module.into(); + assert!(validate(raw).unwrap().environment().get("Mixed_CASE").is_some()); + assert_eq!( + spacetimedb_lib::bsatn::to_vec(&RawModuleDefV10Section::Environment(vec![])).unwrap(), + vec![15, 0, 0, 0, 0] + ); + } + + #[test] + fn environment_rejects_ambiguous_sections_and_nested_declarations() { + let mut duplicate = declared("A"); + duplicate.sections.push(RawModuleDefV10Section::Environment(vec![])); + assert!(validate(duplicate) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::RepeatedEnvironmentDeclaration))); + let nested = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "outer".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "inner".into(), + module: declared("SECRET"), + }])], + }, + }])], + }; + assert!(validate(nested) + .unwrap_err() + .into_iter() + .any(|error| matches!(error, ValidationError::EnvironmentInSubmodule { .. }))); + let empty = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "allowed".into(), + module: RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Environment(vec![])], + }, + }])], + }; + assert!(validate(empty).is_ok()); + assert!(validate(declared("INVALID-NAME")).is_err()); + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 8de4927afa2..c6eef4f74f9 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -171,6 +171,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), + environment: None, }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index e9408a35482..691fef4a3f5 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("module has repeated environment declarations")] + RepeatedEnvironmentDeclaration, + #[error("invalid environment declaration: {error}")] + Environment { + error: spacetimedb_lib::environment::EnvironmentSchemaError, + }, + #[error("submodule {namespace:?} cannot declare environment variables")] + EnvironmentInSubmodule { namespace: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/smoketests/modules/Cargo.lock b/crates/smoketests/modules/Cargo.lock index 69d26e1f356..3bfe68ba8c2 100644 --- a/crates/smoketests/modules/Cargo.lock +++ b/crates/smoketests/modules/Cargo.lock @@ -756,6 +756,13 @@ dependencies = [ "spacetimedb", ] +[[package]] +name = "smoketest-module-environment-publish" +version = "0.1.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "smoketest-module-fail-initial-publish-broken" version = "0.1.0" diff --git a/crates/smoketests/modules/Cargo.toml b/crates/smoketests/modules/Cargo.toml index 63dc67687eb..5184afb1947 100644 --- a/crates/smoketests/modules/Cargo.toml +++ b/crates/smoketests/modules/Cargo.toml @@ -119,6 +119,7 @@ members = [ "new-user-flow", "module-nested-op", "noop", + "environment-publish", "fail-initial-publish-broken", "fail-initial-publish-fixed", diff --git a/crates/smoketests/modules/environment-publish/Cargo.toml b/crates/smoketests/modules/environment-publish/Cargo.toml new file mode 100644 index 00000000000..8ed5366b336 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "smoketest-module-environment-publish" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +spacetimedb.workspace = true diff --git a/crates/smoketests/modules/environment-publish/src/lib.rs b/crates/smoketests/modules/environment-publish/src/lib.rs new file mode 100644 index 00000000000..57d82e8b7d2 --- /dev/null +++ b/crates/smoketests/modules/environment-publish/src/lib.rs @@ -0,0 +1,46 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::env] +pub struct Env { + pub SMOKE_REQUIRED: String, + #[env(values("ready", "other"))] + pub SMOKE_MODE: String, + pub SMOKE_OPTIONAL: Option, + pub SMOKE_EMPTY: Option, + pub SMOKE_NUMBER: Option, + pub SMOKE_FLAG: Option, +} + +#[spacetimedb::table(accessor = initial_environment, public)] +pub struct InitialEnvironment { + required: String, + mode: String, + optional: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.initial_environment().insert(InitialEnvironment { + required: ctx.env.SMOKE_REQUIRED(), + mode: ctx.env.SMOKE_MODE(), + optional: ctx.env.SMOKE_OPTIONAL(), + }); +} + +#[spacetimedb::reducer] +pub fn check_environment( + ctx: &ReducerContext, + required: String, + mode: String, + optional: Option, + empty: Option, + number: Option, + flag: Option, +) { + assert_eq!(ctx.env.SMOKE_REQUIRED(), required); + assert_eq!(ctx.env.SMOKE_MODE(), mode); + assert_eq!(ctx.env.SMOKE_OPTIONAL(), optional); + assert_eq!(ctx.env.SMOKE_EMPTY(), empty); + assert_eq!(ctx.env.SMOKE_NUMBER(), number); + assert_eq!(ctx.env.SMOKE_FLAG(), flag); +} diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 3ec8f5b7141..f9ff97e7fe7 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -802,6 +802,7 @@ pub struct SmoketestBuilder { autopublish: bool, pg_port: Option, server_url_override: Option, + isolated_local_server: bool, cli_path: Option, } @@ -828,6 +829,7 @@ impl SmoketestBuilder { autopublish: true, pg_port: None, server_url_override: None, + isolated_local_server: false, cli_path: None, } } @@ -837,6 +839,14 @@ impl SmoketestBuilder { self } + /// Start an owned local server with a fresh CLI configuration, even in a + /// remote test job. Do not copy the remote job's login or base configuration. + /// This cannot be combined with an explicit server URL. + pub fn isolated_local_server(mut self) -> Self { + self.isolated_local_server = true; + self + } + /// Uses a specific CLI binary instead of the pre-built CLI for this test. pub fn cli_path(mut self, path: impl AsRef) -> Self { self.cli_path = Some(path.as_ref().to_path_buf()); @@ -923,6 +933,15 @@ impl SmoketestBuilder { /// Panics if the CLI/standalone binaries haven't been built or are stale. /// Run `cargo smoketest prepare` to build binaries before running tests. pub fn build(self) -> Smoketest { + assert!( + !self.isolated_local_server || self.server_url_override.is_none(), + "isolated_local_server cannot use an explicit remote server URL" + ); + let inherited_remote = if self.isolated_local_server { + None + } else { + remote_server_url() + }; // Check binaries first - this will panic with a helpful message if missing/stale if self.cli_path.is_none() { let _ = ensure_binaries_built(); @@ -936,7 +955,7 @@ impl SmoketestBuilder { // Check if we're running against a remote server let (guard, server_url, data_dir_fixture) = if let Some(fixture) = self.data_dir_fixture.as_ref() { - if self.server_url_override.is_some() || remote_server_url().is_some() { + if self.server_url_override.is_some() || inherited_remote.is_some() { panic!("data_dir_fixture requires a local server managed by the smoketest harness"); } @@ -963,7 +982,7 @@ impl SmoketestBuilder { } else if let Some(url) = self.server_url_override { eprintln!("[REMOTE] Using explicit server URL: {}", url); (None, url, None) - } else if let Some(remote_url) = remote_server_url() { + } else if let Some(remote_url) = inherited_remote { eprintln!("[REMOTE] Using remote server: {}", remote_url); (None, remote_url, None) } else { @@ -998,7 +1017,9 @@ impl SmoketestBuilder { let module_name = format!("smoketest_module_{}", random_string()); let config_path = project_dir.path().join("config.toml"); - if let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") { + if !self.isolated_local_server + && let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") + { fs::copy(&base_config_path, &config_path) .unwrap_or_else(|err| panic!("failed to copy base smoketest config from {base_config_path}: {err:#}")); } diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs new file mode 100644 index 00000000000..957ace39286 --- /dev/null +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -0,0 +1,458 @@ +//! Publish-only environment configuration through the real CLI and local server. +use serde_json::{json, Value}; +use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_smoketests::{modules, random_string, Smoketest}; +use std::{ + fs, + io::{Read as _, Seek as _}, + path::PathBuf, + process::{Child, Command, Output, Stdio}, + time::{Duration, Instant}, +}; + +const KEYS: &[&str] = &[ + "SMOKE_REQUIRED", + "SMOKE_MODE", + "SMOKE_OPTIONAL", + "SMOKE_EMPTY", + "SMOKE_NUMBER", + "SMOKE_FLAG", +]; + +struct EnvironmentFixture { + test: Smoketest, + database: String, + wasm: PathBuf, +} + +impl EnvironmentFixture { + fn new() -> Self { + // Private CI supplies remote cluster settings to the same test binary. + // This fixture must still create its own server and fresh credentials, + // without changing those settings for other tests in the process. + let remote_settings = [ + "SPACETIME_REMOTE_SERVER", + "SPACETIME_USE_AUTH_HOST", + "SPACETIME_SMOKETEST_BASE_CONFIG_PATH", + ]; + let inherited = remote_settings.map(std::env::var_os); + let test = Smoketest::builder() + .isolated_local_server() + .precompiled_module("environment-publish") + .autopublish(false) + .build(); + assert_eq!(remote_settings.map(std::env::var_os), inherited); + assert!(test.guard.is_some()); + assert!(!test.config_path.exists(), "local fixture copied inherited credentials"); + let address = test + .server_url + .strip_prefix("http://") + .unwrap() + .parse::() + .unwrap(); + assert_eq!(address.ip(), std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); + assert_ne!(address.port(), 0); + let wasm = test.project_dir.path().join("published.wasm"); + fs::copy(modules::precompiled_module("environment-publish"), &wasm).unwrap(); + // A misleading local source proves --bin-path reads declarations from + // these exact bytes, rather than trusting nearby source/config metadata. + fs::create_dir(test.project_dir.path().join("src")).unwrap(); + fs::write( + test.project_dir.path().join("src/lib.rs"), + "#[spacetimedb::env] pub struct Env { pub WRONG_SOURCE_DECLARATION: String }", + ) + .unwrap(); + let fixture = Self { + test, + database: format!("environment-{}", random_string()), + wasm, + }; + fixture.success(&["login", "--server-issued-login", &fixture.test.server_url], &[]); + fixture + } + + fn command(&self, args: &[&str], shell: &[(&str, &str)]) -> Output { + let mut command = Command::new(ensure_binaries_built()); + command.env_clear(); + // Runtime executables are already built. No user credentials, remote + // settings, or ambient module variables enter these child processes. + for key in ["PATH", "SystemRoot", "WINDIR", "TMP", "TEMP", "TMPDIR"] { + if let Some(value) = std::env::var_os(key) { + command.env(key, value); + } + } + command + .env("HOME", self.test.project_dir.path()) + .env("USERPROFILE", self.test.project_dir.path()) + .env("XDG_CONFIG_HOME", self.test.project_dir.path()) + .env("NO_PROXY", "*") + .env("no_proxy", "*") + .envs(shell.iter().copied()) + // Avoid platform directory discovery after clearing the environment, + // including Windows' known-folder lookup for LocalAppData. + .arg("--root-dir") + .arg(self.test.project_dir.path()) + .arg("--config-path") + .arg(&self.test.config_path) + .args(args) + .current_dir(self.test.project_dir.path()) + .stdin(Stdio::null()); + bounded_output(command) + } + + fn success(&self, args: &[&str], shell: &[(&str, &str)]) -> String { + let output = self.command(args, shell); + assert!( + output.status.success(), + "local ENV CLI command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + fn config(&self, environment: Option) { + for file in [ + "spacetime.local.json", + "spacetime.prod.json", + "spacetime.prod.local.json", + ] { + let path = self.test.project_dir.path().join(file); + if path.exists() { + fs::remove_file(path).unwrap(); + } + } + let mut config = json!({"database": self.database}); + if let Some(environment) = environment { + config["env"] = environment; + } + self.write("spacetime.json", config); + } + + fn write(&self, file: &str, value: Value) { + fs::write( + self.test.project_dir.path().join(file), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + } + + fn publish(&self, shell: &[(&str, &str)], extra: &[&str]) -> Output { + let mut args = vec![ + "publish", + &self.database, + "--bin-path", + self.wasm.to_str().unwrap(), + "--server", + &self.test.server_url, + "--yes", + ]; + args.extend_from_slice(extra); + self.command(&args, shell) + } + + fn published(&self, shell: &[(&str, &str)], extra: &[&str]) -> String { + let before = fs::read(&self.wasm).unwrap(); + let output = self.publish(shell, extra); + assert!( + output.status.success(), + "publish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(fs::read(&self.wasm).unwrap(), before); + String::from_utf8(output.stdout).unwrap() + } + + fn get(&self, key: &str) -> String { + self.success( + &[ + "env", + "get", + &self.database, + key, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } + + fn list(&self) -> Vec { + let output = self.success( + &[ + "env", + "list", + &self.database, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ); + let mut lines = output.lines().map(str::trim); + assert_eq!( + lines.next().unwrap().split('|').map(str::trim).collect::>(), + ["key", "value"] + ); + lines + .filter(|line| !line.is_empty() && !line.chars().all(|c| matches!(c, '-' | '+'))) + .map(|line| { + let (key, value) = line.split_once('|').unwrap(); + let key: String = serde_json::from_str(key.trim()).unwrap(); + let value: String = serde_json::from_str(value.trim()).unwrap(); + assert_eq!(self.get(&key), format!("{value}\n")); + key + }) + .collect() + } + + fn typed(&self, required: &str, mode: &str, rest: [Option<&str>; 4]) { + let option = |value: Option<&str>| match value { + Some(value) => json!({"some": value}), + None => json!({"none": []}), + }; + let arguments = [ + json!(required), + json!(mode), + option(rest[0]), + option(rest[1]), + option(rest[2]), + option(rest[3]), + ] + .map(|value| value.to_string()); + let mut args = vec![ + "call", + &self.database, + "check_environment", + "--no-config", + "--server", + &self.test.server_url, + ]; + args.extend(arguments.iter().map(String::as_str)); + self.success(&args, &[]); + } + + fn sql(&self, statement: &str) -> Output { + self.command( + &[ + "sql", + &self.database, + statement, + "--server", + &self.test.server_url, + "--no-config", + ], + &[], + ) + } +} + +// Keep ownership through failure/timeout and avoid pipe backpressure. Output is +// generated fixture data; the cap also prevents accidental unbounded diagnostics. +fn bounded_output(mut command: Command) -> Output { + struct OwnedChild(Option); + impl Drop for OwnedChild { + fn drop(&mut self) { + if let Some(child) = self.0.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + let mut stdout = tempfile::tempfile().unwrap(); + let mut stderr = tempfile::tempfile().unwrap(); + let mut child = OwnedChild(Some( + command + .stdout(Stdio::from(stdout.try_clone().unwrap())) + .stderr(Stdio::from(stderr.try_clone().unwrap())) + .spawn() + .unwrap(), + )); + let deadline = Instant::now() + Duration::from_secs(90); + let status = loop { + assert!( + stdout.metadata().unwrap().len() <= 1024 * 1024 && stderr.metadata().unwrap().len() <= 1024 * 1024, + "local ENV CLI output exceeds bound" + ); + if let Some(status) = child.0.as_mut().unwrap().try_wait().unwrap() { + child.0.take(); // Already reaped: never signal this process identifier again. + break status; + } + assert!(Instant::now() < deadline, "local ENV CLI command timed out"); + std::thread::sleep(Duration::from_millis(10)); + }; + stdout.rewind().unwrap(); + stderr.rewind().unwrap(); + let mut out = Vec::new(); + let mut err = Vec::new(); + stdout.read_to_end(&mut out).unwrap(); + stderr.read_to_end(&mut err).unwrap(); + Output { + status, + stdout: out, + stderr: err, + } +} + +#[test] +fn cli_environment_layers_shell_and_exact_precompiled_declarations() { + let f = EnvironmentFixture::new(); + f.write( + "spacetime.json", + json!({"database":"unused-parent", "env":{ + "SMOKE_REQUIRED":"base-required", "SMOKE_MODE":"ready", "SMOKE_OPTIONAL":"base-optional", + "SMOKE_EMPTY":"base-empty", "SMOKE_FLAG": false + }, "children":[{"database":f.database,"env":{"SMOKE_OPTIONAL":"child-optional"}}]}), + ); + // Use a raw JSON number to verify there is no f64 round trip. + fs::write(f.test.project_dir.path().join("spacetime.local.json"), + r#"{"env":{"SMOKE_REQUIRED":"local-required","SMOKE_NUMBER":9007199254740993123456789},"children":[{"env":{"SMOKE_OPTIONAL":"child-local"}}]}"#).unwrap(); + f.write( + "spacetime.prod.json", + json!({"env":{"SMOKE_REQUIRED":"prod-required","SMOKE_MODE":"other"}, + "children":[{"env":{"SMOKE_EMPTY":"child-prod"}}]}), + ); + f.write( + "spacetime.prod.local.json", + json!({"env":{"SMOKE_REQUIRED":"prod-local-required"}, + "children":[{"env":{"SMOKE_OPTIONAL":"child-final"}}]}), + ); + let output = f.published( + &[ + ("SMOKE_REQUIRED", "shell-required"), + ("SMOKE_EMPTY", ""), + ("SMOKE_UNDECLARED", "ambient-not-published"), + ], + &["--env", "prod"], + ); + for key in KEYS { + assert!(output.contains(key)); + } + for value in [ + "shell-required", + "child-final", + "9007199254740993123456789", + "ambient-not-published", + ] { + assert!(!output.contains(value), "publish display leaked a fixture value"); + } + assert!(output.contains("SMOKE_REQUIRED (shell)")); + assert!(output.contains("SMOKE_NUMBER (config)")); + assert_eq!(f.get("SMOKE_EMPTY"), "\n"); + assert_eq!(f.get("SMOKE_NUMBER"), "9007199254740993123456789\n"); + assert_eq!(f.get("SMOKE_FLAG"), "false\n"); + f.typed( + "shell-required", + "other", + [ + Some("child-final"), + Some(""), + Some("9007199254740993123456789"), + Some("false"), + ], + ); + let mut keys = KEYS.to_vec(); + keys.sort_unstable(); + assert_eq!(f.list(), keys); + let initial = f.sql("SELECT required, mode FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("shell-required") && initial.contains("other")); +} + +#[test] +fn cli_environment_replacement_rejection_and_read_only_commands() { + let f = EnvironmentFixture::new(); + f.config(Some( + json!({"SMOKE_REQUIRED":"initial-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":"remove-me"}), + )); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"replacement-sentinel","SMOKE_MODE":"other"}), + )); + f.published(&[], &[]); + f.typed("replacement-sentinel", "other", [None; 4]); + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); + assert!(!f + .command( + &[ + "env", + "get", + &f.database, + "SMOKE_OPTIONAL", + "--server", + &f.test.server_url, + "--no-config" + ], + &[] + ) + .status + .success()); + for input in [ + json!({"SMOKE_MODE":"ready"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"invalid-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","UNKNOWN":"unknown-sentinel"}), + json!({"SMOKE_REQUIRED":"rejected-sentinel","SMOKE_MODE":"ready","SMOKE_OPTIONAL":{}}), + ] { + f.config(Some(input)); + let output = f.publish(&[], &[]); + assert!(!output.status.success()); + for value in ["rejected-sentinel", "invalid-sentinel", "unknown-sentinel"] { + assert!(!String::from_utf8_lossy(&output.stdout).contains(value)); + assert!(!String::from_utf8_lossy(&output.stderr).contains(value)); + } + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + f.typed("replacement-sentinel", "other", [None; 4]); + } + // Invalid local configuration must not prevent an explicit read-only target. + assert_eq!(f.list(), ["SMOKE_MODE", "SMOKE_REQUIRED"]); + for statement in [ + "SET env.SMOKE_REQUIRED = 'bypass'", + "DELETE env.SMOKE_REQUIRED", + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')", + "UPDATE st_env SET value = 'bypass'", + "DELETE FROM st_env", + ] { + assert!(!f.sql(statement).status.success()); + assert_eq!(f.get("SMOKE_REQUIRED"), "replacement-sentinel\n"); + } + for operation in ["set", "delete", "unset"] { + assert!(!f + .command( + &[ + "env", + operation, + &f.database, + "SMOKE_REQUIRED", + "--server", + &f.test.server_url + ], + &[] + ) + .status + .success()); + } +} + +#[test] +fn cli_environment_initial_rejection_clear_and_omitted_payload() { + let mut f = EnvironmentFixture::new(); + f.config(None); + assert!(!f.publish(&[], &[]).status.success()); + f.config(Some(json!({"SMOKE_REQUIRED":"clear-initial","SMOKE_MODE":"ready"}))); + f.published(&[], &[]); + f.config(Some( + json!({"SMOKE_REQUIRED":"clear-replaced","SMOKE_MODE":"other","SMOKE_EMPTY":""}), + )); + f.published(&[], &["--delete-data"]); + f.typed("clear-replaced", "other", [None, Some(""), None, None]); + let initial = f.sql("SELECT required FROM initial_environment"); + assert!(initial.status.success()); + let initial = String::from_utf8(initial.stdout).unwrap(); + assert!(initial.contains("clear-replaced") && !initial.contains("clear-initial")); + // A legacy module with no ENV declaration receives an empty complete input. + f.config(None); + f.wasm = modules::precompiled_module("noop"); + f.published(&[("SMOKE_REQUIRED", "must-not-be-ambient")], &["--delete-data"]); + assert!(f.list().is_empty()); +} diff --git a/crates/smoketests/tests/standalone/cli/mod.rs b/crates/smoketests/tests/standalone/cli/mod.rs index 8c2b6481318..ee04d64547e 100644 --- a/crates/smoketests/tests/standalone/cli/mod.rs +++ b/crates/smoketests/tests/standalone/cli/mod.rs @@ -1,5 +1,6 @@ mod auth; mod dev; +mod environment; mod generate; mod list; mod server; diff --git a/crates/standalone/src/control_db.rs b/crates/standalone/src/control_db.rs index a637d38afcc..9432b6ee097 100644 --- a/crates/standalone/src/control_db.rs +++ b/crates/standalone/src/control_db.rs @@ -1,3 +1,5 @@ +mod environment; + use anyhow::Context; use sled::transaction::{ self, ConflictableTransactionError, ConflictableTransactionResult, TransactionError, TransactionResult, @@ -339,19 +341,18 @@ impl ControlDb { let scan_key: &[u8] = b""; for result in tree.range(scan_key..) { let (_key, value) = result?; - let database = compat::Database::from_slice(&value)?.into(); + let database = self.decode_database(&value)?; databases.push(database); } Ok(databases) } pub fn get_database_by_id(&self, id: u64) -> Result> { - for database in self.get_databases()? { - if database.id == id { - return Ok(Some(database)); - } - } - Ok(None) + self.db + .open_tree("database")? + .get(id.to_be_bytes())? + .map(|bytes| self.decode_database(&bytes)) + .transpose() } pub fn get_database_by_identity(&self, identity: &Identity) -> Result> { @@ -359,12 +360,13 @@ impl ControlDb { let key = identity.to_be_byte_array(); let value = tree.get(&key[..])?; if let Some(value) = value { - let database = compat::Database::from_slice(&value[..])?.into(); + let database = self.decode_database(&value)?; return Ok(Some(database)); } Ok(None) } + #[cfg(test)] pub fn insert_database(&self, mut database: Database) -> Result { let id = self.db.generate_id()?; let tree = self.db.open_tree("database_by_identity")?; @@ -388,23 +390,6 @@ impl ControlDb { Ok(id) } - pub(crate) fn update_database(&self, database: Database) -> Result<()> { - let Some(stored_database) = self.get_database_by_identity(&database.database_identity)? else { - return Err(Error::DatabaseNotFound(database.database_identity)); - }; - - let tree = self.db.open_tree("database_by_identity")?; - let buf = sled::IVec::from(compat::Database::from(database).to_vec()?); - tree.insert(stored_database.database_identity.to_be_byte_array(), buf.clone())?; - tree.flush()?; - - let tree = self.db.open_tree("database")?; - tree.insert(stored_database.id.to_be_bytes(), buf)?; - tree.flush()?; - - Ok(()) - } - pub fn is_database_locked(&self, database_identity: &Identity) -> Result { let tree = self.db.open_tree("database_locks")?; let key = database_identity.to_be_byte_array(); @@ -419,20 +404,7 @@ impl ControlDb { } pub fn delete_database(&self, id: u64) -> Result> { - let tree = self.db.open_tree("database")?; - let tree_by_identity = self.db.open_tree("database_by_identity")?; - - if let Some(old_value) = tree.get(id.to_be_bytes())? { - let database = compat::Database::from_slice(&old_value[..])?; - let key = database.database_identity().to_be_byte_array(); - - tree_by_identity.remove(&key[..])?; - tree.remove(id.to_be_bytes())?; - tree.flush()?; - return Ok(Some(id)); - } - - Ok(None) + self.delete_database_and_environment(id) } pub fn get_replicas(&self) -> Result> { diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs new file mode 100644 index 00000000000..454dd0e1d70 --- /dev/null +++ b/crates/standalone/src/control_db/environment.rs @@ -0,0 +1,330 @@ +//! Private bootstrap inputs, separate from the historical public Database encoding. +//! +//! Creation and reset atomically persist both database indexes, the generation +//! and initial-program binding, the complete ENV input, and the nominated leader. +//! The transaction is flushed before host launch, so a lost request response can +//! recover the same generation through ordinary leader lookup. +//! +//! Reading the input rechecks the persisted owner, program, and generation in +//! the same transaction. A legacy generation with neither metadata nor input has +//! an empty environment; missing input for a recorded generation is an error. +//! +//! The host reads this input only before the database's first initialization. +//! Reopening an initialized database uses its committed program and `st_env`, +//! including later module updates. Reset replaces the bootstrap input, and +//! database deletion removes it atomically with both indexes and the binding. +use super::*; +use spacetimedb_client_api_messages::publish::PublishRequest; +use spacetimedb_lib::Hash; +use std::collections::BTreeMap; + +const METADATA_TREE: &str = "database_bootstrap"; +const VALUES_TREE: &str = "initial_environment"; + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Bootstrap { + version: u8, + database_id: u64, + identity: Identity, + program: Hash, + generation: u64, +} + +fn invalid() -> Error { + Error::Other(anyhow::anyhow!("invalid private bootstrap record")) +} + +impl Bootstrap { + fn decode(bytes: &[u8], database: &Database) -> Result { + if bytes.len() > 1024 { + return Err(invalid()); + } + let value: Self = serde_json::from_slice(bytes).map_err(|_| invalid())?; + if value.version != 1 + || value.database_id != database.id + || value.identity != database.database_identity + || value.program != database.initial_program + || value.generation == 0 + { + return Err(invalid()); + } + Ok(value) + } +} + +impl ControlDb { + pub(super) fn decode_database(&self, bytes: &[u8]) -> Result { + let mut database: Database = compat::Database::from_slice(bytes)?.into(); + if let Some(bytes) = self.db.open_tree(METADATA_TREE)?.get(database.id.to_be_bytes())? { + database.bootstrap_generation = Bootstrap::decode(&bytes, &database)?.generation; + } + Ok(database) + } + + /// Recheck the exact persisted generation and program before releasing any values. + pub(crate) fn initial_environment(&self, database: &Database) -> Result> { + let databases = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &metadata, &values).transaction(|(databases, metadata, values)| { + let persisted = databases + .get(database.database_identity.to_be_byte_array())? + .ok_or_else(|| { + ConflictableTransactionError::Abort(Error::DatabaseNotFound(database.database_identity)) + })?; + let stored: Database = compat::Database::from_slice(&persisted) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != database.id + || stored.initial_program != database.initial_program + || stored.owner_identity != database.owner_identity + { + return transaction::abort(invalid()); + } + match metadata.get(database.id.to_be_bytes())? { + Some(bytes) => { + let binding = + Bootstrap::decode(&bytes, &stored).map_err(ConflictableTransactionError::Abort)?; + if binding.generation != database.bootstrap_generation { + return transaction::abort(invalid()); + } + let bytes = values + .get(database.id.to_be_bytes())? + .ok_or_else(|| ConflictableTransactionError::Abort(invalid()))?; + Ok(Some(bytes)) + } + None if database.bootstrap_generation == 0 => { + if values.get(database.id.to_be_bytes())?.is_some() { + return transaction::abort(invalid()); + } + Ok(None) + } + None => transaction::abort(invalid()), + } + }); + let bytes = result.map_err(transaction_error)?; + match bytes { + None => Ok(BTreeMap::new()), + Some(bytes) => { + let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; + if !request.module.is_empty() { + return Err(invalid()); + } + Ok(request.environment) + } + } + } + + /// Install the desired initial database, its complete private input and a + /// durable leader nomination together. Cancellation before host launch can + /// therefore recover through the ordinary leader lookup. + pub(crate) fn install_database_with_environment( + &self, + mut database: Database, + expected: Option<&Database>, + environment: BTreeMap, + previous_replicas: &[Replica], + ) -> Result<(Database, Replica)> { + if expected.is_none() { + database.id = self.db.generate_id()?; + } + database.bootstrap_generation = expected + .map_or(Some(1), |old| old.bootstrap_generation.checked_add(1)) + .ok_or_else(invalid)?; + let replica = Replica { + id: self.db.generate_id()?, + database_id: database.id, + node_id: 0, + leader: true, + }; + let input = PublishRequest { + module: Vec::new(), + environment, + } + .encode() + .map_err(|_| invalid())?; + let binding = Bootstrap { + version: 1, + database_id: database.id, + identity: database.database_identity, + program: database.initial_program, + generation: database.bootstrap_generation, + }; + let binding = serde_json::to_vec(&binding).map_err(|_| invalid())?; + let encoded = compat::Database::from(database.clone()).to_vec()?; + let encoded_replica = bsatn::to_vec(&replica)?; + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let replicas = self.db.open_tree("replica")?; + let result: TransactionResult<(), Error> = (&databases, &identities, &metadata, &values, &replicas) + .transaction(|(databases, identities, metadata, values, replicas)| { + let identity_key = database.database_identity.to_be_byte_array(); + match (expected, identities.get(identity_key)?) { + (None, None) => {} + (Some(expected), Some(bytes)) => { + let stored: Database = compat::Database::from_slice(&bytes) + .map_err(|_| ConflictableTransactionError::Abort(invalid()))? + .into(); + if stored.id != expected.id + || stored.initial_program != expected.initial_program + || stored.owner_identity != expected.owner_identity + { + return transaction::abort(invalid()); + } + let generation = match metadata.get(stored.id.to_be_bytes())? { + Some(bytes) => { + Bootstrap::decode(&bytes, &stored) + .map_err(ConflictableTransactionError::Abort)? + .generation + } + None => 0, + }; + if generation != expected.bootstrap_generation { + return transaction::abort(invalid()); + } + } + (None, Some(_)) => { + return transaction::abort(Error::DatabaseAlreadyExists(database.database_identity)) + } + (Some(_), None) => return transaction::abort(Error::DatabaseNotFound(database.database_identity)), + } + databases.insert(&database.id.to_be_bytes(), encoded.clone())?; + identities.insert(&identity_key, encoded.clone())?; + metadata.insert(&database.id.to_be_bytes(), binding.clone())?; + values.insert(&database.id.to_be_bytes(), input.clone())?; + for previous in previous_replicas { + if previous.database_id != database.id { + return transaction::abort(invalid()); + } + replicas.remove(&previous.id.to_be_bytes())?; + } + replicas.insert(&replica.id.to_be_bytes(), encoded_replica.clone())?; + databases.flush(); + Ok(()) + }); + result.map_err(transaction_error)?; + self.db.flush()?; + Ok((database, replica)) + } + + pub(super) fn delete_database_and_environment(&self, id: u64) -> Result> { + let databases = self.db.open_tree("database")?; + let identities = self.db.open_tree("database_by_identity")?; + let metadata = self.db.open_tree(METADATA_TREE)?; + let values = self.db.open_tree(VALUES_TREE)?; + let result: TransactionResult, Error> = + (&databases, &identities, &metadata, &values).transaction(|(databases, identities, metadata, values)| { + let Some(bytes) = databases.get(id.to_be_bytes())? else { + return Ok(None); + }; + let database = + compat::Database::from_slice(&bytes).map_err(|_| ConflictableTransactionError::Abort(invalid()))?; + identities.remove(&database.database_identity().to_be_byte_array())?; + databases.remove(&id.to_be_bytes())?; + metadata.remove(&id.to_be_bytes())?; + values.remove(&id.to_be_bytes())?; + databases.flush(); + Ok(Some(id)) + }); + let result = result.map_err(transaction_error)?; + self.db.flush()?; + Ok(result) + } +} + +fn transaction_error(error: TransactionError) -> Error { + match error { + TransactionError::Abort(error) => error, + TransactionError::Storage(error) => error.into(), + } +} + +#[async_trait::async_trait] +impl spacetimedb::host::InitialEnvironmentSource for ControlDb { + async fn load(&self, database: &Database) -> anyhow::Result> { + let source = self.clone(); + let database = database.clone(); + spacetimedb::util::asyncify(move || source.initial_environment(&database)) + .await + .map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb::messages::control_db::HostType; + + fn database() -> Database { + Database { + id: 0, + database_identity: Identity::ZERO, + owner_identity: Identity::ZERO, + host_type: HostType::Wasm, + initial_program: spacetimedb_lib::hash_bytes(b"module"), + bootstrap_generation: 0, + } + } + + #[test] + fn bootstrap_is_durable_atomic_generation_bound_and_deleted_with_database() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let (original, original_replica) = { + let control = ControlDb::at(temp.path())?; + control.install_database_with_environment( + database(), + None, + BTreeMap::from([("VALUE".into(), "first".into())]), + &[], + )? + }; + let control = ControlDb::at(temp.path())?; + let loaded = control.get_database_by_id(original.id)?.unwrap(); + assert_eq!(loaded.bootstrap_generation, 1); + assert_eq!(control.initial_environment(&loaded)?["VALUE"], "first"); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + original_replica.id + ); + let (replaced, new_replica) = control.install_database_with_environment( + loaded.clone(), + Some(&loaded), + BTreeMap::new(), + &[original_replica], + )?; + assert_eq!(replaced.bootstrap_generation, 2); + assert!(control.initial_environment(&original).is_err()); + assert!(control.initial_environment(&replaced)?.is_empty()); + assert_eq!(control.get_replicas_by_database(loaded.id)?.len(), 1); + assert_eq!( + control.get_leader_replica_by_database(loaded.id).unwrap().id, + new_replica.id + ); + assert!(control + .install_database_with_environment(loaded.clone(), Some(&loaded), BTreeMap::new(), &[]) + .is_err()); + assert_eq!(control.get_database_by_id(loaded.id)?.unwrap().bootstrap_generation, 2); + control.delete_database(loaded.id)?; + assert!(control.initial_environment(&replaced).is_err()); + assert!(control.db.open_tree(VALUES_TREE)?.is_empty()); + Ok(()) + } + + #[test] + fn legacy_absence_is_empty_but_missing_new_values_fails_closed() -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let control = ControlDb::at(temp.path())?; + let id = control.insert_database(database())?; + let legacy = control.get_database_by_id(id)?.unwrap(); + assert!(control.initial_environment(&legacy)?.is_empty()); + let (new, _) = + control.install_database_with_environment(legacy.clone(), Some(&legacy), BTreeMap::new(), &[])?; + control.db.open_tree(VALUES_TREE)?.remove(new.id.to_be_bytes())?; + assert!(control.initial_environment(&new).is_err()); + Ok(()) + } +} diff --git a/crates/standalone/src/environment_tests.rs b/crates/standalone/src/environment_tests.rs new file mode 100644 index 00000000000..abeb5a84193 --- /dev/null +++ b/crates/standalone/src/environment_tests.rs @@ -0,0 +1,190 @@ +//! Actual module persistence with explicit local input, without a server or CLI configuration. +use super::*; +use spacetimedb::host::FunctionArgs; +use spacetimedb_client_api::{ControlStateWriteAccess as _, DatabaseDef}; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue}; +use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; +use spacetimedb_paths::FromPathUnchecked; +use std::collections::BTreeMap; + +type Values = BTreeMap; + +async fn read(env: &StandaloneEnv, database: u64, key: &str) -> anyhow::Result { + let module = env.leader(database).await?.module().await?; + Ok(module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key])?.into()), + ) + .await + .result? + .return_val) +} + +#[tokio::test] +#[ignore = "requires an explicitly configured local environment-test Wasm artifact"] +async fn real_module_reopen_and_no_artifact_reset_preserve_complete_environment_semantics() -> anyhow::Result<()> { + let module_path = std::path::PathBuf::from( + std::env::var_os("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE") + .context("SPACETIMEDB_ENV_STANDALONE_TEST_MODULE must name the owned local fixture")?, + ); + anyhow::ensure!(module_path.is_absolute(), "fixture path must be absolute"); + let bytes = axum::body::Bytes::from(std::fs::read(module_path)?); + anyhow::ensure!(bytes.starts_with(b"\0asm"), "fixture must be a Wasm module"); + let temp = tempfile::tempdir()?; + let keys = temp.path().join("keys"); + std::fs::create_dir(&keys)?; + let ca = CertificateAuthority { + jwt_pub_key_path: PubKeyPath(keys.join("public")), + jwt_priv_key_path: PrivKeyPath(keys.join("private")), + }; + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(temp.path().join("data"))); + data_dir.create()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + let env = StandaloneEnv::init(config, &ca, data_dir, JobCores::without_pinned_cores()).await?; + let test_env = env.clone(); + let run = tokio::spawn(async move { + let env = test_env; + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + let spec = |environment| DatabaseDef { + database_identity: Identity::ZERO, + program_bytes: bytes.clone(), + environment, + num_replicas: None, + host_type: HostType::Wasm, + parent: None, + organization: None, + }; + log::info!("ENV standalone fixture: initial publication"); + assert!(env + .publish_database(&Identity::ZERO, spec(initial.clone()), MigrationPolicy::Compatible) + .await? + .is_none()); + let database = env.control_db.get_database_by_identity(&Identity::ZERO)?.unwrap(); + let replica = env.control_db.get_leader_replica_by_database(database.id).unwrap(); + log::info!("ENV standalone fixture: rejected publication preserves live host"); + let rejected = env + .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .await; + assert!( + rejected.as_ref().is_err() + || rejected + .as_ref() + .unwrap() + .as_ref() + .is_some_and(|result| !result.was_successful()) + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("initial-required".to_owned())) + ); + log::info!("ENV standalone fixture: same-program update"); + let mut updated = initial.clone(); + updated.insert("REQUIRED".into(), "republished".into()); + assert!(env + .publish_database(&Identity::ZERO, spec(updated), MigrationPolicy::Compatible) + .await? + .unwrap() + .was_successful()); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // The old private bootstrap input intentionally remains distinct. Ordinary + // reopen must consult persisted st_module/st_env and skip that input. + assert_eq!( + env.control_db.initial_environment(&database)?["REQUIRED"], + "initial-required" + ); + log::info!("ENV standalone fixture: positive close and normal reopen"); + let replica_id = replica.id; + env.own_publication( + move |owner| async move { owner.host_controller.exit_module_host_and_join(replica_id).await }, + ) + .await?; + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + // Init asserts initial-required. A successful reopen with republished proves + // that init was not run again and old bootstrap input was not restored. + assert!(env + .reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: Values::new(), + num_replicas: None, + host_type: None, + } + ) + .await + .is_err()); + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 1 + ); + assert_eq!( + read(&env, database.id, "REQUIRED").await?, + AlgebraicValue::from(Some("republished".to_owned())) + ); + log::info!("ENV standalone fixture: complete no-artifact reset"); + let mut reset = initial; + reset.insert("EMPTY".into(), "reset-input".into()); + env.reset_database( + &Identity::ZERO, + DatabaseResetDef { + database_identity: Identity::ZERO, + program_bytes: None, + environment: reset, + num_replicas: None, + host_type: None, + }, + ) + .await?; + assert_eq!( + env.control_db + .get_database_by_id(database.id)? + .unwrap() + .bootstrap_generation, + 2 + ); + assert_ne!( + env.control_db.get_leader_replica_by_database(database.id).unwrap().id, + replica.id + ); + assert_eq!( + read(&env, database.id, "EMPTY").await?, + AlgebraicValue::from(Some("reset-input".to_owned())) + ); + anyhow::Ok(()) + }) + .await; + // Join physical cleanup even if an assertion in the accepted fixture task + // panicked. Never let a failed test detach its database host. + log::info!("ENV standalone fixture: positive cleanup"); + env.delete_database(&Identity::ZERO, &Identity::ZERO).await?; + run??; + assert!(env.control_db.get_database_by_identity(&Identity::ZERO)?.is_none()); + Ok(()) +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index a79f9814f5d..c0ad9638d92 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -1,4 +1,6 @@ mod control_db; +#[cfg(test)] +mod environment_tests; pub mod subcommands; pub mod util; pub mod version; @@ -16,7 +18,7 @@ use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; -use spacetimedb::messages::control_db::{Database, Node, Replica}; +use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::metrics::ENGINE_METRICS; use spacetimedb::subscription::row_list_builder_pool::BsatnRowListBuilderPool; use spacetimedb::util::jobs::JobCores; @@ -34,7 +36,8 @@ use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; -use std::sync::Arc; +use std::sync::{Arc, Weak}; +#[cfg(test)] use std::time::Duration; pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL}; @@ -51,6 +54,8 @@ pub struct StandaloneOptions { pub struct StandaloneEnv { control_db: ControlDb, + publication_lock: Arc>, + weak_self: Weak, program_store: Arc, host_controller: HostController, client_actor_index: ClientActorIndex, @@ -90,7 +95,8 @@ impl StandaloneEnv { Arc::new(()), persistence_provider, db_cores, - ); + ) + .with_initial_environment_source(Arc::new(control_db.clone())); let client_actor_index = ClientActorIndex::new(); let jwt_keys = certs.get_or_create_keys()?; @@ -102,8 +108,10 @@ impl StandaloneEnv { metrics_registry.register(Box::new(&*DB_METRICS)).unwrap(); metrics_registry.register(Box::new(&*DATA_SIZE_METRICS)).unwrap(); - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|weak_self| Self { control_db, + publication_lock: Arc::new(tokio::sync::RwLock::new(())), + weak_self: weak_self.clone(), program_store, host_controller, client_actor_index, @@ -176,20 +184,14 @@ impl NodeDelegate for StandaloneEnv { } async fn leader(&self, database_id: u64) -> Result { - let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { - return Err(GetLeaderHostError::NoSuchReplica); - }; - - let Some(database) = self.control_db.get_database_by_id(database_id)? else { - return Err(GetLeaderHostError::NoSuchDatabase); - }; - - self.host_controller - .get_or_launch_module_host(database, leader.id) - .await - .map_err(|source| GetLeaderHostError::LaunchError { source })?; - - Ok(Host::new(leader.id, self.host_controller.clone())) + let guard = self.publication_lock.clone().read_owned().await; + let owner = self.weak_self.upgrade().expect("standalone owner exists during lookup"); + tokio::spawn(async move { + let _guard = guard; + owner.leader_with_publication_lock_held(database_id).await + }) + .await + .map_err(|error| GetLeaderHostError::LaunchError { source: error.into() })? } fn module_logs_dir(&self, replica_id: u64) -> ModuleLogsDir { @@ -276,6 +278,157 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { publisher: &Identity, spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, + ) -> anyhow::Result> { + let publisher = *publisher; + self.own_publication(move |owner| async move { owner.publish_database_owned(&publisher, spec, policy).await }) + .await + } + + async fn migrate_plan( + &self, + spec: spacetimedb_client_api::DatabaseDef, + style: PrettyPrintStyle, + ) -> anyhow::Result { + let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; + + match existing_db { + Some(db) => { + let host = self.leader(db.id).await?; + self.host_controller + .migrate_plan( + db, + spec.host_type, + host.replica_id, + spec.program_bytes.to_vec().into(), + style, + ) + .await + } + None => anyhow::bail!( + "Database `{}` does not exist", + spec.database_identity.to_abbreviated_hex() + ), + } + } + + async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + let database_identity = *database_identity; + self.own_publication(move |owner| async move { + owner.delete_database_owned(&caller_identity, &database_identity).await + }) + .await + } + + async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let caller_identity = *_caller_identity; + self.own_publication(move |owner| async move { owner.reset_database_owned(&caller_identity, spec).await }) + .await + } + + async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { + let balance = self + .control_db + .get_energy_balance(identity)? + .unwrap_or(EnergyBalance::ZERO); + + let balance = balance.saturating_add_energy(amount); + + self.control_db.set_energy_balance(*identity, balance)?; + Ok(()) + } + async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { + // The energy balance code is obsolete. + Ok(()) + } + + async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { + Ok(self.control_db.spacetime_register_tld(tld, *identity)?) + } + + async fn create_dns_record( + &self, + owner_identity: &Identity, + domain: &DomainName, + database_identity: &Identity, + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) + } + + async fn replace_dns_records( + &self, + database_identity: &Identity, + owner_identity: &Identity, + domain_names: &[DomainName], + ) -> anyhow::Result { + Ok(self + .control_db + .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) + } + + async fn set_database_lock( + &self, + _caller_identity: &Identity, + database_identity: &Identity, + locked: bool, + ) -> anyhow::Result<()> { + let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { + anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); + }; + self.control_db.set_database_lock(database_identity, locked)?; + Ok(()) + } +} + +impl StandaloneEnv { + /// Admission and completion are owned together. Losing an HTTP waiter cannot + /// release the reset fence while HostController still owns accepted work. + async fn own_publication(&self, operation: F) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, + Fut: std::future::Future> + Send + 'static, + { + let guard = self.publication_lock.clone().write_owned().await; + let owner = self + .weak_self + .upgrade() + .expect("standalone owner exists during publication"); + tokio::spawn(async move { + let _guard = guard; + operation(owner).await + }) + .await? + } + + /// Look up or start the current leader while the caller holds the publication + /// lock. Retain the read or write guard through this future's completion. + /// Ordinary lookup holds a read guard; publication and reset hold a write + /// guard, so calling `leader()` here would acquire the lock again and deadlock. + async fn leader_with_publication_lock_held(&self, database_id: u64) -> Result { + let Some(leader) = self.control_db.get_leader_replica_by_database(database_id) else { + return Err(GetLeaderHostError::NoSuchReplica); + }; + + let Some(database) = self.control_db.get_database_by_id(database_id)? else { + return Err(GetLeaderHostError::NoSuchDatabase); + }; + + self.host_controller + .get_or_launch_module_host(database, leader.id) + .await + .map_err(|source| GetLeaderHostError::LaunchError { source })?; + + Ok(Host::new(leader.id, self.host_controller.clone())) + } + + async fn publish_database_owned( + &self, + publisher: &Identity, + spec: spacetimedb_client_api::DatabaseDef, + policy: MigrationPolicy, ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; @@ -301,28 +454,42 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { // Instantiate a temporary database in order to check that the module is valid. // This will e.g. typecheck RLS filters. self.host_controller - .check_module_validity(database.clone(), program) + .check_module_validity_with_environment(database.clone(), program, spec.environment.clone()) .await?; let program_hash = self.program_store.put(&spec.program_bytes).await?; debug_assert_eq!(_hash_for_assert, program_hash); - let database_id = self.control_db.insert_database(database)?; - - self.schedule_replicas(database_id, num_replicas).await?; + let (database, replica) = + self.control_db + .install_database_with_environment(database, None, spec.environment, &[])?; + // The leader nomination and input are durable already. If this + // waiter is cancelled, ordinary lookup resumes the same input. + self.on_insert_replica(&replica).await?; + debug_assert_eq!(database.id, replica.database_id); Ok(None) } // The database already exists, so we'll try to update it. // If that fails, we'll keep the old one. Some(database) => { + anyhow::ensure!( + database.owner_identity == *publisher, + "database ownership changed before publication" + ); let database_id = database.id; let database_identity = database.database_identity; - let leader = self.leader(database_id).await?; + let leader = self.leader_with_publication_lock_held(database_id).await?; let update_result = leader - .update(database, spec.host_type, spec.program_bytes.to_vec().into(), policy) + .update_with_environment( + database, + spec.host_type, + spec.program_bytes.to_vec().into(), + policy, + spec.environment, + ) .await?; if update_result.was_successful() { let replicas = self.control_db.get_replicas_by_database(database_id)?; @@ -372,37 +539,18 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { } } - async fn migrate_plan( + async fn delete_database_owned( &self, - spec: spacetimedb_client_api::DatabaseDef, - style: PrettyPrintStyle, - ) -> anyhow::Result { - let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; - - match existing_db { - Some(db) => { - let host = self.leader(db.id).await?; - self.host_controller - .migrate_plan( - db, - spec.host_type, - host.replica_id, - spec.program_bytes.to_vec().into(), - style, - ) - .await - } - None => anyhow::bail!( - "Database `{}` does not exist", - spec.database_identity.to_abbreviated_hex() - ), - } - } - - async fn delete_database(&self, _caller_identity: &Identity, database_identity: &Identity) -> anyhow::Result<()> { + caller_identity: &Identity, + database_identity: &Identity, + ) -> anyhow::Result<()> { let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { return Ok(()); }; + anyhow::ensure!( + database.owner_identity == *caller_identity, + "database ownership changed before deletion" + ); self.control_db.delete_database(database.id)?; for instance in self.control_db.get_replicas_by_database(database.id)? { @@ -412,93 +560,55 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { Ok(()) } - async fn reset_database(&self, _caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { - let mut database = self + async fn reset_database_owned(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + let previous = self .control_db .get_database_by_identity(&spec.database_identity)? .with_context(|| format!("Database `{}` does not exist", spec.database_identity))?; - let database_id = database.id; - - if let Some(program) = spec.program_bytes { - if let Some(host_type) = spec.host_type { - database.host_type = host_type; + anyhow::ensure!( + previous.owner_identity == *caller_identity, + "database ownership changed before reset" + ); + let mut database = previous.clone(); + let program = match spec.program_bytes { + Some(bytes) => { + let host_type = spec.host_type.unwrap_or(database.host_type); + Program::from_bytes(host_type.into(), &bytes[..]) + } + None => { + // A reset without an artifact retains the currently committed + // module, not the original bootstrap program or its old values. + let module = self + .leader_with_publication_lock_held(database.id) + .await? + .module() + .await?; + module + .relational_db() + .program()? + .context("database is not initialized")? } - let program_bytes = &program[..]; - let program = Program::from_bytes(database.host_type.into(), program_bytes); - let _hash_for_assert = program.hash; - - database.initial_program = program.hash; - - self.host_controller - .check_module_validity(database.clone(), program) - .await?; - let _stored_hash_for_assert = self.program_store.put(program_bytes).await?; - debug_assert_eq!(_hash_for_assert, _stored_hash_for_assert); - } - self.control_db.update_database(database)?; - - for instance in self.control_db.get_replicas_by_database(database_id)? { - self.delete_replica(instance.id).await?; - } - // Standalone only support a single replica. - let num_replicas = 1; - self.schedule_replicas(database_id, num_replicas).await?; - - Ok(()) - } - - async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { - let balance = self - .control_db - .get_energy_balance(identity)? - .unwrap_or(EnergyBalance::ZERO); - - let balance = balance.saturating_add_energy(amount); - - self.control_db.set_energy_balance(*identity, balance)?; - Ok(()) - } - async fn withdraw_energy(&self, _identity: &Identity, _amount: EnergyQuanta) -> anyhow::Result<()> { - // The energy balance code is obsolete. - Ok(()) - } - - async fn register_tld(&self, identity: &Identity, tld: Tld) -> anyhow::Result { - Ok(self.control_db.spacetime_register_tld(tld, *identity)?) - } - - async fn create_dns_record( - &self, - owner_identity: &Identity, - domain: &DomainName, - database_identity: &Identity, - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_insert_domain(database_identity, domain.clone(), *owner_identity, true)?) - } - - async fn replace_dns_records( - &self, - database_identity: &Identity, - owner_identity: &Identity, - domain_names: &[DomainName], - ) -> anyhow::Result { - Ok(self - .control_db - .spacetime_replace_domains(database_identity, owner_identity, domain_names)?) - } - - async fn set_database_lock( - &self, - _caller_identity: &Identity, - database_identity: &Identity, - locked: bool, - ) -> anyhow::Result<()> { - let Some(_database) = self.control_db.get_database_by_identity(database_identity)? else { - anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); }; - self.control_db.set_database_lock(database_identity, locked)?; + database.host_type = HostType::from(program.kind); + database.initial_program = program.hash; + self.host_controller + .check_module_validity_with_environment(database.clone(), program.clone(), spec.environment.clone()) + .await?; + let stored = self.program_store.put(&program.bytes).await?; + anyhow::ensure!(stored == program.hash, "stored reset program changed"); + let previous_replicas = self.control_db.get_replicas_by_database(database.id)?; + // Keep old nominations until all requested closes succeed. The owned + // publication guard excludes leader admission across close and commit. + for replica in &previous_replicas { + self.on_delete_replica(replica.id).await?; + } + let (_, replica) = self.control_db.install_database_with_environment( + database, + Some(&previous), + spec.environment, + &previous_replicas, + )?; + self.on_insert_replica(&replica).await?; Ok(()) } } @@ -567,33 +677,16 @@ impl StandaloneEnv { Ok(()) } - async fn schedule_replicas(&self, database_id: u64, num_replicas: u8) -> Result<(), anyhow::Error> { - // Just scheduling a bunch of replicas to the only machine - for i in 0..num_replicas { - let replica = Replica { - id: 0, - database_id, - node_id: 0, - leader: i == 0, - }; - self.insert_replica(replica).await?; - } - - Ok(()) - } - async fn on_insert_replica(&self, instance: &Replica) -> Result<(), anyhow::Error> { if instance.leader { - let database = self - .control_db - .get_database_by_id(instance.database_id)? + self.leader_with_publication_lock_held(instance.database_id) + .await .with_context(|| { format!( - "unknown database: id: {}, instance: {}", + "failed to start leader for database {}, replica {}", instance.database_id, instance.id ) })?; - self.leader(database.id).await?; } Ok(()) @@ -604,9 +697,7 @@ impl StandaloneEnv { // replicas which have been deleted. This will just drop // them from memory, but will not remove them from disk. We need // some kind of database lifecycle manager long term. - self.host_controller - .exit_module_host(replica_id, Duration::from_secs(30)) - .await?; + self.host_controller.exit_module_host_and_join(replica_id).await?; Ok(()) } @@ -690,4 +781,74 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cancelled_publication_waiter_keeps_mutations_and_leader_admission_fenced() -> Result<()> { + let tempdir = TempDir::new()?; + // Use one subdir for keys and another for the data dir. + let keys = tempdir.path().join("keys"); + let root = tempdir.path().join("data"); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(root)); + + fs::create_dir(&keys)?; + data_dir.create()?; + + let pub_key = PubKeyPath(keys.join("public")); + let priv_key = PrivKeyPath(keys.join("private")); + let ca = CertificateAuthority { + jwt_pub_key_path: pub_key, + jwt_priv_key_path: priv_key, + }; + + // Create the keys. + ca.get_or_create_keys()?; + let config = StandaloneOptions { + db_config: db::Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + durability: DurabilityConfig::default(), + websocket: WebSocketOptions::default(), + module_http: ModuleHttpConfig::default(), + wasm: WasmConfig::default(), + v8: V8Config::default(), + }; + + let env = StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?; + + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let owner = env.clone(); + let waiter = tokio::spawn(async move { + owner + .own_publication(move |_owner| async move { + let _ = started.send(()); + release_rx.await?; + Ok(()) + }) + .await + }); + started_rx.await?; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + + // Both a later mutation and ordinary leader lookup must stay behind the + // accepted operation even after its request future has disappeared. + let next_owner = env.clone(); + let mut next = tokio::spawn(async move { next_owner.own_publication(|_| async { Ok(()) }).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut next) + .await + .is_err()); + let read_owner = env.clone(); + let mut reader = tokio::spawn(async move { read_owner.leader(u64::MAX).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut reader) + .await + .is_err()); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(5), next).await???; + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), reader).await??, + Err(GetLeaderHostError::NoSuchReplica) + )); + Ok(()) + } } diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 13cb9872b96..63799d77e72 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -49,6 +49,55 @@ pub fn invoke_cli(paths: &SpacetimePaths, args: &[&str]) { } // If CUSTOM_SPACETIMEDB_PATH is missing, fall through to the default behavior. + if cmd == "publish" { + // Publishing inspects exact module bytes in a child process. This + // function is linked into a libtest under target/*/deps, so current_exe + // cannot locate the standalone companion of the actual runtime CLI. + // Use the same explicit build artifacts as the test server guard. + let cli = spacetimedb_guard::ensure_binaries_built(); + let inspector = cli + .with_file_name("spacetimedb-standalone") + .with_extension(std::env::consts::EXE_EXTENSION); + assert!( + inspector.is_file(), + "SDK tests require the standalone schema inspector beside the CLI: {}", + inspector.display() + ); + assert!( + sub_args.get_one::("server").is_some(), + "SDK publication must use its explicit test server" + ); + let root = paths + .to_root_dir() + .expect("SDK tests require an isolated root directory"); + let status = RUNTIME.block_on(async { + let mut child = tokio::process::Command::new(cli) + .arg("--root-dir") + .arg(root) + .arg("--config-path") + .arg(paths.cli_config_dir.cli_toml()) + .args(args) + .arg("--no-config") + .env("SPACETIMEDB_SCHEMA_EXTRACTOR", inspector) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("Failed to start the pre-built publish CLI"); + // Output streams directly to the test log, without accumulating a + // second buffer. The inspector retains its own tighter bounds. + match tokio::time::timeout(std::time::Duration::from_secs(120), child.wait()).await { + Ok(status) => status.expect("Failed to reap the pre-built publish CLI"), + Err(_) => { + let _ = child.start_kill(); + child.wait().await.expect("Failed to reap timed-out publish CLI"); + panic!("SDK module publication timed out"); + } + } + }); + assert!(status.success(), "SDK module publication failed"); + return; + } + // Default: run in-process CLI (fast/path-friendly for tests). let config = Config::new_with_localhost(paths.cli_config_dir.cli_toml()); RUNTIME diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 465e22c3d6e..09d35280562 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::env; use std::future::Future; use std::panic::AssertUnwindSafe; @@ -67,6 +68,46 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Publish a complete configuration through the standalone control API. + pub async fn republish_environment( + &self, + environment: BTreeMap, + ) -> anyhow::Result { + let program = self + .client + .module() + .relational_db() + .program()? + .expect("published module"); + self.republish_program(program.bytes.into(), program.kind.into(), environment) + .await + } + + /// Publish exact replacement artifact bytes and their complete configuration. + pub async fn republish_program( + &self, + program_bytes: Bytes, + host_type: HostType, + environment: BTreeMap, + ) -> anyhow::Result { + self.env + .publish_database( + &Identity::ZERO, + DatabaseDef { + database_identity: self.db_identity, + program_bytes, + environment, + num_replicas: None, + host_type, + parent: None, + organization: None, + }, + MigrationPolicy::Compatible, + ) + .await? + .ok_or_else(|| anyhow::anyhow!("expected an update to the existing database")) + } + async fn call_reducer_result(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result { let result = self .client @@ -235,6 +276,18 @@ pub enum CompilationMode { } impl CompiledModule { + /// Use an artifact built by an external toolchain, such as Linux NativeAOT. + /// This changes only compilation; publication and execution still use the + /// same in-process standalone server as locally compiled fixtures. + pub fn from_artifact(name: &str, host_type: HostType, path: PathBuf) -> Self { + Self { + name: name.to_owned(), + path, + host_type, + program_bytes: OnceLock::new(), + } + } + pub fn compile(name: &str, mode: CompilationMode) -> Self { let (path, host_type) = spacetimedb_cli::build( &module_path(name), @@ -245,12 +298,7 @@ impl CompiledModule { None, ) .expect("Module compilation failed"); - Self { - name: name.to_owned(), - path, - host_type: host_type.parse().unwrap(), - program_bytes: OnceLock::new(), - } + Self::from_artifact(name, host_type.parse().unwrap(), path) } pub fn path(&self) -> &Path { @@ -279,10 +327,22 @@ impl CompiledModule { where R: FnOnce(ModuleHandle) -> F, F: Future, + { + self.with_module_async_with_environment(config, BTreeMap::new(), routine) + } + + pub fn with_module_async_with_environment( + &self, + config: Config, + environment: BTreeMap, + routine: R, + ) where + R: FnOnce(ModuleHandle) -> F, + F: Future, { with_runtime(move |runtime| { runtime.block_on(async { - let module = self.load_module(config, None).await; + let module = self.load_module_with_environment(config, None, environment).await; let env = module.env.clone(); let db_identity = module.db_identity; let routine_result = AssertUnwindSafe(routine(module)).catch_unwind().await.map(drop); @@ -311,6 +371,16 @@ impl CompiledModule { /// without resetting the database. /// This is used to speed up benchmarks running under callgrind (it allows them to reuse native-compiled wasm modules). pub async fn load_module(&self, config: Config, reuse_db_path: Option<&RootDir>) -> ModuleHandle { + self.load_module_with_environment(config, reuse_db_path, BTreeMap::new()) + .await + } + + pub async fn load_module_with_environment( + &self, + config: Config, + reuse_db_path: Option<&RootDir>, + environment: BTreeMap, + ) -> ModuleHandle { let paths = match reuse_db_path { Some(path) => SpacetimePaths::from_root_dir(path), None => { @@ -350,6 +420,7 @@ impl CompiledModule { DatabaseDef { database_identity: db_identity, program_bytes: self.program_bytes(), + environment, num_replicas: None, host_type: self.host_type, parent: None, diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs new file mode 100644 index 00000000000..a8187619fca --- /dev/null +++ b/crates/testing/tests/environment.rs @@ -0,0 +1,505 @@ +//! Actual publication and module calls exercise configuration atomicity and ABI enforcement. +use serial_test::serial; +use spacetimedb::client::{messages::SerializableMessage, OutboundMessage}; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_client_api_messages::websocket::v1 as ws_v1; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, ModuleHandle, DEFAULT_CONFIG}; +use std::collections::BTreeMap; +use std::time::Duration; + +type Values = BTreeMap; + +async fn sql(module: &ModuleHost, statement: &str) -> Vec { + spacetimedb::sql::execute::run( + module.relational_db().clone(), + statement.to_string(), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await + .unwrap() + .rows +} + +async fn publish(handle: &ModuleHandle, values: &Values) -> ModuleHost { + let result = handle.republish_environment(values.clone()).await.unwrap(); + assert!(result.was_successful(), "configuration publication failed"); + handle.client.module() +} + +async fn read(module: &ModuleHost, key: &str) -> AlgebraicValue { + module + .call_procedure( + Identity::ZERO, + None, + None, + "read_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()), + ) + .await + .result + .unwrap() + .return_val +} + +async fn next_message(handle: &mut ModuleHandle) -> OutboundMessage { + tokio::time::timeout(Duration::from_secs(10), handle.recv_message()) + .await + .expect("timed out waiting for environment subscription update") + .expect("environment subscription disconnected") +} + +async fn expect_view_update(handle: &mut ModuleHandle) { + let message = next_message(handle).await; + assert!(matches!(message, OutboundMessage::V1(SerializableMessage::TxUpdate(_)))); + // Replacing the one-row view must send both the old row's deletion and + // the new row's insertion to an already connected subscriber. + assert_eq!(message.num_rows(), Some(2)); +} + +async fn check_submodule_scope(handle: &mut ModuleHandle, values: &mut Values) { + values.insert("EMPTY".into(), "root-visible".into()); + let module = publish(handle, values).await; + assert!(module.info.module_def.reducer_by_name("lib.env_read_reducer").is_some()); + let child = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "lib.env_read_reducer", + FunctionArgs::Nullary, + ) + .await; + assert!(child.is_err() || child.unwrap().outcome.into_result().is_err()); + for procedure in ["lib.env_read_procedure", "lib.env_read_in_tx"] { + assert!(module.info.module_def.procedure_by_name(procedure).is_some()); + let result = module + .call_procedure(Identity::ZERO, None, None, procedure, FunctionArgs::Nullary) + .await; + assert!(result.result.is_err(), "submodule procedure read the root environment"); + } + for view in ["lib.env_read_view", "lib.env_read_sql_view", "env_read_root_sql_view"] { + assert!(module.info.module_def.view_by_name_with_module(view).is_some()); + let result = spacetimedb::sql::execute::run( + module.relational_db().clone(), + format!("SELECT * FROM {view}"), + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await; + // Host access must be denied, independently of the general view-error + // transport contract tracked in #5912. A failed view may yield no rows. + match result { + Ok(result) => assert!(result.rows.is_empty(), "forbidden view exposed rows"), + Err(error) => { + let error = format!("{error:#}"); + assert!(!error.contains("not found"), "view failed before dispatch: {error}"); + assert!(!error.contains("root-visible"), "view error exposed environment data"); + } + } + } + + // HTTP routes are root entries today. Calling an exported child callback + // as an ordinary helper retains that root entry's authority. + assert_eq!( + &handle.call_http_route_get("/env-child").await.unwrap()[..], + b"root-visible" + ); +} + +// Qualification can pin locally built inputs without invoking a nested build. +// Ordinary test runs keep the existing compilation path when no pin is supplied. +fn compiled_fixture(name: &str) -> CompiledModule { + use spacetimedb::messages::control_db::HostType; + let input = match name { + "environment-test" => Some(("SPACETIMEDB_ENV_RUST_MODULE", HostType::Wasm)), + "module-test-ts" => Some(("SPACETIMEDB_ENV_TYPESCRIPT_MODULE", HostType::Js)), + "module-test-cs" => Some(("SPACETIMEDB_ENV_CSHARP_MODULE", HostType::Wasm)), + _ => None, + }; + if let Some((key, host_type)) = input + && let Some(path) = std::env::var_os(key) + { + let path = std::path::PathBuf::from(path); + assert!(path.is_absolute() && path.is_file(), "invalid explicit module artifact"); + CompiledModule::from_artifact(name, host_type, path) + } else { + CompiledModule::compile(name, CompilationMode::Debug) + } +} + +fn exercise_fixture(name: &str) { + let initial = if name == "environment-test" { + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]) + } else { + Values::new() + }; + let compiled = compiled_fixture(name); + compiled.with_module_async_with_environment(DEFAULT_CONFIG, initial.clone(), |mut handle| async move { + let mut values = initial; + for (key, expected) in [ + ("MISSING", None), + ("EMPTY", Some("".to_string())), + ("UTF8", Some("hΓ©llo 🌍".to_string())), + ("NUL", Some("before\0after".to_string())), + ("MAXIMUM", Some("Γ©".repeat(4096))), + ] { + if let Some(value) = &expected { + values.insert(key.into(), value.clone()); + } + let module = publish(&handle, &values).await; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, expected.clone()]).unwrap().into()), + ) + .await + .unwrap(); + result.outcome.into_result().unwrap(); + assert_eq!(read(&module, key).await, AlgebraicValue::from(expected.clone())); + if expected.is_some() { + values.insert(key.into(), "updated".into()); + let module = publish(&handle, &values).await; + assert_eq!( + read(&module, key).await, + AlgebraicValue::from(Some("updated".to_string())) + ); + values.remove(key); + let module = publish(&handle, &values).await; + assert_eq!(read(&module, key).await, AlgebraicValue::from(None::)); + } + } + if name == "environment-test" { + // Required values cannot be inherited from the previous publish, + // and an invalid literal cannot replace the previous configuration. + for invalid in [ + Values::new(), + Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "invalid-secret-marker".into()), + ]), + ] { + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("initial-required".to_string())) + ); + } + // A same-module publish does not run init again. Its new required + // value deliberately differs from the value init asserted. + values.insert("REQUIRED".into(), "republished".into()); + values.insert("HANDLER".into(), "handler snapshot".into()); + let module = publish(&handle, &values).await; + let (_, body) = module + .call_http_handler( + module.info.module_def.http_handler_ids_and_defs().next().unwrap().0, + spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: "/environment".into(), + version: spacetimedb_lib::http::Version::Http11, + }, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(&body[..], b"handler snapshot"); + let view = "SELECT * FROM environment_value"; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + let subscribe = ws_v1::ClientMessage::::Subscribe(ws_v1::Subscribe { + query_strings: [view.into()].into(), + request_id: 71, + }); + handle.send(bsatn::to_vec(&subscribe).unwrap()).await.unwrap(); + let initial_update = next_message(&mut handle).await; + assert!(matches!( + initial_update, + OutboundMessage::V1(SerializableMessage::Subscribe(_)) + )); + assert_eq!(initial_update.num_rows(), Some(1)); + for value in ["first", "second"] { + values.insert("WATCHED".into(), value.into()); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![Some(value.to_string())]]); + expect_view_update(&mut handle).await; + } + let mut invalid = values.clone(); + invalid.insert("WATCHED".into(), "fail-view".into()); + let failed = handle.republish_environment(invalid).await; + assert!(failed.as_ref().is_err() || !failed.as_ref().unwrap().was_successful()); + assert_eq!( + sql(&handle.client.module(), view).await, + vec![product![Some("second".to_string())]] + ); + values.remove("WATCHED"); + let module = publish(&handle, &values).await; + assert_eq!(sql(&module, view).await, vec![product![None::]]); + expect_view_update(&mut handle).await; + values.insert("LIMIT".into(), "x".repeat(8192)); + let module = publish(&handle, &values).await; + for _ in 0..2 { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "bounded_environment_sources", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + } + } + if name == "module-test-ts" { + check_submodule_scope(&mut handle, &mut values).await; + } + for key in ["UNDECLARED", "A=B"] { + let result = handle + .client + .module() + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![key, None::]).unwrap().into()), + ) + .await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); + } + }); +} + +#[test] +#[serial] +fn rust_environment_publish_is_atomic_and_reads_follow_declared_configuration() { + exercise_fixture("environment-test"); +} + +#[test] +#[serial] +fn rust_module_test_environment_publish_and_checked_reads() { + exercise_fixture("module-test"); +} + +#[test] +#[serial] +fn typescript_environment_publish_and_checked_reads() { + exercise_fixture("module-test-ts"); +} + +#[test] +#[serial] +fn cpp_environment_publish_and_checked_reads() { + exercise_fixture("module-test-cpp"); +} + +#[test] +#[serial] +fn csharp_environment_publish_and_checked_reads() { + exercise_fixture("module-test-cs"); +} + +#[cfg(feature = "allow_loopback_http_for_tests")] +#[test] +#[serial] +fn suspended_procedure_cannot_read_environment_from_a_replacement_program() { + use anyhow::Context as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + compiled_fixture("environment-test").with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + for explicit_tx in [false, true] { + let old = handle.client.module(); + let program = old.relational_db().program().unwrap().unwrap(); + let old_hash = program.hash; + let mut replacement = program.bytes.to_vec(); + // A valid custom section changes the exact program hash without + // changing the schema or behavior of this real Wasm module. + replacement.extend_from_slice(&[0, 3, 1, b'e', u8::from(explicit_tx)]); + let values = Values::from([ + ("REQUIRED".into(), "new-program-value".into()), + ("MODE".into(), "ready".into()), + ]); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/hold", listener.local_addr().unwrap()); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mut server = tokio::spawn(async move { + let (mut stream, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept()).await??; + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + anyhow::ensure!(request.len() < 4096, "test request exceeded its header bound"); + request.push(tokio::time::timeout(Duration::from_secs(10), stream.read_u8()).await??); + } + entered_tx + .send(()) + .map_err(|_| anyhow::anyhow!("test coordinator closed"))?; + tokio::time::timeout(Duration::from_secs(20), release_rx).await??; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await?; + stream.shutdown().await?; + anyhow::Ok(()) + }); + let call = old.call_procedure( + Identity::ZERO, + None, + None, + "read_environment_after_http", + FunctionArgs::Bsatn(bsatn::to_vec(&product![url, explicit_tx]).unwrap().into()), + ); + let publish_while_suspended = async { + tokio::time::timeout(Duration::from_secs(10), entered_rx) + .await + .context("procedure never reached owned HTTP barrier")??; + let publish = handle.republish_program(replacement.into(), program.kind.into(), values); + let release_after_commit = async { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + if old.relational_db().program()?.unwrap().hash != old_hash { + return anyhow::Ok(()); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .context("replacement program did not commit while procedure was suspended")??; + release_tx.send(()).map_err(|_| anyhow::anyhow!("HTTP barrier closed")) + }; + // Drive publication even when it waits for the old procedure + // to finish; release only after the new program is committed. + let (published, released) = tokio::join!(publish, release_after_commit); + released?; + anyhow::ensure!(published?.was_successful(), "replacement publication failed"); + anyhow::Ok(()) + }; + let (result, coordinated) = tokio::join!(call, publish_while_suspended); + let server_result = match tokio::time::timeout(Duration::from_secs(35), &mut server).await { + Ok(result) => result.unwrap(), + Err(_) => { + server.abort(); + let _ = server.await; + Err(anyhow::anyhow!("owned HTTP test server did not finish")) + } + }; + // Join the owned listener and publication before reporting any + // assertion failure, so failure cannot leave a live test host. + coordinated.unwrap(); + server_result.unwrap(); + assert!(result.result.is_err(), "old procedure read the replacement environment"); + assert_eq!( + read(&handle.client.module(), "REQUIRED").await, + AlgebraicValue::from(Some("new-program-value".to_string())) + ); + } + }, + ); +} + +// The actual host keeps ENV as exact strings; generated Rust accessors decode +// the selected enum variant after initial publish and complete replacements. +#[test] +#[serial] +fn rust_environment_enums_preserve_exact_typed_mappings() { + let initial = Values::from([ + ("REQUIRED".into(), "initial-required".into()), + ("MODE".into(), "ready".into()), + ]); + compiled_fixture("environment-test").with_module_async_with_environment( + DEFAULT_CONFIG, + initial.clone(), + |handle| async move { + let mut values = initial; + for (value, index) in [ + ("ready", 0u8), + ("other", 1), + ("in progress", 2), + ("Ready", 3), + ("", 4), + ("hΓ©llo\0δΈ–η•Œ", 5), + ] { + values.insert("MODE".into(), value.into()); + values.insert("TYPED".into(), value.into()); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![index, Some(index)]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + assert_eq!( + read(&module, "MODE").await, + AlgebraicValue::from(Some(value.to_string())) + ); + } + for rejected in ["InProgress", "READY", "in progress "] { + let mut invalid = values.clone(); + invalid.insert("TYPED".into(), rejected.into()); + let result = handle.republish_environment(invalid).await; + assert!(result.as_ref().is_err() || !result.as_ref().unwrap().was_successful()); + assert_eq!( + read(&handle.client.module(), "TYPED").await, + AlgebraicValue::from(Some("hΓ©llo\0δΈ–η•Œ".to_string())) + ); + } + values.remove("TYPED"); + let module = publish(&handle, &values).await; + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_typed_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&product![5u8, None::]).unwrap().into()), + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md index f8aaf847553..dd805b69ecf 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md +++ b/docs/docs/00200-core-concepts/00100-databases/00300-spacetime-publish.md @@ -104,6 +104,12 @@ spacetime publish --delete-data For all available publishing options and flags, see the [`spacetime publish` CLI reference](../../00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md#spacetime-publish). +### Environment Variables + +Modules can declare environment variables for configuration and secrets. Each publish supplies the complete set of values from the target's `env` configuration and declared shell variables. Required values must be supplied on every publish; omitted optional values are removed. The module and its environment update atomically. + +See [Environment Variables](./00700-environment-variables.md) for declarations, reading values in module code, publishing examples, and private tables for secrets that need to change without republishing. + ## Next Steps After publishing: diff --git a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md index 1bfb9a2a3a4..78b1db00454 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md +++ b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md @@ -251,6 +251,12 @@ export default authSchema; +## Environment variables + +Only the root module can declare a nonempty [environment](./00700-environment-variables.md). Including a submodule with environment declarations causes publication to fail. A module with such declarations can still be published independently as a root module. + +Submodules have no separate environment-variable namespace, and their host-dispatched entry points cannot read the root module's environment. Root module code can pass configuration values to helpers explicitly. Ordinary helper calls retain the calling entry point's access, including calls to helpers defined in submodules. + ## Client Subscriptions Client subscriptions use the same namespace structure as server-side access. Submodule tables and views are queried as `.`. diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md new file mode 100644 index 00000000000..b1fc7fe9d9f --- /dev/null +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -0,0 +1,278 @@ +--- +title: Environment Variables +slug: /databases/environment-variables +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Environment Variables + +Environment variables store configuration and secrets for a database, such as API keys and deployment settings. A module declares the names it accepts and any allowed values. Each publish supplies the complete set of values for that module. Module code reads them through `ctx.env`, or `ctx.Env` in C#. + +Use environment variables for configuration that changes when a module is published. For secrets or configuration that must change without publishing, use a [private table](#dynamically-editable-or-untyped-secrets). + +This guide assumes a module set up using a quickstart, such as the [Rust quickstart](../../00100-intro/00200-quickstarts/00500-rust.md), and familiarity with [publishing](./00300-spacetime-publish.md). + +## Declare and read variables + +Declare every environment key in the module. All values are strings. A declaration can accept any string, one exact string, or a set of allowed strings. Optional declarations permit an absent value. + +These examples declare a required `API_KEY`, a required `MODE` restricted to `development` or `production`, and an optional `LOG_LEVEL` restricted to `info` or `debug`. + + + + +Pass the declaration as the `env` option to `schema`. Include the module's existing tables in the first argument if it has any. + +```typescript +import { schema, t } from 'spacetimedb/server'; + +const spacetimedb = schema( + {}, + { + env: { + API_KEY: t.string(), + MODE: t.enum('Mode', ['development', 'production']), + LOG_LEVEL: t.enum('LogLevel', ['info', 'debug']).optional(), + }, + } +); + +export default spacetimedb; +``` + +Inside a reducer, procedure, or view, read values from its context: + +```typescript +const apiKey: string = ctx.env.API_KEY; +const mode: 'development' | 'production' = ctx.env.MODE; +const logLevel: 'info' | 'debug' | undefined = ctx.env.LOG_LEVEL; +const checked: string | null = ctx.env.get('LOG_LEVEL'); +``` + +Within an environment declaration, a simple enum specifies allowed strings. Enums used elsewhere in the module retain their usual tagged representation. An enum with one case restricts the value to that string. Enums with payloads cannot be used as environment constraints. + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors return `undefined`; `ctx.env.get` returns `null` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get('get')`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `withTx`. + + + + +Declare allowed strings with enums, then add one environment declaration to the module: + +```rust +#[derive(spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "development")] + Development, + #[env(value = "production")] + Production, +} + +#[derive(spacetimedb::EnvironmentValue)] +pub enum LogLevel { + #[env(value = "info")] + Info, + #[env(value = "debug")] + Debug, +} + +#[spacetimedb::env] +pub struct Env { + pub API_KEY: String, + pub MODE: Mode, + pub LOG_LEVEL: Option, +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```rust +let api_key: String = ctx.env.API_KEY(); +let mode: Mode = ctx.env.MODE(); +let log_level: Option = ctx.env.LOG_LEVEL(); +let checked: Option = ctx.env.get("LOG_LEVEL"); +``` + +`String` accepts any string. An enum restricts values to its variants, and `Option` permits absence. Type aliases work for these types. Without an attribute, a variant accepts its exact Rust name. An explicit mapping such as `#[env(value = "in progress")] InProgress` supports spaces, capitalization, Unicode, or the empty string. Mappings must be unique; variants cannot have payloads. A one-variant enum declares a single allowed string. These mappings affect environment reads and declarations only, not the enum's ordinary serialization. + +Existing `#[env(values(...))]` constraints on `String` and `Option` fields remain supported. Use enum variants to constrain typed enum fields. + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `None` for an absent optional value. + +Key names are exact and case-sensitive. The name `get` is reserved for the getter; read a declaration named `get` with `ctx.env.get("get")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. + + + + +Add one environment declaration to the module: + +```csharp +#nullable enable + +[SpacetimeDB.Env] +public partial struct EnvironmentSchema +{ + public string API_KEY; + [SpacetimeDB.EnvValues("development", "production")] + public string MODE; + [SpacetimeDB.EnvValues("info", "debug")] + public string? LOG_LEVEL; +} +``` + +Inside a reducer, procedure, or view, read values from its context: + +```csharp +string apiKey = ctx.Env.API_KEY; +string mode = ctx.Env.MODE; +string? logLevel = ctx.Env.LOG_LEVEL; +string? checkedValue = ctx.Env.Get("LOG_LEVEL"); +``` + +`string` requires a value, and `string?` permits absence. `[SpacetimeDB.EnvValues(...)]` restricts the allowed strings. Supplying one string makes it an exact-value constraint. + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.Env.Get` return `null` for an absent optional value. + +Key names are exact and case-sensitive. Names that collide with `Get`, `ModuleEnvironment`, or inherited `Object` methods are available through the string-key getter, for example `ctx.Env.Get("GetType")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `WithTx`. + + + + +Declare the environment in a dedicated header named `environment.h`: + +```cpp +#pragma once +#include + +SPACETIMEDB_ENV( + (API_KEY, std::string), + (MODE, std::string, ("development", "production")), + (LOG_LEVEL, std::optional, ("info", "debug")) +) +``` + +In `CMakeLists.txt`, set the header path **before** adding the SpacetimeDB module library directory: + +```cmake +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") +``` + +The library's CMake target includes this declaration consistently in the library and module source files that use the context type. Including it manually in just one source file is insufficient. + +Inside a reducer or procedure, read values from its context: + +```cpp +std::string api_key = ctx.env.API_KEY(); +std::string mode = ctx.env.MODE(); +std::optional log_level = ctx.env.LOG_LEVEL(); +std::optional checked = ctx.env.get("LOG_LEVEL"); +``` + +`std::string` requires a value, and `std::optional` permits absence. The optional third element restricts the allowed strings. Supplying one string makes it an exact-value constraint. + +The string-key getter checks the same declaration and permissions as named accessors. Reading an undeclared key fails. Optional named accessors and `ctx.env.get` return `std::nullopt` for an absent optional value. + +Key names are exact and case-sensitive. Names reserved by the generated accessor type, including `get`, remain available through the string-key getter, for example `ctx.env.get("get")`. A module with no environment declarations accepts no keys. + +Reads inside a transaction use that transaction's snapshot. In a procedure outside a transaction, each read uses a separate snapshot. To read several keys consistently, group the reads in `with_tx`. + + + + +## Supply values when publishing + +Add non-secret defaults to the selected database target in `spacetime.json`: + +```json +{ + "database": "env-example", + "server": "http://127.0.0.1:3000", + "module-path": "./spacetimedb", + "env": { + "MODE": "development", + "LOG_LEVEL": "info" + } +} +``` + +With a local server running on port 3000, publish from the directory containing that configuration. This example uses a disposable development value: + +```bash +API_KEY='development-only-key' spacetime publish +``` + +For real credentials, supply the value through the publishing process's environment, `spacetime.local.json`, or `spacetime.{environment}.local.json`. Keep checked-in `spacetime.json` and `spacetime.{environment}.json` limited to non-secret defaults. Ensure the local files are ignored by Git: + +```gitignore +spacetime.local.json +spacetime.*.local.json +``` + +The `.local` naming convention does not itself prevent a file from being committed. + +The CLI resolves each declared key in this order: + +1. A value in the publishing process's environment. +2. A value in the resolved configuration's `env` map. +3. Absence, which is accepted only for an optional declaration. + +A shell value overrides JSON even when it is an empty string or the key is absent from JSON. Already-exported variables behave the same as inline assignments. Unrelated shell variables are ignored unless the module declares their names. The CLI displays supplied key names and their sources, without printing their values. + +The configuration files `spacetime.json`, `spacetime.local.json`, `spacetime.{environment}.json`, and `spacetime.{environment}.local.json` apply in increasing precedence, where _environment_ is the environment selected with `--env`. Their `env` maps merge by key, as do maps inherited by child database targets. A higher-precedence value replaces that key while preserving unrelated keys. An empty map does not erase inherited keys. + +JSON strings pass through unchanged. Booleans and numbers are converted to strings, so `false` supplies `"false"`; declarations still validate strings. Use JSON strings when exact numeric spelling matters. Arrays, objects, and `null` are rejected, as are JSON keys the module has not declared. An invalid effective value rejects the publish rather than falling back to a lower-precedence value. For an absent optional value, omit its property from the JSON object instead of setting it to `null`. Also remove any inherited or shell value for that key, as described below. + +### Every publish replaces the complete environment + +Publishing validates the supplied values and installs them atomically with the module, before initialization or migration. Invalid environment configuration leaves the previous module and values unchanged. Changing only values still requires publishing, and does not rerun `init` on an existing database. + +Previously stored values are **not** defaults for the next publish. Every publish must supply each required value again. An optional value omitted from all effective inputs is removed. To remove `LOG_LEVEL` in the example, remove it from every applicable configuration layer and unset any exported `LOG_LEVEL` before publishing. An empty string is a value, not a deletion instruction. + +The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. + +For publishing from an HTTP client or a module procedure, see the [HTTP publish format and example](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values). Supply the module and complete environment in the request body; project configuration and shell overrides are CLI conveniences. + +## Inspect published values + +The database owner and collaborators with private-table read access can inspect the environment. For the local database above: + +```bash +spacetime env list env-example --server http://127.0.0.1:3000 +spacetime env get env-example MODE --server http://127.0.0.1:3000 +``` + +`env list` prints a table of keys and values. `env get` prints the requested value and fails if it is absent. Both are explicit inspection commands and include secrets in their output. Automatic publish output still omits values. + +Values are stored in the private system table `st_env`. Authorized SQL reads are also supported: + +```sql +SELECT key FROM st_env; +SELECT value FROM st_env WHERE key = 'MODE'; +``` + +SQL writes to `st_env`, module-side writes, and separate CLI setters are not supported. Changes go through a publish with the complete desired environment. + +## Access and limits + +Reducers, procedures, views, and HTTP handlers entered by the host in the root module can read its declared environment. Host-dispatched submodule entry points cannot read it, and submodules cannot declare a nonempty environment. There is no separate environment-variable namespace to configure for a submodule. A module with environment declarations can be published independently as a root module, but cannot be included as a submodule with those declarations. See [Submodules](./00600-submodules.md) for this restriction. Ordinary helper calls retain their calling entry point's access, including helpers defined in libraries or submodules. Root code can also pass a value to a helper explicitly. + +A procedure suspended across a publish cannot read values belonging to a replacement program. Environment reads in views participate in dependency tracking, so publishing changed values refreshes affected views. Module code remains responsible for what it returns or logs: returning a secret from a public view exposes that value to clients. + +Keys must match `[A-Za-z_][A-Za-z0-9_]*`. The limits are 256 bytes per key, 8 KiB per value, and 256 declarations per database. Length limits count UTF-8 bytes. Values may be empty if their declaration accepts an empty string. + +## Dynamically editable or untyped secrets + +Use an ordinary [private table](../00300-tables/00400-access-permissions.md) when a secret must change without republishing, or when its keys and allowed values should not be declared in the environment schema. For example, a table with a string primary-key column and a string value column can store arbitrary secret names and values. The table itself still has typed columns; its individual keys need no environment declarations or constraints. + +Update that table through reducers that explicitly authorize the caller. Keeping a table private controls direct client reads; it does not authorize calls to a reducer that modifies or returns its contents. Apply the same care to views, procedure results, and logs. Private tables follow the database's normal private-table permissions, including administrative reads. + +Both approaches are supported. Environment declarations additionally guarantee that required values are validated and available before `init` or migration runs. Private-table values follow the table's ordinary update and migration behavior. They do not receive environment schema validation or complete replacement on every publish. diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 6a065748f69..c820d565bb6 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -11,6 +11,9 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime`↴](#spacetime) * [`spacetime publish`↴](#spacetime-publish) +* [`spacetime env`↴](#spacetime-env) +* [`spacetime env get`↴](#spacetime-env-get) +* [`spacetime env list`↴](#spacetime-env-list) * [`spacetime delete`↴](#spacetime-delete) * [`spacetime logs`↴](#spacetime-logs) * [`spacetime call`↴](#spacetime-call) @@ -48,6 +51,7 @@ This document contains the help content for the `spacetime` command-line program ###### **Subcommands:** * `publish` β€” Create and update a SpacetimeDB database +* `env` β€” Inspect published database environment variables * `delete` β€” Deletes a SpacetimeDB database * `logs` β€” Prints logs from a SpacetimeDB database * `call` β€” Invokes a function (reducer or procedure) in a database. WARNING: This command is UNSTABLE and subject to breaking changes. @@ -82,7 +86,7 @@ Create and update a SpacetimeDB database **Usage:** `spacetime publish [OPTIONS] [name|identity]` -Run `spacetime help publish` for more detailed information. +Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information. ###### **Arguments:** @@ -137,6 +141,66 @@ Run `spacetime help publish` for more detailed information. +## `spacetime env` + +Inspect published database environment variables + +**Usage:** `spacetime env ` + +###### **Subcommands:** + +* `get` β€” Read one published environment value +* `list` β€” List published environment keys and values + + + +## `spacetime env get` + +Read one published environment value + +**Usage:** `spacetime env get [OPTIONS] ` + +###### **Arguments:** + +* `` β€” The declared environment key to read +* `` β€” The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` β€” The nickname, host name or URL of the server +* `--anonymous` β€” Perform this action with an anonymous identity +* `-y`, `--yes` β€” Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` β€” Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` β€” Ignore project configuration when resolving the database target + + + +## `spacetime env list` + +List published environment keys and values + +**Usage:** `spacetime env list [OPTIONS] ` + +###### **Arguments:** + +* `` β€” The database name, identity, or configured target + +###### **Options:** + +* `-s`, `--server ` β€” The nickname, host name or URL of the server +* `--anonymous` β€” Perform this action with an anonymous identity +* `-y`, `--yes` β€” Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--confirmed ` β€” Instruct the server to deliver only updates of confirmed transactions + + Possible values: `true`, `false` + +* `--no-config` β€” Ignore project configuration when resolving the database target + + + ## `spacetime delete` Deletes a SpacetimeDB database diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index e6137bea0df..52faf7df576 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -41,7 +41,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -76,7 +76,7 @@ If no `Authorization` header is provided, a new anonymous identity will be creat #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). #### Returns @@ -98,6 +98,43 @@ If a database with the given name exists, but the identity provided in the `Auth } } ``` +### Publishing with environment values + +Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish+json` with this JSON body: + +```json +{ + "module": "", + "environment": { + "API_KEY": "development-only-key", + "MODE": "development" + } +} +``` + +`module` uses standard padded Base64. `environment` maps declared names to strings. The server validates the complete map against the module's declarations and installs both in one transaction. Missing required values reject the publish; omitted optional values are removed. Omitting `environment` is equivalent to `{}`, including when publishing unchanged module bytes. + +For example, use `curl`, `jq`, and `base64` to publish a Wasm module to a local server. Export `SPACETIME_TOKEN` with a token authorized to publish and `API_KEY` with the required value. Change `module.wasm` to the artifact you built. + +```bash +base64 < module.wasm | + jq --raw-input --slurp '{ + module: gsub("[\\r\\n]"; ""), + environment: { API_KEY: env.API_KEY, MODE: "development" } + }' | + curl --fail-with-body --request PUT \ + 'http://127.0.0.1:3000/v1/database/env-example?host_type=wasm' \ + --header "Authorization: Bearer $SPACETIME_TOKEN" \ + --header 'Content-Type: application/vnd.spacetimedb.publish+json' \ + --data-binary @- +``` + +`jq` handles JSON escaping for the supplied value, including quotes and newlines. For a procedure or another HTTP client, construct the same JSON object with that language's JSON serializer and Base64 encoder. + +Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. + +Raw module bodies continue to work and supply an empty environment. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. + ## `GET /v1/database/:name_or_identity` Get a database's identity, owner identity, host type, number of replicas and a hash of its WASM module. diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 3ff913ba0ff..accf916617a 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -3,6 +3,7 @@ import type * as Preset from '@docusaurus/preset-classic'; import rehypeShiki, { RehypeShikiOptions } from '@shikijs/rehype'; import bash from 'shiki/langs/bash.mjs'; import c from 'shiki/langs/c.mjs'; +import cmake from 'shiki/langs/cmake.mjs'; import csharp from 'shiki/langs/csharp.mjs'; import fsharp from 'shiki/langs/fsharp.mjs'; import json from 'shiki/langs/json.mjs'; @@ -158,6 +159,7 @@ const config: Config = { toml, python, c, + cmake, cpp, protobuf, fsharp, diff --git a/modules/environment-test/Cargo.toml b/modules/environment-test/Cargo.toml new file mode 100644 index 00000000000..9ae6466b734 --- /dev/null +++ b/modules/environment-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "environment-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs new file mode 100644 index 00000000000..c72d1bd8625 --- /dev/null +++ b/modules/environment-test/src/lib.rs @@ -0,0 +1,147 @@ +use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; + +type RequiredString = String; +type OptionalString = Option; + +#[derive(Debug, PartialEq, spacetimedb::EnvironmentValue)] +pub enum Mode { + #[env(value = "ready")] + Ready, + #[env(value = "other")] + Other, + #[env(value = "in progress")] + InProgress, + #[env(value = "Ready")] + Capitalized, + #[env(value = "")] + Empty, + #[env(value = "hΓ©llo\0δΈ–η•Œ")] + Unicode, +} + +type OptionalMode = Option; + +#[spacetimedb::env] +pub struct Env { + pub REQUIRED: RequiredString, + pub MODE: Mode, + pub TYPED: OptionalMode, + pub MISSING: OptionalString, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, + pub HANDLER: Option, + pub WATCHED: Option, + pub LIMIT: Option, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + assert_eq!(ctx.env.REQUIRED(), "initial-required"); + assert_eq!(ctx.env.MODE(), Mode::Ready); + assert_eq!(ctx.env.TYPED(), None); +} + +#[spacetimedb::reducer] +pub fn expect_typed_environment(ctx: &ReducerContext, required: u8, optional: Option) { + fn index(mode: Mode) -> u8 { + match mode { + Mode::Ready => 0, + Mode::Other => 1, + Mode::InProgress => 2, + Mode::Capitalized => 3, + Mode::Empty => 4, + Mode::Unicode => 5, + } + } + assert_eq!(index(ctx.env.MODE()), required); + assert_eq!(ctx.env.TYPED().map(index), optional); + assert_eq!(index(ctx.as_read_only().env.MODE()), required); +} + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.get(&key), expected); + assert_eq!(ctx.as_read_only().env.get(&key), expected); + assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} + +/// The test server holds this request while the database installs a new program. +#[spacetimedb::procedure] +pub fn read_environment_after_http(ctx: &mut ProcedureContext, url: String, explicit_tx: bool) -> Option { + let request = spacetimedb::http::Request::builder() + .uri(url) + .extension(spacetimedb::http::Timeout::from(spacetimedb::TimeDuration::from( + std::time::Duration::from_secs(30), + ))) + .body(()) + .unwrap(); + assert!(ctx.http.send(request).unwrap().status().is_success()); + if explicit_tx { + ctx.with_tx(|tx| tx.env.REQUIRED().into()) + } else { + ctx.env.REQUIRED().into() + } +} + +#[derive(SpacetimeType)] +pub struct EnvironmentValue { + pub value: Option, +} + +#[spacetimedb::view(accessor = environment_value, public)] +pub fn environment_value(ctx: &AnonymousViewContext) -> Option { + let value = ctx.env.WATCHED(); + assert_ne!(value.as_deref(), Some("fail-view")); + Some(EnvironmentValue { value }) +} + +/// Hand-written ABI callers cannot retain unbounded host allocations. +#[spacetimedb::reducer] +pub fn bounded_environment_sources(_ctx: &ReducerContext) { + use spacetimedb::sys::raw::{self, BytesSource}; + let mut sources = Vec::new(); + for i in 0..=256 { + let mut source = BytesSource::INVALID; + let status = unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }; + if i == 256 { + assert_eq!(status, 9); // NO_SPACE + } else { + assert_eq!(status, 0); + assert!(source != BytesSource::INVALID); + sources.push(source); + } + } + let mut buffer = [0u8; 8192]; + let mut len = buffer.len(); + let status = unsafe { raw::bytes_source_read(sources[0], buffer.as_mut_ptr(), &mut len) }; + assert_eq!(status, -1); + assert_eq!(len, buffer.len()); + let mut source = BytesSource::INVALID; + assert_eq!(unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }, 0); + assert!(source != BytesSource::INVALID); + // The remaining sources are released when this invocation ends. +} + +#[spacetimedb::http::handler] +pub fn handler_environment( + ctx: &mut spacetimedb::http::HandlerContext, + _request: spacetimedb::http::Request, +) -> spacetimedb::http::Response { + let outside = ctx.env.get("HANDLER"); + ctx.with_tx(|tx| assert_eq!(tx.env.get("HANDLER"), outside)); + spacetimedb::http::Response::new(spacetimedb::http::Body::from_bytes(outside.unwrap())) +} + +#[spacetimedb::http::router] +pub fn router() -> spacetimedb::http::Router { + spacetimedb::http::Router::new().get("/environment", handler_environment) +} diff --git a/modules/module-test-cpp/CMakeLists.txt b/modules/module-test-cpp/CMakeLists.txt index ae5c37ca153..eccf682be83 100644 --- a/modules/module-test-cpp/CMakeLists.txt +++ b/modules/module-test-cpp/CMakeLists.txt @@ -34,6 +34,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() +# Declare the same context type in module and SDK compilation units. +set(SPACETIMEDB_ENV_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/environment.h") + # Link the SpacetimeDB library add_subdirectory(${SPACETIMEDB_CPP_LIBRARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/spacetimedb_cpp_library) target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) diff --git a/modules/module-test-cpp/environment.h b/modules/module-test-cpp/environment.h new file mode 100644 index 00000000000..e0e7e5d34f7 --- /dev/null +++ b/modules/module-test-cpp/environment.h @@ -0,0 +1,9 @@ +#pragma once +#include +SPACETIMEDB_ENV( + (MISSING, std::optional), + (EMPTY, std::optional), + (UTF8, std::optional), + (NUL, std::optional), + (MAXIMUM, std::optional) +) diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index 3a93f0b20ec..fe47c4ce2a1 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -720,3 +720,17 @@ SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { SPACETIMEDB_HTTP_ROUTER(router) { return Router().get("/get", get_simple); } + +SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + // Parentheses avoid the existing enum helper macro named EMPTY(). + if ((ctx.env.EMPTY)() != ctx.env.get("EMPTY")) LOG_PANIC("named environment mismatch"); + if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); + return Ok(); +} +SPACETIMEDB_PROCEDURE(std::optional, read_environment, ProcedureContext ctx, std::string key) { + const auto outside = ctx.env.get(key); + ctx.with_tx([&](TxContext& tx) { + if (tx.env.get(key) != outside) LOG_PANIC("transaction environment value mismatch"); + }); + return outside; +} diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs new file mode 100644 index 00000000000..b0630c30ec3 --- /dev/null +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -0,0 +1,36 @@ +#pragma warning disable STDB_UNSTABLE +namespace SpacetimeDB.Modules.ModuleTestCs; + +using SpacetimeDB; + +[SpacetimeDB.Env] +public partial struct ModuleEnvironmentSchema +{ + public string? MISSING; + public string? EMPTY; + public string? UTF8; + public string? NUL; + public string? MAXIMUM; +} + +public static partial class EnvironmentTests +{ + [Reducer] + public static void expect_environment(ReducerContext ctx, string key, string? expected) + { + if (ctx.Env.EMPTY != ctx.Env.Get("EMPTY")) throw new Exception("named environment mismatch"); + if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); + } + + [Procedure] + public static string? read_environment(ProcedureContext ctx, string key) + { + var outside = ctx.Env.Get(key); + ctx.WithTx(tx => + { + if (tx.Env.Get(key) != outside) throw new Exception("transaction environment value mismatch"); + return true; + }); + return outside; + } +} diff --git a/modules/module-test-ts/src/environment_sys.d.ts b/modules/module-test-ts/src/environment_sys.d.ts new file mode 100644 index 00000000000..d2abea72f80 --- /dev/null +++ b/modules/module-test-ts/src/environment_sys.d.ts @@ -0,0 +1,4 @@ +// Raw host ABI used to verify that SDK context changes cannot grant authority. +declare module 'spacetime:sys@2.2' { + export function env_get(key: string): string | null; +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 0eba467a7d0..000e25be0e4 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -249,7 +249,13 @@ const spacetimedb = schema({ ), tableToRemove: table({ name: 'table_to_remove' }, { id: t.u32() }), lib: libSubmodule, -}); +}, { env: { + MISSING: t.string().optional(), + EMPTY: t.string().optional(), + UTF8: t.string().optional(), + NUL: t.string().optional(), + MAXIMUM: t.string().optional(), +} }); export default spacetimedb; // ───────────────────────────────────────────────────────────────────────────── @@ -549,6 +555,47 @@ export const libHello = spacetimedb.httpHandler((ctx, req) => { return libSubmodule.libHello(ctx.as.lib, req); }); +// Ordinary JS delegation retains this root host entry, even with ctx.as.lib. +// Direct host dispatch to lib.envReadHandler is the separate denied case. +export const envReadChildHandler = spacetimedb.httpHandler((ctx, req) => + libSubmodule.envReadHandler(ctx.as.lib, req) +); + +// Root entries must use the checked accessor too; returning raw st_env SQL is +// forbidden even when the same entry could legitimately call ctx.env.get. +export const envReadRootSqlView = spacetimedb.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => libSubmodule.uncheckedEnvironmentQuery(ctx.from.player) +); + export const router = spacetimedb.httpRouter( - new Router().get('/get', getSimple).get('/lib-hello', libHello) + new Router().get('/get', getSimple).get('/lib-hello', libHello).get('/env-child', envReadChildHandler) +); + +// Dedicated environment ABI integration exercised by crates/testing. +export const expectEnvironment = spacetimedb.reducer( + { name: 'expect_environment' }, + { key: t.string(), expected: t.option(t.string()) }, + (ctx, { key, expected }) => { + if (libSubmodule.readRootEnvironmentHelper() !== ctx.env.get('EMPTY')) throw new Error('helper environment scope mismatch'); + if (ctx.env.EMPTY !== (ctx.env.get('EMPTY') ?? undefined)) throw new Error('named environment mismatch'); + if (ctx.env.get(key) !== (expected ?? null)) { + throw new Error('environment value mismatch'); + } + } +); +export const readEnvironment = spacetimedb.procedure( + { name: 'read_environment' }, + { key: t.string() }, + t.option(t.string()), + (ctx, { key }) => { + const outside = ctx.env.get(key); + ctx.withTx(tx => { + if (tx.env.get(key) !== outside) { + throw new Error('transaction environment value mismatch'); + } + }); + return outside ?? undefined; + } ); diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts index 3c2832d602c..466770be2ff 100644 --- a/modules/module-test-ts/src/lib_submodule.ts +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -1,4 +1,6 @@ +/// import { schema, table, t, SyncResponse } from 'spacetimedb/server'; +import { env_get } from 'spacetime:sys@2.2'; const libData = table( { name: 'libData', public: true }, @@ -26,3 +28,37 @@ export const libCount = libSubmoduleSchema.procedure(t.u64(), ctx => export const libHello = libSubmoduleSchema.httpHandler((_ctx, _req) => { return new SyncResponse('Hello from lib submodule!'); }); + +// Ordinary helpers retain their caller's host scope. Exported module callbacks +// below are entered through the lib namespace and must be rejected by the host. +export function readRootEnvironmentHelper(): string | null { + return env_get('EMPTY'); +} +export const envReadReducer = libSubmoduleSchema.reducer(() => { + env_get('EMPTY'); +}); +export const envReadProcedure = libSubmoduleSchema.procedure(t.string(), () => + env_get('EMPTY') ?? '' +); +export const envReadInTx = libSubmoduleSchema.procedure(t.string(), ctx => + ctx.withTx(() => env_get('EMPTY') ?? '') +); +export const envReadView = libSubmoduleSchema.view( + { public: true }, t.array(t.object('EnvReadRow', { value: t.string() })), () => [{ value: env_get('EMPTY') ?? '' }] +); +export const envReadHandler = libSubmoduleSchema.httpHandler(() => + new SyncResponse(env_get('EMPTY') ?? '') +); + +// Deliberately forge module-returned SQL through an ordinary query object's +// runtime brand. SDK types are not a security boundary for this host feature. +export function uncheckedEnvironmentQuery(source: object) { + return Object.assign(Object.create(source), { + toSql: () => 'SELECT * FROM st_env', + }) as { key: string; value: string }[]; +} +export const envReadSqlView = libSubmoduleSchema.view( + { public: true }, + t.array(t.object('EnvSqlRow', { key: t.string(), value: t.string() })), + ctx => uncheckedEnvironmentQuery(ctx.from.libData) +); diff --git a/modules/module-test/src/environment.rs b/modules/module-test/src/environment.rs new file mode 100644 index 00000000000..6adce646498 --- /dev/null +++ b/modules/module-test/src/environment.rs @@ -0,0 +1,26 @@ +//! Optional declarations keep this general-purpose test module publishable +//! without configuration, just like the C#, C++, and TypeScript examples. + +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::env] +pub struct Env { + pub MISSING: Option, + pub EMPTY: Option, + pub UTF8: Option, + pub NUL: Option, + pub MAXIMUM: Option, +} + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.EMPTY(), ctx.env.get("EMPTY")); + assert_eq!(ctx.env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index fc1851b21b0..9a2d6790d53 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -10,6 +10,8 @@ use spacetimedb::{ }; use spacetimedb::{log, ProcedureContext}; +pub mod environment; + pub type TestAlias = TestA; // ───────────────────────────────────────────────────────────────────────────── diff --git a/tools/ci/commands/workflow-coordinator/src/main.rs b/tools/ci/commands/workflow-coordinator/src/main.rs index 2ff8101d999..521b19245d0 100644 --- a/tools/ci/commands/workflow-coordinator/src/main.rs +++ b/tools/ci/commands/workflow-coordinator/src/main.rs @@ -197,6 +197,8 @@ struct Repository { #[derive(Clone, Debug, Deserialize, PartialEq, Eq)] struct PullRequestRef { sha: String, + #[serde(rename = "ref")] + branch_name: String, repo: Option, } @@ -340,10 +342,29 @@ fn related_private_pr(public_pr_number: Option) -> Result 1 { - bail!("found multiple open linked private PRs"); + let public_branch = if pulls.len() > 1 { + Some(pull_request(PUBLIC_REPO, public_pr_number)?.head.branch_name) + } else { + None + }; + select_related_private_pr(pulls, public_branch.as_deref()) +} + +fn select_related_private_pr(mut pulls: Vec, public_branch: Option<&str>) -> Result> { + if pulls.len() <= 1 { + return Ok(pulls.pop()); + } + + // Timeline references include historical links and links to other layers of + // a PR stack. A unique shared branch name identifies the companion PR; the + // exact public-submodule SHA is still checked before selecting its CI run. + if let Some(public_branch) = public_branch.filter(|branch| !branch.is_empty()) { + pulls.retain(|pull| pull.head.branch_name == public_branch); + if pulls.len() == 1 { + return Ok(pulls.pop()); + } } - Ok(pulls.pop()) + bail!("found multiple open linked private PRs without a unique matching head branch") } fn resolve_private_source(public_pr_number: Option) -> Result { @@ -574,6 +595,7 @@ mod tests { state: "open".to_owned(), head: PullRequestRef { sha: "private-sha".to_owned(), + branch_name: "tyler/environment-variables".to_owned(), repo: Some(Repository { full_name: PRIVATE_REPO.to_owned(), }), @@ -581,6 +603,73 @@ mod tests { } } + #[test] + fn related_private_pr_prefers_the_unique_exact_public_head_branch() { + let matching = pull(); + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/environment-variables-followup".to_owned(); + for candidates in [ + vec![matching.clone(), downstream.clone()], + vec![downstream, matching.clone()], + ] { + assert_eq!( + select_related_private_pr(candidates, Some("tyler/environment-variables")).unwrap(), + Some(matching.clone()) + ); + } + } + + #[test] + fn related_private_pr_preserves_absent_and_single_candidate_behavior() { + assert_eq!(select_related_private_pr(Vec::new(), None).unwrap(), None); + for public_branch in [None, Some("unrelated-branch")] { + assert_eq!( + select_related_private_pr(vec![pull()], public_branch).unwrap(), + Some(pull()) + ); + } + } + + #[test] + fn related_private_pr_rejects_multiple_matching_branches() { + let mut duplicate = pull(); + duplicate.number = 43; + assert!(select_related_private_pr(vec![pull(), duplicate], Some("tyler/environment-variables")).is_err()); + } + + #[test] + fn related_private_pr_rejects_missing_or_unmatched_public_branch() { + let mut downstream = pull(); + downstream.number = 43; + downstream.head.branch_name = "tyler/v10-abi-extensions".to_owned(); + for public_branch in [None, Some(""), Some("tyler/unrelated")] { + assert!(select_related_private_pr(vec![pull(), downstream.clone()], public_branch).is_err()); + } + } + + #[test] + fn selected_companion_still_requires_the_exact_public_submodule() { + let selected = select_related_private_pr(vec![pull()], None).unwrap().unwrap(); + assert!(ensure_public_submodule_matches(selected.number, "old-public-sha", "requested-public-sha").is_err()); + ensure_public_submodule_matches(selected.number, "requested-public-sha", "requested-public-sha").unwrap(); + } + + #[test] + fn pull_request_head_branch_uses_the_github_ref_field() { + let parsed: PullRequest = serde_json::from_value(serde_json::json!({ + "number": 42, + "state": "open", + "head": { + "sha": "private-sha", + "ref": "tyler/environment-variables", + "repo": { "full_name": PRIVATE_REPO } + } + })) + .unwrap(); + assert_eq!(parsed, pull()); + } + fn run(id: u64, title: &str, created_at: &str) -> WorkflowRun { WorkflowRun { id,