diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52dc405c..7e7d6c5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,7 @@ jobs: - library_config - datadog-js-zstd - pipeline + - sketches steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: 'Use composite action' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 256da64b..c8573d07 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,7 @@ jobs: - library_config - datadog-js-zstd - pipeline + - sketches steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: 'Use composite action' diff --git a/Cargo.lock b/Cargo.lock index d852cbf4..1960a25b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -858,7 +858,7 @@ dependencies = [ [[package]] name = "libdd-capabilities" version = "2.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "bytes", @@ -882,8 +882,9 @@ dependencies = [ [[package]] name = "libdd-capabilities-impl" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ + "anyhow", "bytes", "http", "http-body-util", @@ -929,7 +930,7 @@ dependencies = [ [[package]] name = "libdd-common" version = "5.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "bytes", @@ -970,7 +971,7 @@ dependencies = [ "libc", "libdd-common 4.2.0", "libdd-libunwind-sys", - "libdd-telemetry", + "libdd-telemetry 5.0.0", "nix 0.29.0", "num-derive", "num-traits", @@ -992,13 +993,14 @@ dependencies = [ [[package]] name = "libdd-data-pipeline" version = "7.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "arc-swap", "async-trait", "bytes", "either", + "futures", "getrandom 0.2.17", "http", "http-body-util", @@ -1008,9 +1010,10 @@ dependencies = [ "libdd-ddsketch 1.1.0", "libdd-dogstatsd-client", "libdd-shared-runtime 2.0.0", + "libdd-telemetry 6.0.0", "libdd-tinybytes", "libdd-trace-normalization", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", "libdd-trace-stats", "libdd-trace-utils", "rmp-serde", @@ -1021,6 +1024,7 @@ dependencies = [ "tokio-util", "tracing", "uuid", + "web-time", ] [[package]] @@ -1034,7 +1038,7 @@ dependencies = [ [[package]] name = "libdd-ddsketch" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "prost", ] @@ -1042,7 +1046,7 @@ dependencies = [ [[package]] name = "libdd-dogstatsd-client" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "cadence", @@ -1054,26 +1058,12 @@ dependencies = [ [[package]] name = "libdd-library-config" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=353134770b312b7ccd2df6afabc253090b948e5f#353134770b312b7ccd2df6afabc253090b948e5f" -dependencies = [ - "anyhow", - "memfd", - "rand", - "rmp", - "rmp-serde", - "serde", - "serde_yaml", -] - -[[package]] -name = "libdd-library-config" -version = "2.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=7cdeb7896e92d1ba38bde495934e112dac2eda25#7cdeb7896e92d1ba38bde495934e112dac2eda25" +version = "3.0.0" +source = "git+https://github.com/DataDog/libdatadog.git?rev=0c6e2a5df2a163d34c4f385353ffc5d7257c72f4#0c6e2a5df2a163d34c4f385353ffc5d7257c72f4" dependencies = [ "anyhow", "libc", - "libdd-trace-protobuf 3.0.2", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=0c6e2a5df2a163d34c4f385353ffc5d7257c72f4)", "memfd", "prost", "rand", @@ -1114,7 +1104,7 @@ dependencies = [ [[package]] name = "libdd-shared-runtime" version = "2.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "async-trait", "futures", @@ -1155,10 +1145,39 @@ dependencies = [ "winver", ] +[[package]] +name = "libdd-telemetry" +version = "6.0.0" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "futures", + "getrandom 0.2.17", + "hashbrown 0.15.5", + "http", + "libc", + "libdd-capabilities 2.1.0", + "libdd-common 5.1.0", + "libdd-ddsketch 1.1.0", + "libdd-shared-runtime 2.0.0", + "serde", + "serde_json", + "sys-info", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "winver", +] + [[package]] name = "libdd-tinybytes" version = "1.1.1" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "serde", ] @@ -1166,21 +1185,21 @@ dependencies = [ [[package]] name = "libdd-trace-normalization" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", ] [[package]] name = "libdd-trace-obfuscation" version = "5.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "fluent-uri", "libdd-common 5.1.0", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", "libdd-trace-utils", "log", "percent-encoding", @@ -1190,8 +1209,8 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" -version = "3.0.2" -source = "git+https://github.com/DataDog/libdatadog.git?rev=7cdeb7896e92d1ba38bde495934e112dac2eda25#7cdeb7896e92d1ba38bde495934e112dac2eda25" +version = "4.0.0" +source = "git+https://github.com/DataDog/libdatadog.git?rev=0c6e2a5df2a163d34c4f385353ffc5d7257c72f4#0c6e2a5df2a163d34c4f385353ffc5d7257c72f4" dependencies = [ "prost", "serde", @@ -1201,7 +1220,7 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "prost", "serde", @@ -1211,7 +1230,7 @@ dependencies = [ [[package]] name = "libdd-trace-stats" version = "6.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "arc-swap", @@ -1225,20 +1244,22 @@ dependencies = [ "libdd-ddsketch 1.1.0", "libdd-dogstatsd-client", "libdd-shared-runtime 2.0.0", + "libdd-telemetry 6.0.0", "libdd-trace-obfuscation", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", "libdd-trace-utils", "rmp-serde", "serde", "tokio", "tokio-util", "tracing", + "web-time", ] [[package]] name = "libdd-trace-utils" version = "9.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=6356aee4c69b51f27e7e82e43b1797f60b74905a#6356aee4c69b51f27e7e82e43b1797f60b74905a" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "base64", @@ -1256,7 +1277,7 @@ dependencies = [ "libdd-common 5.1.0", "libdd-tinybytes", "libdd-trace-normalization", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", "prost", "rand", "rmp", @@ -1293,7 +1314,7 @@ version = "0.2.0" dependencies = [ "anyhow", "getrandom 0.2.17", - "libdd-library-config 1.0.0", + "libdd-library-config", "serde", "serde-wasm-bindgen", "wasm-bindgen", @@ -1747,7 +1768,7 @@ dependencies = [ "libdd-common 5.1.0", "libdd-data-pipeline", "libdd-shared-runtime 2.0.0", - "libdd-trace-protobuf 4.0.0", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f)", "libdd-trace-stats", "libdd-trace-utils", "rmp-serde", @@ -1757,6 +1778,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", + "web-time", ] [[package]] @@ -1807,8 +1829,8 @@ name = "process-discovery" version = "0.1.0" dependencies = [ "anyhow", - "libdd-library-config 2.0.0", - "libdd-trace-protobuf 3.0.2", + "libdd-library-config", + "libdd-trace-protobuf 4.0.0 (git+https://github.com/DataDog/libdatadog.git?rev=0c6e2a5df2a163d34c4f385353ffc5d7257c72f4)", "napi", "napi-derive", ] @@ -2255,6 +2277,14 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "sketches" +version = "0.1.0" +dependencies = [ + "libdd-ddsketch 1.1.0", + "wasm-bindgen", +] + [[package]] name = "slab" version = "0.4.12" @@ -2671,6 +2701,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/capabilities/Cargo.toml b/crates/capabilities/Cargo.toml index f468e7d5..edb40adc 100644 --- a/crates/capabilities/Cargo.toml +++ b/crates/capabilities/Cargo.toml @@ -15,9 +15,7 @@ http = "1" bytes = "1.4" futures-core = "0.3" anyhow = "1" -# TODO: Replace this temporary libdatadog PR rev with the official release/tag -# that contains DataDog/libdatadog#2235, then regenerate Cargo.lock. -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a" } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f" } [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/crates/capabilities/src/env.rs b/crates/capabilities/src/env.rs new file mode 100644 index 00000000..716d4b4a --- /dev/null +++ b/crates/capabilities/src/env.rs @@ -0,0 +1,28 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Wasm implementation of [`EnvCapability`] backed by Node.js `process.env`. + +use wasm_bindgen::prelude::*; + +use libdd_capabilities::env::{EnvCapability, EnvError}; + +#[wasm_bindgen(module = "/src/env_transport.js")] +extern "C" { + #[wasm_bindgen(js_name = "get")] + fn js_env_get(name: &str) -> JsValue; +} + +#[derive(Debug, Clone)] +pub struct WasmEnvCapability; + +impl EnvCapability for WasmEnvCapability { + fn new() -> Self { + Self + } + + fn get(&self, name: &str) -> Result, EnvError> { + // Node coerces every process.env value to a string, so NotUnicode is unreachable here. + Ok(js_env_get(name).as_string()) + } +} diff --git a/crates/capabilities/src/env_transport.js b/crates/capabilities/src/env_transport.js new file mode 100644 index 00000000..92d6504b --- /dev/null +++ b/crates/capabilities/src/env_transport.js @@ -0,0 +1,7 @@ +'use strict' + +const { env } = process + +module.exports.get = (name) => { + return env[name] +} diff --git a/crates/capabilities/src/file.rs b/crates/capabilities/src/file.rs new file mode 100644 index 00000000..fe8eb4e6 --- /dev/null +++ b/crates/capabilities/src/file.rs @@ -0,0 +1,156 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Wasm implementation of [`FileCapability`] backed by Node.js `fs`. +//! +//! The JS transport is imported via `wasm_bindgen(module = ...)` from +//! `filesystem.js`, which ships alongside the wasm output. + +use std::future::Future; + +use bytes::Bytes; +use js_sys::{self, Reflect, Uint8Array}; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +use libdd_capabilities::file::{FileCapability, FileError, FileMetadata}; +use libdd_capabilities::maybe_send::MaybeSend; + +#[wasm_bindgen(module = "/src/filesystem.js")] +extern "C" { + #[wasm_bindgen(js_name = "readFile", catch)] + fn js_read_file(path: &str) -> Result; + + #[wasm_bindgen(js_name = "writeFile", catch)] + fn js_write_file(path: &str, data: &[u8]) -> Result; + + #[wasm_bindgen(js_name = "metadata", catch)] + fn js_metadata(path: &str) -> Result; + + #[wasm_bindgen(js_name = "exists", catch)] + fn js_exists(path: &str) -> Result; +} + +#[derive(Debug, Clone)] +pub struct WasmFileCapability; + +impl FileCapability for WasmFileCapability { + fn new() -> Self { + Self + } + + #[allow(clippy::manual_async_fn)] + fn read(&self, path: &str) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = + js_read_file(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + let array = Uint8Array::unchecked_from_js(value); + Ok(Bytes::from(array.to_vec())) + } + } + + #[allow(clippy::manual_async_fn)] + fn write( + &self, + path: &str, + contents: Bytes, + ) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = js_write_file(&path, &contents) + .map_err(|e| map_js_error(&e, &path))?; + JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + Ok(()) + } + } + + #[allow(clippy::manual_async_fn)] + fn metadata( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = + js_metadata(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + parse_metadata(&value, &path) + } + } + + #[allow(clippy::manual_async_fn)] + fn exists(&self, path: &str) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = js_exists(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + value + .as_bool() + .ok_or_else(|| FileError::Io(anyhow::anyhow!("exists({path}) did not return a boolean"))) + } + } +} + +fn map_js_error(err: &JsValue, path: &str) -> FileError { + let code = Reflect::get(err, &JsValue::from_str("code")) + .ok() + .and_then(|v| v.as_string()); + match code.as_deref() { + Some("ENOENT") => FileError::NotFound(path.to_owned()), + Some("EACCES") | Some("EPERM") => FileError::PermissionDenied(path.to_owned()), + _ => { + let message = Reflect::get(err, &JsValue::from_str("message")) + .ok() + .and_then(|v| v.as_string()) + .unwrap_or_else(|| format!("{err:?}")); + FileError::Io(anyhow::anyhow!("{message} (path: {path})")) + } + } +} + +fn parse_metadata(value: &JsValue, path: &str) -> Result { + let size = read_bigint_u64(value, "size", path)?; + // Node populates `stat().ino` on every platform, so `inode` is always Some. + let inode = Some(read_bigint_u64(value, "inode", path)?); + let is_file = read_bool(value, "is_file", path)?; + let is_dir = read_bool(value, "is_dir", path)?; + Ok(FileMetadata { + size, + inode, + is_file, + is_dir, + }) +} + +fn read_bigint_u64(value: &JsValue, key: &str, path: &str) -> Result { + let v = Reflect::get(value, &JsValue::from_str(key)) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) could not read field `{key}`")))?; + if v.is_undefined() || v.is_null() { + return Err(FileError::Io(anyhow::anyhow!("metadata({path}) missing field `{key}`"))); + } + let bigint = js_sys::BigInt::try_from(v) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) field `{key}` is not a BigInt")))?; + u64::try_from(bigint) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) field `{key}` overflows u64"))) +} + +fn read_bool(value: &JsValue, key: &str, path: &str) -> Result { + Reflect::get(value, &JsValue::from_str(key)) + .ok() + .and_then(|v| v.as_bool()) + .ok_or_else(|| { + FileError::Io(anyhow::anyhow!( + "metadata({path}) is missing boolean field `{key}`" + )) + }) +} diff --git a/crates/capabilities/src/filesystem.js b/crates/capabilities/src/filesystem.js new file mode 100644 index 00000000..014049a0 --- /dev/null +++ b/crates/capabilities/src/filesystem.js @@ -0,0 +1,38 @@ +// Lazy `require('node:fs')` — see http_transport.js. The cached accessor +// avoids paying the module-resolution cost on every call. + +'use strict' + +let _fs +function fs () { + return _fs ??= require('node:fs') +} + +module.exports.readFile = function (path) { + return fs().promises.readFile(path) +} + +module.exports.writeFile = function (path, data) { + // Copy off the wasm-memory view before the async write; a memory grow would + // otherwise detach the underlying ArrayBuffer mid-write. + return fs().promises.writeFile(path, Buffer.from(data)) +} + +module.exports.metadata = function (path) { + return fs().promises.stat(path, { bigint: true }).then(s => ({ + size: s.size, + inode: s.ino, + is_file: s.isFile(), + is_dir: s.isDirectory(), + })) +} + +module.exports.exists = function (path) { + return fs().promises.stat(path).then( + () => true, + (error) => { + if (error && error.code === 'ENOENT') return false + throw error + }, + ) +} diff --git a/crates/capabilities/src/lib.rs b/crates/capabilities/src/lib.rs index f7cb4b01..e76a3812 100644 --- a/crates/capabilities/src/lib.rs +++ b/crates/capabilities/src/lib.rs @@ -4,35 +4,43 @@ //! Wasm capability implementations for libdatadog-nodejs. //! //! [`WasmCapabilities`] is the bundle struct that implements every capability -//! trait `TraceExporter` requires (HTTP, sleep, log output) using wasm_bindgen -//! and JS transports. The wasm binding crate pins this type as the capability -//! generic for libdatadog's `TraceExporter`, mirroring libdatadog's native -//! `NativeCapabilities`. +//! trait `TraceExporter` requires using wasm_bindgen and JS transports. The +//! wasm binding crate pins this type as the capability generic for libdatadog's +//! `TraceExporter`, mirroring libdatadog's native `NativeCapabilities`. use std::future::Future; use std::time::Duration; +use libdd_capabilities::env::{EnvCapability, EnvError}; +use libdd_capabilities::file::{FileCapability, FileError, FileMetadata}; use libdd_capabilities::http::HttpError; use libdd_capabilities::{HttpClientCapability, LogWriterCapability, MaybeSend, SleepCapability}; +pub mod env; +pub mod file; pub mod http; pub mod sleep; +pub use env::WasmEnvCapability; +pub use file::WasmFileCapability; pub use http::WasmHttpClient; pub use sleep::WasmSleepCapability; /// Bundle of wasm platform capabilities for libdatadog's `TraceExporter`. /// -/// Mirrors libdatadog's native `NativeCapabilities`: delegates HTTP to -/// [`WasmHttpClient`] and sleep to [`WasmSleepCapability`]. Log output is a -/// no-op (see the [`LogWriterCapability`] impl). Per-function bounds stay +/// Mirrors libdatadog's native `NativeCapabilities`. Per-function bounds stay /// minimal in libdatadog (e.g. stats-only code uses [`WasmHttpClient`] /// directly), so this bundle is only needed where the full `TraceExporter` /// capability set is. #[derive(Clone, Debug)] pub struct WasmCapabilities { + /// Outbound HTTP requests routed through the JS fetch/socket transport. http: WasmHttpClient, + /// Async sleep backed by `setTimeout` via wasm-bindgen. sleep: WasmSleepCapability, + /// Filesystem access delegated to the Node.js `fs` transport. + file: WasmFileCapability, + env: WasmEnvCapability, } impl Default for WasmCapabilities { @@ -46,16 +54,15 @@ impl WasmCapabilities { Self { http: WasmHttpClient::new_client(), sleep: WasmSleepCapability, + file: WasmFileCapability, + env: WasmEnvCapability, } } } impl HttpClientCapability for WasmCapabilities { fn new_client() -> Self { - Self { - http: WasmHttpClient::new_client(), - sleep: WasmSleepCapability, - } + Self::new() } fn request( @@ -68,10 +75,7 @@ impl HttpClientCapability for WasmCapabilities { impl SleepCapability for WasmCapabilities { fn new() -> Self { - Self { - http: WasmHttpClient::new_client(), - sleep: WasmSleepCapability, - } + Self::new() } fn sleep(&self, duration: Duration) -> impl Future + MaybeSend { @@ -89,3 +93,45 @@ impl LogWriterCapability for WasmCapabilities { Ok(()) } } + +impl FileCapability for WasmCapabilities { + fn new() -> Self { + Self::new() + } + + fn read( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + self.file.read(path) + } + + fn write( + &self, + path: &str, + contents: ::bytes::Bytes, + ) -> impl Future> + MaybeSend { + self.file.write(path, contents) + } + + fn metadata( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + self.file.metadata(path) + } + + fn exists(&self, path: &str) -> impl Future> + MaybeSend { + self.file.exists(path) + } +} + +impl EnvCapability for WasmCapabilities { + fn new() -> Self { + Self::new() + } + + fn get(&self, name: &str) -> Result, EnvError> { + self.env.get(name) + } +} diff --git a/crates/library_config/Cargo.toml b/crates/library_config/Cargo.toml index be76bd45..b6deab09 100644 --- a/crates/library_config/Cargo.toml +++ b/crates/library_config/Cargo.toml @@ -8,7 +8,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] anyhow = "1" -libdd-library-config = { git = "https://github.com/DataDog/libdatadog.git", rev = "353134770b312b7ccd2df6afabc253090b948e5f" } +libdd-library-config = { git = "https://github.com/DataDog/libdatadog.git", rev = "0c6e2a5df2a163d34c4f385353ffc5d7257c72f4" } wasm-bindgen = "0.2.100" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index c4440cf9..8056816e 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -14,18 +14,17 @@ js-sys = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1" libdatadog-nodejs-capabilities = { path = "../capabilities" } -# TODO: Replace these temporary libdatadog PR revs with the official release/tag -# that contains DataDog/libdatadog#2235, then regenerate Cargo.lock. -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a" } -libdd-common = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false } -libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false, features = ["change-buffer"] } -libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false } -libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", rev = "6356aee4c69b51f27e7e82e43b1797f60b74905a", default-features = false } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f" } +libdd-common = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false, features = ["telemetry"] } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false, features = ["change-buffer"] } +libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } rmp-serde = "1" bytes = "1" http = "1" +web-time = "1" console_error_panic_hook = "0.1" [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/pipeline/src/lib.rs b/crates/pipeline/src/lib.rs index 31188af5..eb25866e 100644 --- a/crates/pipeline/src/lib.rs +++ b/crates/pipeline/src/lib.rs @@ -1,7 +1,7 @@ use libdatadog_nodejs_capabilities::WasmCapabilities; use libdd_data_pipeline::trace_exporter::agent_response::AgentResponse; use libdd_data_pipeline::trace_exporter::{ - TraceExporter, TraceExporterBuilder, TraceExporterOutputFormat, + TelemetryConfig, TraceExporter, TraceExporterBuilder, TraceExporterOutputFormat, }; use libdd_data_pipeline::OtlpProtocol; use libdd_shared_runtime::LocalRuntime; @@ -197,13 +197,17 @@ pub struct WasmSpanState { /// Datadog agent. libdatadog maps its internal traces to OTLP, so no /// JS-formatted spans are involved. Like `use_v05`, only takes effect if /// set before the first send (when the exporter is built). - otlp_endpoint: RefCell>, + otlp_endpoint: Cell>, /// OTLP wire protocol (`http/json` default, or `http/protobuf`). Only /// applied when `otlp_endpoint` is set. otlp_protocol: Cell>, /// Extra HTTP headers for OTLP export (e.g. collector auth), as key/value /// pairs. Only applied when `otlp_endpoint` is set. - otlp_headers: RefCell>, + otlp_headers: Cell>, + /// When set, the lazily-built exporter has telemetry enabled with this + /// config. Only takes effect if set before the first send + /// (when the exporter is built). + telemetry_config: Cell>, /// Latched message from a failed lazy `build_async`. Building is one-shot and /// a failure is fatal (bad config), so once set every send returns it (as a /// distinguishable error) instead of a misleading "builder already consumed", @@ -335,9 +339,10 @@ impl WasmSpanState { prepared_spans: RefCell::new(Vec::new()), sending: Cell::new(false), use_v05: Cell::new(false), - otlp_endpoint: RefCell::new(None), + otlp_endpoint: Cell::new(None), otlp_protocol: Cell::new(None), - otlp_headers: RefCell::new(Vec::new()), + otlp_headers: Cell::new(Vec::new()), + telemetry_config: Cell::new(None), build_error: RefCell::new(None), }) } @@ -359,7 +364,7 @@ impl WasmSpanState { /// Takes precedence over `setUseV05` (OTLP bypasses the agent entirely). #[wasm_bindgen(js_name = "setOtlpEndpoint")] pub fn set_otlp_endpoint(&self, url: String) { - *self.otlp_endpoint.borrow_mut() = Some(url); + self.otlp_endpoint.set(Some(url)); } /// Select the OTLP wire protocol: `http/json` (default) or `http/protobuf`. @@ -374,6 +379,31 @@ impl WasmSpanState { Ok(()) } + /// Enable telemetry on the lazily-built trace exporter. + /// + /// Must be called before the first `sendPreparedChunk`. Later calls have + /// no effect. + /// + /// # Arguments + /// + /// - `heartbeat_ms`: sets the metric-flush cadence. Set to 0 to defer to + /// libdatadog's default interval. + /// - `runtime_id`: tags telemetry payloads with the tracer's runtime id when provided + /// - `debug_enabled`: toggles libdd-telemetry's verbose logging + #[wasm_bindgen(js_name = "enableTelemetry")] + pub fn enable_telemetry( + &self, + heartbeat_ms: u32, + runtime_id: Option, + debug_enabled: bool, + ) { + self.telemetry_config.set(Some(TelemetryConfig { + heartbeat: u64::from(heartbeat_ms), + runtime_id, + debug_enabled, + })); + } + /// Set extra HTTP headers for OTLP export as a flat `[key, value, ...]` /// array (the host flattens its key/value map). Only takes effect with an /// OTLP endpoint set, before the first send. A trailing unpaired element on @@ -386,7 +416,7 @@ impl WasmSpanState { .chunks_exact(2) .map(|pair| (pair[0].clone(), pair[1].clone())) .collect(); - *self.otlp_headers.borrow_mut() = headers; + self.otlp_headers.set(headers); } #[wasm_bindgen] @@ -520,16 +550,19 @@ impl WasmSpanState { // When an OTLP endpoint is configured, libdatadog exports traces via // OTLP HTTP to that endpoint instead of the Datadog agent (mutually // exclusive with the agent v0.4/v0.5 path). - if let Some(url) = self.otlp_endpoint.borrow().as_deref() { - builder.set_otlp_endpoint(url); - if let Some(protocol) = self.otlp_protocol.get() { + if let Some(url) = self.otlp_endpoint.take() { + builder.set_otlp_endpoint(&url); + if let Some(protocol) = self.otlp_protocol.take() { builder.set_otlp_protocol(protocol); } - let headers = self.otlp_headers.borrow(); + let headers = self.otlp_headers.take(); if !headers.is_empty() { - builder.set_otlp_headers(headers.clone()); + builder.set_otlp_headers(headers); } } + if let Some(cfg) = self.telemetry_config.take() { + builder.enable_telemetry(cfg); + } match builder.build_async::().await { Ok(built) => *exporter_slot = Some(built), Err(e) => { diff --git a/crates/pipeline/src/stats.rs b/crates/pipeline/src/stats.rs index 127a21e6..3e65cb46 100644 --- a/crates/pipeline/src/stats.rs +++ b/crates/pipeline/src/stats.rs @@ -7,13 +7,13 @@ //! HTTP transport for flushing stats to the Datadog agent's `/v0.6/stats` //! endpoint. -use std::time::{Duration, SystemTime}; +use web_time::{Duration, SystemTime}; -/// Wall-clock now() for wasm. `std::time::SystemTime::now()` is unimplemented on -/// `wasm32-unknown-unknown` (it panics/traps), so derive the time from JS -/// `Date.now()` (milliseconds since the Unix epoch). +/// Wall-clock now() for wasm. Delegates to `web_time::SystemTime::now()`, +/// which routes to JS `Date.now()` on `wasm32-unknown-unknown` (native +/// `std::time::SystemTime::now()` is unimplemented on that target and traps). fn now() -> SystemTime { - SystemTime::UNIX_EPOCH + Duration::from_millis(js_sys::Date::now() as u64) + SystemTime::now() } use bytes::Bytes; @@ -67,6 +67,7 @@ impl StatsCollector { ], Vec::new(), None, + Vec::new(), ), meta, agent_url, diff --git a/crates/process_discovery/Cargo.toml b/crates/process_discovery/Cargo.toml index 44042286..32e9449e 100644 --- a/crates/process_discovery/Cargo.toml +++ b/crates/process_discovery/Cargo.toml @@ -8,11 +8,8 @@ crate-type = ["cdylib", "rlib"] [dependencies] anyhow = "1" -# Pointed at the merge commit that introduced ThreadLocalMetadata (caller-supplied -# schema version + extra process-context attributes). Swap back to a tagged release -# once one that includes 7cdeb7896e92d1ba38bde495934e112dac2eda25 is published. -libdd-library-config = { git = "https://github.com/DataDog/libdatadog.git", rev = "7cdeb7896e92d1ba38bde495934e112dac2eda25", features = ["otel-thread-ctx"] } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "7cdeb7896e92d1ba38bde495934e112dac2eda25" } +libdd-library-config = { git = "https://github.com/DataDog/libdatadog.git", rev = "0c6e2a5df2a163d34c4f385353ffc5d7257c72f4", features = ["otel-thread-ctx"] } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "0c6e2a5df2a163d34c4f385353ffc5d7257c72f4" } napi = { version = "2" } napi-derive = { version = "2", default-features = false } diff --git a/crates/sketches/Cargo.toml b/crates/sketches/Cargo.toml new file mode 100644 index 00000000..6747f461 --- /dev/null +++ b/crates/sketches/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sketches" +version = "0.1.0" +edition = "2021" +description = "Wasm bindings for Datadog's DDSketch" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +libdd-ddsketch = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f" } +wasm-bindgen = "0.2" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = ["-O", "--enable-bulk-memory", "--enable-nontrapping-float-to-int"] diff --git a/crates/sketches/src/lib.rs b/crates/sketches/src/lib.rs new file mode 100644 index 00000000..0b55db3d --- /dev/null +++ b/crates/sketches/src/lib.rs @@ -0,0 +1,37 @@ +use libdd_ddsketch::DDSketch as InnerDDSketch; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[derive(Default)] +pub struct DDSketch { + inner: InnerDDSketch, +} + +#[wasm_bindgen] +impl DDSketch { + #[wasm_bindgen(constructor)] + pub fn new() -> Self { + Self::default() + } + + pub fn add(&mut self, point: f64) -> Result<(), JsError> { + self.inner + .add(point) + .map_err(|error| JsError::new(&error.to_string())) + } + + #[wasm_bindgen(js_name = addWithCount)] + pub fn add_with_count(&mut self, point: f64, count: f64) -> Result<(), JsError> { + self.inner + .add_with_count(point, count) + .map_err(|error| JsError::new(&error.to_string())) + } + + pub fn count(&self) -> f64 { + self.inner.count() + } + + pub fn encode(&self) -> Vec { + self.inner.clone().encode_to_vec() + } +} diff --git a/eslint.config.js b/eslint.config.js index e7b67365..690debfb 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -80,9 +80,9 @@ module.exports = [ }, }, { - // Loaded by Rust via `wasm_bindgen(module = ".../http_transport.js")`, so the - // snake_case filename must match the Rust module path. - files: ['**/http_transport.js'], + // Loaded by Rust via `wasm_bindgen(module = ".../_transport.js")`, so + // the snake_case filename must match the Rust module path. + files: ['**/*_transport.js'], rules: { 'unicorn/filename-case': 'off', }, diff --git a/package.json b/package.json index 6bd8a096..47982da9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@datadog/libdatadog", - "version": "0.18.1", + "version": "0.19.0", "description": "Node.js binding for libdatadog", "main": "index.js", "scripts": { @@ -9,7 +9,7 @@ "build-debug": "mkdir -p target && yarn -s cargo-build > ./target/out.ndjson && yarn -s copy-artifacts", "build-release": "mkdir -p target && yarn -s cargo-build-release > ./target/out.ndjson && yarn -s copy-artifacts", "build-all": "mkdir -p target && yarn -s cargo-build -- --workspace > ./target/out.ndjson && yarn -s copy-artifacts && yarn -s build-wasm", - "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js library_config && node scripts/build-wasm.js datadog-js-zstd && node scripts/build-wasm.js pipeline", + "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js", "cargo-build-release": "yarn -s cargo-build -- --release", "cargo-build": "cargo build --message-format=json-render-diagnostics", "copy-artifacts": "node ./scripts/copy-artifacts", diff --git a/scripts/build-wasm.js b/scripts/build-wasm.js index b5a20a6d..93b338e6 100644 --- a/scripts/build-wasm.js +++ b/scripts/build-wasm.js @@ -15,7 +15,12 @@ const childProcess = require('node:child_process') const isMacOS = os.platform() === 'darwin' const noWasmOpt = isMacOS ? '--no-opt' : '' -const library = process.argv[2] +const libraries = [ + 'library_config', + 'datadog-js-zstd', + 'pipeline', + 'sketches', +] const env = { ...process.env, @@ -44,8 +49,10 @@ if (isMacOS) { env.CXX_wasm32_unknown_unknown = `${llvmBinDir}/clang++` } -childProcess.execSync( - `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ../../prebuilds/${library}`, { - env, - }, -) +for (const library of libraries) { + childProcess.execSync( + `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ../../prebuilds/${library}`, { + env, + }, + ) +} diff --git a/test/env-transport.js b/test/env-transport.js new file mode 100644 index 00000000..f45a3779 --- /dev/null +++ b/test/env-transport.js @@ -0,0 +1,31 @@ +'use strict' + +// The transport shim is plain CommonJS, so drive it directly. + +const { describe, it, before, after } = require('node:test') +const assert = require('node:assert') + +const envTransport = require('../crates/capabilities/src/env_transport') + +describe('env_transport', () => { + const NAME = 'LIBDD_CAP_TEST_ENV_TRANSPORT' + let savedValue + + before(() => { + savedValue = process.env[NAME] + }) + after(() => { + if (savedValue === undefined) delete process.env[NAME] + else process.env[NAME] = savedValue + }) + + it('returns undefined for an unset var', () => { + delete process.env[NAME] + assert.strictEqual(envTransport.get(NAME), undefined) + }) + + it('returns the value when the var is set', () => { + process.env[NAME] = 'value1' + assert.strictEqual(envTransport.get(NAME), 'value1') + }) +}) diff --git a/test/filesystem.js b/test/filesystem.js new file mode 100644 index 00000000..ec14bccd --- /dev/null +++ b/test/filesystem.js @@ -0,0 +1,51 @@ +'use strict' + +// The shim is plain CommonJS, so drive it directly. + +const { describe, it, before, after } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const filesystem = require('../crates/capabilities/src/filesystem') + +describe('filesystem', () => { + let tmp + before(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'libdd-file-')) + }) + after(() => { + fs.rmSync(tmp, { recursive: true, force: true }) + }) + + it('writes and reads a file round-trip', async () => { + const p = path.join(tmp, 'hello.bin') + await filesystem.writeFile(p, Buffer.from('hello')) + const got = await filesystem.readFile(p) + assert.strictEqual(Buffer.from(got).toString('utf8'), 'hello') + }) + + it('readFile on a missing path rejects with ENOENT', async () => { + const p = path.join(tmp, 'does-not-exist') + await assert.rejects(filesystem.readFile(p), error => error && error.code === 'ENOENT') + }) + + it('metadata reports size, kind, and a positive inode', async () => { + const p = path.join(tmp, 'meta.bin') + fs.writeFileSync(p, '0123456789') + const m = await filesystem.metadata(p) + assert.strictEqual(m.size, 10n) + assert.strictEqual(m.is_file, true) + assert.strictEqual(m.is_dir, false) + assert.ok(m.inode > 0n, `expected positive inode, got ${m.inode}`) + }) + + it('exists returns true for a present path and false for a missing one', async () => { + const present = path.join(tmp, 'here') + fs.writeFileSync(present, '') + const absent = path.join(tmp, 'gone') + assert.strictEqual(await filesystem.exists(present), true) + assert.strictEqual(await filesystem.exists(absent), false) + }) +}) diff --git a/test/wasm/sketches/index.js b/test/wasm/sketches/index.js new file mode 100644 index 00000000..dfae84f6 --- /dev/null +++ b/test/wasm/sketches/index.js @@ -0,0 +1,29 @@ +'use strict' + +const assert = require('node:assert') + +const loader = require('../../../load') +const { DDSketch } = loader.load('sketches') + +const sketch = new DDSketch() +assert.strictEqual(sketch.count(), 0) + +sketch.add(1) +sketch.addWithCount(2, 3) +assert.strictEqual(sketch.count(), 4) + +assert.throws(() => sketch.add(-1), /point is invalid/) +assert.throws(() => sketch.addWithCount(1, Number.NaN), /count is invalid/) + +const encoded = sketch.encode() +assert(encoded instanceof Uint8Array) +assert(encoded.length > 0) + +assert.strictEqual(sketch.count(), 4) +sketch.add(3) +assert.strictEqual(sketch.count(), 5) + +const reencoded = sketch.encode() +assert(reencoded instanceof Uint8Array) +assert(reencoded.length > 0) +assert.strictEqual(sketch.count(), 5)