From cdb19a5ede9cc58293ebd3122cc8886e9a5a7ca9 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 8 Sep 2026 15:56:36 +0000 Subject: [PATCH 1/2] feat(storage): add OpenTelemetry metrics gating and configuration Introduce OpenTelemetry client metrics infrastructure and gating for Google Cloud Storage. - Add _opentelemetry_metrics module with a hidden development gate (_ENABLE_METRICS_DEV_GATE) and environment variable handling (GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS, GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS). - Add enable_metrics and enable_advanced_metrics options and properties to Client. - Add unit tests for gating resolution, environment variable overrides, and Client configuration. Refs: b/489239033 --- .../cloud/storage/_opentelemetry_metrics.py | 124 +++++++++++++++ .../google/cloud/storage/client.py | 32 ++++ .../tests/unit/test__opentelemetry_metrics.py | 142 ++++++++++++++++++ .../tests/unit/test_client.py | 44 ++++++ 4 files changed, 342 insertions(+) create mode 100644 packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py create mode 100644 packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py new file mode 100644 index 000000000000..a967b1d6f746 --- /dev/null +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -0,0 +1,124 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Manages OpenTelemetry metrics instruments and gating for GCS client.""" + +import logging +import os +from typing import Any, Dict, Optional + +from google.cloud.storage.version import __version__ + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# 1. Hidden Development Gate +# --------------------------------------------------------------------------- +# Must remain False in production branches. +# Only enabled during active development and test runs. +_ENABLE_METRICS_DEV_GATE = False + +# --------------------------------------------------------------------------- +# 2. Standardized Configuration and Environment Variable Names +# --------------------------------------------------------------------------- +ENABLE_OTEL_METRICS_ENV_VAR = "GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS" +ENABLE_DEBUG_METRICS_ENV_VAR = "GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS" + +_DEFAULT_ENABLE_METRICS = False +_DEFAULT_ENABLE_DEBUG_METRICS = False + +# --------------------------------------------------------------------------- +# 3. Optional OpenTelemetry Dependency Check +# --------------------------------------------------------------------------- +try: + from opentelemetry import metrics + + HAS_OPENTELEMETRY_METRICS = True +except ImportError: + HAS_OPENTELEMETRY_METRICS = False + logger.debug( + "OpenTelemetry metrics package is not installed. " + "GCS client metrics are disabled." + ) + + +def _parse_bool_env(name: str, default: bool = False) -> bool: + """Parses a boolean from an environment variable.""" + val = os.environ.get(name) + if val is None: + return default + return str(val).strip().lower() in {"1", "true", "yes", "on"} + + +def is_metrics_enabled(client_setting: Optional[bool] = None) -> bool: + """Evaluates whether standard GCS metrics should be recorded. + + Args: + client_setting: Optional boolean configured on the client instance. + Takes precedence over the environment variable if specified. + + Returns: + bool: True if metrics recording is enabled, False otherwise. + """ + if not HAS_OPENTELEMETRY_METRICS: + return False + + if not _ENABLE_METRICS_DEV_GATE: + return False + + if client_setting is not None: + return bool(client_setting) + + return _parse_bool_env(ENABLE_OTEL_METRICS_ENV_VAR, _DEFAULT_ENABLE_METRICS) + + +def is_advanced_metrics_enabled(client_setting: Optional[bool] = None) -> bool: + """Evaluates whether high-frequency debug metrics should be recorded. + + Args: + client_setting: Optional boolean configured on the client instance. + Takes precedence over the environment variable if specified. + + Returns: + bool: True if advanced metrics recording is enabled, False otherwise. + """ + if not is_metrics_enabled(client_setting): + return False + + if client_setting is not None: + return bool(client_setting) + + return _parse_bool_env( + ENABLE_DEBUG_METRICS_ENV_VAR, _DEFAULT_ENABLE_DEBUG_METRICS + ) + + +# --------------------------------------------------------------------------- +# 4. Standard Common Attributes & Meter Provider +# --------------------------------------------------------------------------- +def get_common_attributes() -> Dict[str, Any]: + """Returns standard GCS client attributes for metrics.""" + return { + "gcp.client.service": "storage", + "gcp.client.version": __version__, + "gcp.client.repo": "googleapis/google-cloud-python", + "gcp.client.artifact": "google-cloud-storage", + } + + +def get_meter(): + """Returns the OpenTelemetry Meter for Google Cloud Storage.""" + if not HAS_OPENTELEMETRY_METRICS: + return None + return metrics.get_meter("google.cloud.storage", __version__) diff --git a/packages/google-cloud-storage/google/cloud/storage/client.py b/packages/google-cloud-storage/google/cloud/storage/client.py index d109ea3db095..f0def181ce3d 100644 --- a/packages/google-cloud-storage/google/cloud/storage/client.py +++ b/packages/google-cloud-storage/google/cloud/storage/client.py @@ -126,6 +126,18 @@ class Client(ClientWithProject): (Optional) An API key. Mutually exclusive with any other credentials. This parameter is an alias for setting `client_options.api_key` and will supercede any api key set in the `client_options` parameter. + + :type enable_metrics: bool or None + :param enable_metrics: + (Optional) Whether to enable OpenTelemetry metrics. If None, falls back + to the GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS environment variable, + or False if unset. + + :type enable_advanced_metrics: bool or None + :param enable_advanced_metrics: + (Optional) Whether to enable advanced/debug OpenTelemetry metrics. If + None, falls back to the GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS + environment variable, or False if unset. Requires enable_metrics to be True. """ SCOPE = ( @@ -146,6 +158,8 @@ def __init__( extra_headers={}, *, api_key=None, + enable_metrics=None, + enable_advanced_metrics=None, ): self._base_connection = None @@ -293,6 +307,24 @@ def __init__( self._connection = connection self._batch_stack = _LocalStack() self._bucket_metadata_cache = BucketMetadataCache(self) + self._enable_metrics = enable_metrics + self._enable_advanced_metrics = enable_advanced_metrics + + @property + def metrics_enabled(self) -> bool: + """Returns True if metrics recording is active for this client.""" + from google.cloud.storage import _opentelemetry_metrics + + return _opentelemetry_metrics.is_metrics_enabled(self._enable_metrics) + + @property + def advanced_metrics_enabled(self) -> bool: + """Returns True if advanced metrics recording is active for this client.""" + from google.cloud.storage import _opentelemetry_metrics + + return _opentelemetry_metrics.is_advanced_metrics_enabled( + self._enable_advanced_metrics + ) def close(self): """Close the client and clear any cached metadata or active connections.""" diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py new file mode 100644 index 000000000000..09c2ecae0fbe --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -0,0 +1,142 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage.version import __version__ + + +@pytest.mark.parametrize( + "env_val,default,expected", + [ + ("1", False, True), + ("true", False, True), + ("True", False, True), + ("yes", False, True), + ("on", False, True), + ("0", True, False), + ("false", True, False), + ("no", True, False), + ("off", True, False), + ("invalid", False, False), + ], +) +def test_parse_bool_env(monkeypatch, env_val, default, expected): + monkeypatch.setenv("TEST_BOOL_ENV", env_val) + assert _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV", default) == expected + + +def test_parse_bool_env_default(monkeypatch): + monkeypatch.delenv("TEST_BOOL_ENV_MISSING", raising=False) + assert _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV_MISSING", True) is True + assert _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV_MISSING", False) is False + + +def test_dev_gate_locked_disables_metrics(monkeypatch): + """When _ENABLE_METRICS_DEV_GATE is False, metrics must remain disabled.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + + assert _opentelemetry_metrics.is_metrics_enabled() is False + assert _opentelemetry_metrics.is_metrics_enabled(client_setting=True) is False + + +def test_dev_gate_unlocked_respects_env_var(monkeypatch): + """When dev gate is open, environment variable enables metrics.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + assert _opentelemetry_metrics.is_metrics_enabled() is True + + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "false") + assert _opentelemetry_metrics.is_metrics_enabled() is False + + +def test_metrics_default_is_disabled_when_env_unset(monkeypatch): + """When env var is unset, default value is False.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.delenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", raising=False) + + assert _opentelemetry_metrics.is_metrics_enabled() is False + + +def test_client_setting_overrides_env_var(monkeypatch): + """Client constructor parameter must take precedence over env var.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + # Client disables while env var is True + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + assert _opentelemetry_metrics.is_metrics_enabled(client_setting=False) is False + + # Client enables while env var is False + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "false") + assert _opentelemetry_metrics.is_metrics_enabled(client_setting=True) is True + + +def test_advanced_metrics_requires_base_metrics(monkeypatch): + """Advanced metrics cannot be active if base metrics are disabled.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "false") + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS", "true") + + assert _opentelemetry_metrics.is_advanced_metrics_enabled() is False + + +def test_advanced_metrics_enabled(monkeypatch): + """Advanced metrics is enabled when both base and debug flags are True.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS", "true") + + assert _opentelemetry_metrics.is_advanced_metrics_enabled() is True + + +def test_advanced_metrics_client_setting_overrides_env_var(monkeypatch): + """Client setting overrides advanced metrics env var.""" + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_DEBUG_METRICS", "false") + + assert ( + _opentelemetry_metrics.is_advanced_metrics_enabled(client_setting=True) is True + ) + + +def test_otel_missing_disables_metrics(monkeypatch): + """If opentelemetry-api is not installed, metrics must gracefully disable.""" + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", False) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setenv("GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS", "true") + + assert _opentelemetry_metrics.is_metrics_enabled() is False + assert _opentelemetry_metrics.get_meter() is None + + +def test_get_common_attributes(): + """Verify common attributes conform to GCS OTel specification.""" + attrs = _opentelemetry_metrics.get_common_attributes() + assert attrs["gcp.client.service"] == "storage" + assert attrs["gcp.client.version"] == __version__ + assert attrs["gcp.client.repo"] == "googleapis/google-cloud-python" + assert attrs["gcp.client.artifact"] == "google-cloud-storage" + + +def test_get_meter(): + """Verify get_meter returns a meter or None.""" + meter = _opentelemetry_metrics.get_meter() + if _opentelemetry_metrics.HAS_OPENTELEMETRY_METRICS: + assert meter is not None + else: + assert meter is None diff --git a/packages/google-cloud-storage/tests/unit/test_client.py b/packages/google-cloud-storage/tests/unit/test_client.py index 9de5cf366688..488de0962df9 100644 --- a/packages/google-cloud-storage/tests/unit/test_client.py +++ b/packages/google-cloud-storage/tests/unit/test_client.py @@ -259,6 +259,50 @@ def test_ctor_w_universe_domain_and_matched_credentials(self): self.assertEqual(client.api_endpoint, expected_api_endpoint) self.assertEqual(client.universe_domain, universe_domain) + def test_ctor_w_enable_metrics(self): + PROJECT = "PROJECT" + client = self._make_one(project=PROJECT, enable_metrics=True) + self.assertTrue(client._enable_metrics) + + def test_ctor_w_enable_advanced_metrics(self): + PROJECT = "PROJECT" + client = self._make_one(project=PROJECT, enable_advanced_metrics=True) + self.assertTrue(client._enable_advanced_metrics) + + def test_client_metrics_enabled_property(self): + from google.cloud.storage import _opentelemetry_metrics + + PROJECT = "PROJECT" + client = self._make_one(project=PROJECT, enable_metrics=True) + + with mock.patch.object( + _opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True + ): + self.assertTrue(client.metrics_enabled) + + with mock.patch.object( + _opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False + ): + self.assertFalse(client.metrics_enabled) + + def test_client_advanced_metrics_enabled_property(self): + from google.cloud.storage import _opentelemetry_metrics + + PROJECT = "PROJECT" + client = self._make_one( + project=PROJECT, enable_metrics=True, enable_advanced_metrics=True + ) + + with mock.patch.object( + _opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True + ): + self.assertTrue(client.advanced_metrics_enabled) + + with mock.patch.object( + _opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False + ): + self.assertFalse(client.advanced_metrics_enabled) + def test_ctor_w_universe_domain_and_mismatched_credentials(self): PROJECT = "PROJECT" universe_domain = "example.com" From 6ced32b58037c2d4750f46bb2855bbe7b1e8c88d Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 17 Sep 2026 08:48:45 +0000 Subject: [PATCH 2/2] fix(storage): resolve ruff formatting and mock credentials in metrics tests - Format _opentelemetry_metrics.py and test__opentelemetry_metrics.py to match ruff 88-character line length limit. - Pass mock credentials to Client constructor in unit tests to prevent DefaultCredentialsError on CI runners. Refs: b/489239033 [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 4 +--- .../tests/unit/test__opentelemetry_metrics.py | 4 +++- .../tests/unit/test_client.py | 21 +++++++++++++++---- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index a967b1d6f746..4f21c56b453f 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -99,9 +99,7 @@ def is_advanced_metrics_enabled(client_setting: Optional[bool] = None) -> bool: if client_setting is not None: return bool(client_setting) - return _parse_bool_env( - ENABLE_DEBUG_METRICS_ENV_VAR, _DEFAULT_ENABLE_DEBUG_METRICS - ) + return _parse_bool_env(ENABLE_DEBUG_METRICS_ENV_VAR, _DEFAULT_ENABLE_DEBUG_METRICS) # --------------------------------------------------------------------------- diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 09c2ecae0fbe..3d304cbb0460 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -41,7 +41,9 @@ def test_parse_bool_env(monkeypatch, env_val, default, expected): def test_parse_bool_env_default(monkeypatch): monkeypatch.delenv("TEST_BOOL_ENV_MISSING", raising=False) assert _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV_MISSING", True) is True - assert _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV_MISSING", False) is False + assert ( + _opentelemetry_metrics._parse_bool_env("TEST_BOOL_ENV_MISSING", False) is False + ) def test_dev_gate_locked_disables_metrics(monkeypatch): diff --git a/packages/google-cloud-storage/tests/unit/test_client.py b/packages/google-cloud-storage/tests/unit/test_client.py index 488de0962df9..77d06248c1ea 100644 --- a/packages/google-cloud-storage/tests/unit/test_client.py +++ b/packages/google-cloud-storage/tests/unit/test_client.py @@ -261,19 +261,28 @@ def test_ctor_w_universe_domain_and_matched_credentials(self): def test_ctor_w_enable_metrics(self): PROJECT = "PROJECT" - client = self._make_one(project=PROJECT, enable_metrics=True) + credentials = _make_credentials() + client = self._make_one( + project=PROJECT, credentials=credentials, enable_metrics=True + ) self.assertTrue(client._enable_metrics) def test_ctor_w_enable_advanced_metrics(self): PROJECT = "PROJECT" - client = self._make_one(project=PROJECT, enable_advanced_metrics=True) + credentials = _make_credentials() + client = self._make_one( + project=PROJECT, credentials=credentials, enable_advanced_metrics=True + ) self.assertTrue(client._enable_advanced_metrics) def test_client_metrics_enabled_property(self): from google.cloud.storage import _opentelemetry_metrics PROJECT = "PROJECT" - client = self._make_one(project=PROJECT, enable_metrics=True) + credentials = _make_credentials() + client = self._make_one( + project=PROJECT, credentials=credentials, enable_metrics=True + ) with mock.patch.object( _opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True @@ -289,8 +298,12 @@ def test_client_advanced_metrics_enabled_property(self): from google.cloud.storage import _opentelemetry_metrics PROJECT = "PROJECT" + credentials = _make_credentials() client = self._make_one( - project=PROJECT, enable_metrics=True, enable_advanced_metrics=True + project=PROJECT, + credentials=credentials, + enable_metrics=True, + enable_advanced_metrics=True, ) with mock.patch.object(