diff --git a/.changes/next-release/bugfix-dynamodb-49868.json b/.changes/next-release/bugfix-dynamodb-49868.json new file mode 100644 index 000000000000..9bc06cb55fd6 --- /dev/null +++ b/.changes/next-release/bugfix-dynamodb-49868.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "dynamodb", + "description": "``dynamodb scan`` and ``dynamodb query`` now sum ``ConsumedCapacity`` across all pages when auto-pagination is enabled, instead of reporting only the final page. Includes nested table and secondary-index capacity (fixes CLI-4199)." +} diff --git a/awscli/botocore/data/dynamodb/2012-08-10/paginators-1.sdk-extras.json b/awscli/botocore/data/dynamodb/2012-08-10/paginators-1.sdk-extras.json new file mode 100644 index 000000000000..59d7be9cb0ee --- /dev/null +++ b/awscli/botocore/data/dynamodb/2012-08-10/paginators-1.sdk-extras.json @@ -0,0 +1,38 @@ +{ + "merge": { + "pagination": { + "Query": { + "aggregate_numeric_keys": [ + "ConsumedCapacity.CapacityUnits", + "ConsumedCapacity.ReadCapacityUnits", + "ConsumedCapacity.WriteCapacityUnits", + "ConsumedCapacity.Table.CapacityUnits", + "ConsumedCapacity.Table.ReadCapacityUnits", + "ConsumedCapacity.Table.WriteCapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.CapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.ReadCapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.WriteCapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.CapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.ReadCapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.WriteCapacityUnits" + ] + }, + "Scan": { + "aggregate_numeric_keys": [ + "ConsumedCapacity.CapacityUnits", + "ConsumedCapacity.ReadCapacityUnits", + "ConsumedCapacity.WriteCapacityUnits", + "ConsumedCapacity.Table.CapacityUnits", + "ConsumedCapacity.Table.ReadCapacityUnits", + "ConsumedCapacity.Table.WriteCapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.CapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.ReadCapacityUnits", + "ConsumedCapacity.GlobalSecondaryIndexes.*.WriteCapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.CapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.ReadCapacityUnits", + "ConsumedCapacity.LocalSecondaryIndexes.*.WriteCapacityUnits" + ] + } + } + } +} diff --git a/awscli/botocore/paginate.py b/awscli/botocore/paginate.py index 24889c9600cd..7f35f24e54fc 100644 --- a/awscli/botocore/paginate.py +++ b/awscli/botocore/paginate.py @@ -14,6 +14,7 @@ import base64 import json import logging +from copy import deepcopy from functools import partial from itertools import tee @@ -27,6 +28,59 @@ log = logging.getLogger(__name__) +def _is_summable_number(value): + # Match the numeric types used by the existing result_key aggregation + # (int, float). Strings are intentionally excluded here (unlike that path, + # which concatenates them) so string leaves such as TableName are kept + # from the first page rather than concatenated. Booleans are ints in + # Python but should never be summed as numbers. + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _add_numeric_path(accumulator, page_value, segments): + """Add the numeric leaf at ``segments`` from ``page_value`` into ``accumulator``. + + ``segments`` is an explicit path within a response member, e.g. + ``('CapacityUnits',)``, ``('Table', 'CapacityUnits')``, or + ``('GlobalSecondaryIndexes', '*', 'CapacityUnits')``. A ``'*'`` segment + matches every key at that level (for maps keyed by runtime-defined names + such as index names); every other segment is matched literally. + + Only the exact leaf named by the full path is summed — this is a strict, + path-based allowlist. Intermediate keys seen for the first time (e.g. a new + index name on a later page) are seeded by deep-copy; anything not on a + configured path is left untouched (it was seeded from the first page). + Booleans and cross-page type mismatches are never summed. + """ + if not segments: + return + if not isinstance(page_value, dict) or not isinstance(accumulator, dict): + return + seg, rest = segments[0], segments[1:] + keys = ( + list(page_value) + if seg == '*' + else ([seg] if seg in page_value else []) + ) + for key in keys: + value = page_value[key] + if rest: + if key not in accumulator: + # First time this (possibly dynamic) key appears; seed subtree. + accumulator[key] = deepcopy(value) + else: + _add_numeric_path(accumulator[key], value, rest) + elif key not in accumulator: + accumulator[key] = ( + deepcopy(value) if isinstance(value, (dict, list)) else value + ) + elif _is_summable_number(value) and _is_summable_number( + accumulator[key] + ): + accumulator[key] = accumulator[key] + value + # else: non-numeric or cross-page type mismatch -> keep first value. + + class TokenEncoder: """Encodes dictionaries into opaque strings. @@ -201,6 +255,7 @@ def __init__( starting_token, page_size, op_kwargs, + aggregate_numeric_keys=None, ): self._method = method self._input_token = input_token @@ -214,6 +269,8 @@ def __init__( self._op_kwargs = op_kwargs self._resume_token = None self._non_aggregate_key_exprs = non_aggregate_keys + # Maps member -> list of segment-tuples; may be omitted by callers. + self._aggregate_numeric_keys = aggregate_numeric_keys or {} self._non_aggregate_part = {} self._token_encoder = TokenEncoder() self._token_decoder = TokenDecoder() @@ -480,6 +537,9 @@ def result_key_iters(self): def build_full_result(self): complete_result = {} + # Running totals for members that are aggregated by recursively + # summing their numeric leaves (e.g. DynamoDB's ConsumedCapacity). + aggregate_numeric_totals = {} for response in self: page = response # We want to try to catch operation object pagination @@ -489,6 +549,36 @@ def build_full_result(self): # uses. We can remove it though once operation objects are removed. if isinstance(response, tuple) and len(response) == 2: page = response[1] + for member, paths in self._aggregate_numeric_keys.items(): + page_value = page.get(member) + if page_value is None: + continue + if member not in aggregate_numeric_totals: + # Seed the whole member from the first page it appears on; + # later pages only add the configured leaf paths, so any + # field NOT on a path keeps this first-page value. + aggregate_numeric_totals[member] = ( + deepcopy(page_value) + if isinstance(page_value, (dict, list)) + else page_value + ) + elif isinstance(page_value, dict) and isinstance( + aggregate_numeric_totals[member], dict + ): + for segments in paths: + if segments: + _add_numeric_path( + aggregate_numeric_totals[member], + page_value, + segments, + ) + elif _is_summable_number(page_value) and _is_summable_number( + aggregate_numeric_totals[member] + ): + # The whole member is a bare number (opted in as a path with + # no leaf, e.g. just "SomeCount"). + aggregate_numeric_totals[member] += page_value + # Any other/unexpected shape: keep the first page's value. # We're incrementally building the full response page # by page. For each page in the response we need to # inject the necessary components from the page @@ -523,6 +613,9 @@ def build_full_result(self): existing_value + result_value, ) merge_dicts(complete_result, self.non_aggregate_part) + # Overlay the recursively-summed totals last so they take precedence + # over any single-page value merged in via the non-aggregate keys. + complete_result.update(aggregate_numeric_totals) if self.resume_token is not None: complete_result['NextToken'] = self.resume_token return complete_result @@ -602,6 +695,9 @@ def __init__(self, method, pagination_config, model): self._output_token = self._get_output_tokens(self._pagination_cfg) self._input_token = self._get_input_tokens(self._pagination_cfg) self._more_results = self._get_more_results_token(self._pagination_cfg) + self._aggregate_numeric_keys = self._get_aggregate_numeric_keys( + self._pagination_cfg + ) self._non_aggregate_keys = self._get_non_aggregate_keys( self._pagination_cfg ) @@ -615,9 +711,41 @@ def result_keys(self): def _get_non_aggregate_keys(self, config): keys = [] for key in config.get('non_aggregate_keys', []): + # A member that is aggregated across pages takes precedence over + # any non-aggregate declaration for the same member (or a path + # nested under it). This lets a member be moved to + # aggregate_numeric_keys via an overlay without having to edit the + # upstream-synced non_aggregate_keys list (and lets that list keep + # receiving unrelated upstream additions). + if self._is_aggregated(key): + continue keys.append(jmespath.compile(key)) return keys + def _is_aggregated(self, non_aggregate_key): + for aggregate_key in self._aggregate_numeric_keys: + if ( + non_aggregate_key == aggregate_key + or non_aggregate_key.startswith(f'{aggregate_key}.') + ): + return True + return False + + def _get_aggregate_numeric_keys(self, config): + # A list of explicit dotted paths to numeric leaves that are summed + # across pages, e.g. "ConsumedCapacity.CapacityUnits" or + # "ConsumedCapacity.GlobalSecondaryIndexes.*.CapacityUnits" (a "*" + # segment matches every key at that level, for maps keyed by + # runtime-defined names). This is a strict allowlist: only the exact + # leaf at each full path is totaled; any other field (strings, or + # numbers on a path that is not configured) is preserved from the first + # page. Parsed into member -> list of segment-tuples (after the member). + paths = {} + for path in config.get('aggregate_numeric_keys', []): + segments = path.split('.') + paths.setdefault(segments[0], []).append(tuple(segments[1:])) + return paths + def _get_output_tokens(self, config): output = [] output_token = config['output_token'] @@ -670,6 +798,7 @@ def paginate(self, **kwargs): page_params['StartingToken'], page_params['PageSize'], kwargs, + self._aggregate_numeric_keys, ) def _extract_paging_params(self, kwargs): diff --git a/tests/functional/botocore/test_paginator_config.py b/tests/functional/botocore/test_paginator_config.py index 39c4c2ba04a2..b942cd032a56 100644 --- a/tests/functional/botocore/test_paginator_config.py +++ b/tests/functional/botocore/test_paginator_config.py @@ -26,6 +26,7 @@ 'limit_key', 'more_results', 'non_aggregate_keys', + 'aggregate_numeric_keys', ] ) MEMBER_NAME_CHARS = set(string.ascii_letters + string.digits) @@ -170,6 +171,45 @@ def test_lint_pagination_configs( _validate_new_numeric_keys( operation_name, page_config, service_model, record_property ) + _validate_aggregate_numeric_keys(operation_name, page_config) + + +def _validate_aggregate_numeric_keys(operation_name, page_config): + # aggregate_numeric_keys is a list of explicit dotted paths to the numeric + # leaves to sum, e.g. "ConsumedCapacity.CapacityUnits" or + # "ConsumedCapacity.GlobalSecondaryIndexes.*.CapacityUnits". A "*" matches + # every key at that level and may only appear as an interior segment (not + # the top-level member, which must be a real output member, nor the leaf). + config_value = page_config.get('aggregate_numeric_keys', []) + if not isinstance(config_value, list): + raise AssertionError( + f"aggregate_numeric_keys for operation {operation_name} must be a " + "list of dotted leaf paths." + ) + for path in config_value: + if not isinstance(path, str) or not path: + raise AssertionError( + f"aggregate_numeric_keys entry {path!r} for operation " + f"{operation_name} must be a non-empty string path." + ) + segments = path.split('.') + if any(segment == '' for segment in segments): + raise AssertionError( + f"aggregate_numeric_keys path '{path}' for operation " + f"{operation_name} has an empty segment (leading/trailing or " + "doubled '.')." + ) + if segments[0] == '*': + raise AssertionError( + f"aggregate_numeric_keys path '{path}' for operation " + f"{operation_name} must start with a top-level output member, " + "not '*'." + ) + if segments[-1] == '*': + raise AssertionError( + f"aggregate_numeric_keys path '{path}' for operation " + f"{operation_name} must end with a leaf field-name, not '*'." + ) def _validate_known_pagination_keys(page_config): @@ -323,8 +363,25 @@ def _get_all_page_output_keys(page_config): yield 'output_token', key if 'more_results' in page_config: yield 'more_results', page_config['more_results'] + # aggregate_numeric_keys is a list of dotted paths; only the top-level + # member of each path is an output member to account for. + aggregate_members = { + path.split('.')[0] + for path in page_config.get('aggregate_numeric_keys', []) + } for key in page_config.get('non_aggregate_keys', []): + # A member that is aggregated across pages takes precedence over any + # non_aggregate declaration for the same member (mirroring + # Paginator._get_non_aggregate_keys). Skip it here so it is only + # accounted for once. + if any( + key == member or key.startswith(f'{member}.') + for member in aggregate_members + ): + continue yield 'non_aggregate_keys', key + for member in aggregate_members: + yield 'aggregate_numeric_keys', member def _get_list_value(page_config, key): diff --git a/tests/functional/dynamodb/test_pagination.py b/tests/functional/dynamodb/test_pagination.py index 41f5f8da5e00..53b6cddf010c 100644 --- a/tests/functional/dynamodb/test_pagination.py +++ b/tests/functional/dynamodb/test_pagination.py @@ -17,7 +17,7 @@ class TestPagination(BaseAWSCommandParamsTest): def setUp(self): - super(TestPagination, self).setUp() + super().setUp() self.first_response = { "Items": [{"Key": {"B": "MjEzNw=="}}], "Count": 1, @@ -47,3 +47,110 @@ def test_pagination_disabled_works(self): stdout, _, _ = self.run_cmd(cmd, expected_rc=0) # Ensure the base64 encoded last evaluated key is in stdout self.assertIn('"MjEzNw=="', stdout) + + +class TestConsumedCapacityAggregation(BaseAWSCommandParamsTest): + """ConsumedCapacity must be summed across pages, not taken from one page. + + See CLI-4199: auto-pagination previously reported only a single page's + ConsumedCapacity, undercounting the true cost of a Scan/Query. + """ + + def _page(self, consumed_capacity, last_key=None): + page = { + "Items": [{"Key": {"S": "item"}}], + "Count": 1, + "ScannedCount": 1, + "ConsumedCapacity": consumed_capacity, + } + if last_key is not None: + page["LastEvaluatedKey"] = last_key + return page + + def test_scan_sums_total_consumed_capacity(self): + self.parsed_responses = [ + self._page( + {"TableName": "T", "CapacityUnits": 100.0}, + last_key={"Key": {"S": "a"}}, + ), + self._page({"TableName": "T", "CapacityUnits": 102.0}), + ] + cmd = ( + 'dynamodb scan --table-name T --output json ' + '--return-consumed-capacity TOTAL' + ) + stdout, _, _ = self.run_cmd(cmd, expected_rc=0) + result = json.loads(stdout) + self.assertEqual(result["ConsumedCapacity"]["CapacityUnits"], 202.0) + self.assertEqual(result["ConsumedCapacity"]["TableName"], "T") + self.assertEqual(result["Count"], 2) + self.assertEqual(result["ScannedCount"], 2) + + def test_scan_sums_index_consumed_capacity(self): + # Index maps are keyed by a user-defined index name; the total must + # sum each index across pages regardless of that name. + self.parsed_responses = [ + self._page( + { + "TableName": "T", + "CapacityUnits": 100.0, + "Table": {"CapacityUnits": 0.0}, + "GlobalSecondaryIndexes": { + "my-index": {"CapacityUnits": 100.0} + }, + }, + last_key={"Key": {"S": "a"}}, + ), + self._page( + { + "TableName": "T", + "CapacityUnits": 102.0, + "Table": {"CapacityUnits": 0.0}, + "GlobalSecondaryIndexes": { + "my-index": {"CapacityUnits": 102.0} + }, + } + ), + ] + cmd = ( + 'dynamodb scan --table-name T --index-name my-index ' + '--output json --return-consumed-capacity INDEXES' + ) + stdout, _, _ = self.run_cmd(cmd, expected_rc=0) + cc = json.loads(stdout)["ConsumedCapacity"] + self.assertEqual(cc["CapacityUnits"], 202.0) + self.assertEqual( + cc["GlobalSecondaryIndexes"]["my-index"]["CapacityUnits"], 202.0 + ) + + def test_query_sums_total_consumed_capacity(self): + self.parsed_responses = [ + self._page( + {"TableName": "T", "CapacityUnits": 5.5}, + last_key={"Key": {"S": "a"}}, + ), + self._page({"TableName": "T", "CapacityUnits": 4.5}), + ] + cmd = ( + 'dynamodb query --table-name T --output json ' + '--key-condition-expression Id=:id ' + '--expression-attribute-values {":id":{"S":"x"}} ' + '--return-consumed-capacity TOTAL' + ) + stdout, _, _ = self.run_cmd(cmd, expected_rc=0) + result = json.loads(stdout) + self.assertEqual(result["ConsumedCapacity"]["CapacityUnits"], 10.0) + + def test_no_consumed_capacity_key_when_not_requested(self): + self.parsed_responses = [ + { + "Items": [{"Key": {"S": "a"}}], + "Count": 1, + "ScannedCount": 1, + "LastEvaluatedKey": {"Key": {"S": "a"}}, + }, + {"Items": [{"Key": {"S": "b"}}], "Count": 1, "ScannedCount": 1}, + ] + cmd = 'dynamodb scan --table-name T --output json' + stdout, _, _ = self.run_cmd(cmd, expected_rc=0) + self.assertNotIn("ConsumedCapacity", json.loads(stdout)) diff --git a/tests/unit/botocore/test_paginate.py b/tests/unit/botocore/test_paginate.py index 8c20681e618a..8ebe6839fc51 100644 --- a/tests/unit/botocore/test_paginate.py +++ b/tests/unit/botocore/test_paginate.py @@ -18,6 +18,7 @@ PaginatorModel, TokenDecoder, TokenEncoder, + _add_numeric_path, ) from tests import mock, unittest @@ -1621,5 +1622,271 @@ def test_str_page_size(self): self.method.assert_called_with(MaxItems='1') +class TestAddNumericPath(unittest.TestCase): + def test_sums_leaf(self): + acc = {'CapacityUnits': 100.0} + _add_numeric_path(acc, {'CapacityUnits': 102.5}, ('CapacityUnits',)) + self.assertEqual(acc, {'CapacityUnits': 202.5}) + + def test_sums_nested_static_path(self): + acc = {'Table': {'CapacityUnits': 1.0}} + _add_numeric_path( + acc, {'Table': {'CapacityUnits': 2.0}}, ('Table', 'CapacityUnits') + ) + self.assertEqual(acc, {'Table': {'CapacityUnits': 3.0}}) + + def test_wildcard_sums_dynamic_key(self): + acc = {'GSI': {'idx': {'CapacityUnits': 5.0}}} + _add_numeric_path( + acc, + {'GSI': {'idx': {'CapacityUnits': 7.0}}}, + ('GSI', '*', 'CapacityUnits'), + ) + self.assertEqual(acc, {'GSI': {'idx': {'CapacityUnits': 12.0}}}) + + def test_wildcard_seeds_new_dynamic_key(self): + # An index first seen on a later page is seeded, not skipped. + acc = {'GSI': {'idx1': {'CapacityUnits': 5.0}}} + _add_numeric_path( + acc, + { + 'GSI': { + 'idx1': {'CapacityUnits': 1.0}, + 'idx2': {'CapacityUnits': 9.0}, + } + }, + ('GSI', '*', 'CapacityUnits'), + ) + self.assertEqual( + acc, + { + 'GSI': { + 'idx1': {'CapacityUnits': 6.0}, + 'idx2': {'CapacityUnits': 9.0}, + } + }, + ) + + def test_does_not_touch_unconfigured_sibling(self): + # The AemousCapacity case: a numeric leaf NOT on a configured path is + # left as the first-page value, even though its leaf name matches an + # aggregated one elsewhere. + acc = { + 'CapacityUnits': 100.0, + 'AemousCapacity': {'CapacityUnits': 123.0}, + } + _add_numeric_path( + acc, + { + 'CapacityUnits': 102.0, + 'AemousCapacity': {'CapacityUnits': 123.0}, + }, + ('CapacityUnits',), + ) + self.assertEqual(acc['CapacityUnits'], 202.0) + self.assertEqual(acc['AemousCapacity'], {'CapacityUnits': 123.0}) + + def test_missing_path_in_page_is_noop(self): + acc = {'CapacityUnits': 100.0} + _add_numeric_path(acc, {}, ('CapacityUnits',)) + self.assertEqual(acc, {'CapacityUnits': 100.0}) + + def test_empty_segments_is_noop(self): + # Defensive: an empty path must not raise (IndexError) and change nothing. + acc = {'CapacityUnits': 100.0} + _add_numeric_path(acc, {'CapacityUnits': 5.0}, ()) + self.assertEqual(acc, {'CapacityUnits': 100.0}) + + def test_type_mismatch_keeps_first(self): + acc = {'CapacityUnits': 'T'} + _add_numeric_path(acc, {'CapacityUnits': 3.0}, ('CapacityUnits',)) + self.assertEqual(acc, {'CapacityUnits': 'T'}) + + def test_does_not_sum_boolean(self): + acc = {'CapacityUnits': True} + _add_numeric_path(acc, {'CapacityUnits': True}, ('CapacityUnits',)) + self.assertIs(acc['CapacityUnits'], True) + + +class TestAggregateNumericKeys(unittest.TestCase): + def setUp(self): + self.method = mock.Mock() + self.model = mock.Mock() + self.paginate_config = { + 'output_token': 'NextToken', + 'input_token': 'NextToken', + 'result_key': 'Items', + 'aggregate_numeric_keys': [ + 'ConsumedCapacity.CapacityUnits', + 'ConsumedCapacity.GlobalSecondaryIndexes.*.CapacityUnits', + ], + } + self.paginator = Paginator( + self.method, self.paginate_config, self.model + ) + + def test_config_parsed(self): + self.assertEqual( + self.paginator._aggregate_numeric_keys, + { + 'ConsumedCapacity': [ + ('CapacityUnits',), + ('GlobalSecondaryIndexes', '*', 'CapacityUnits'), + ] + }, + ) + + def test_aggregated_key_dropped_from_non_aggregate_keys(self): + # A member in both lists is aggregated; the non_aggregate declaration + # for it (and any other non_aggregate members) is handled correctly. + config = { + 'output_token': 'NextToken', + 'input_token': 'NextToken', + 'result_key': 'Items', + 'non_aggregate_keys': ['ConsumedCapacity', 'SomethingElse'], + 'aggregate_numeric_keys': ['ConsumedCapacity.CapacityUnits'], + } + paginator = Paginator(self.method, config, self.model) + kept = [k.expression for k in paginator._non_aggregate_keys] + # ConsumedCapacity is aggregated, so it must not be treated as + # non-aggregate; unrelated members are preserved. + self.assertEqual(kept, ['SomethingElse']) + + def test_aggregated_key_drops_nested_non_aggregate_paths(self): + config = { + 'output_token': 'NextToken', + 'input_token': 'NextToken', + 'result_key': 'Items', + 'non_aggregate_keys': ['ConsumedCapacity.TableName', 'Other'], + 'aggregate_numeric_keys': ['ConsumedCapacity.CapacityUnits'], + } + paginator = Paginator(self.method, config, self.model) + kept = [k.expression for k in paginator._non_aggregate_keys] + self.assertEqual(kept, ['Other']) + + def test_aggregation_wins_when_member_in_both_lists(self): + # Even with ConsumedCapacity declared non-aggregate, the result is the + # cross-page sum, not a single page's value. + config = { + 'output_token': 'NextToken', + 'input_token': 'NextToken', + 'result_key': 'Items', + 'non_aggregate_keys': ['ConsumedCapacity'], + 'aggregate_numeric_keys': ['ConsumedCapacity.CapacityUnits'], + } + paginator = Paginator(self.method, config, self.model) + self.method.side_effect = [ + { + 'Items': ['a'], + 'ConsumedCapacity': {'CapacityUnits': 100.0}, + 'NextToken': 'tok', + }, + {'Items': ['b'], 'ConsumedCapacity': {'CapacityUnits': 102.0}}, + ] + result = paginator.paginate().build_full_result() + self.assertEqual(result['ConsumedCapacity']['CapacityUnits'], 202.0) + + def test_unconfigured_nested_leaf_not_summed_end_to_end(self): + # The exact concern: a NEW nested numeric field (AemousCapacity) whose + # leaf name matches an aggregated one must NOT be summed, because its + # full path is not configured. + self.method.side_effect = [ + { + 'Items': ['a'], + 'ConsumedCapacity': { + 'CapacityUnits': 100.0, + 'AemousCapacity': {'CapacityUnits': 123.0}, + }, + 'NextToken': 'tok', + }, + { + 'Items': ['b'], + 'ConsumedCapacity': { + 'CapacityUnits': 102.0, + 'AemousCapacity': {'CapacityUnits': 123.0}, + }, + }, + ] + cc = self.paginator.paginate().build_full_result()['ConsumedCapacity'] + self.assertEqual( + cc['CapacityUnits'], 202.0 + ) # configured path -> summed + # Not on a configured path -> kept from first page, not doubled. + self.assertEqual(cc['AemousCapacity'], {'CapacityUnits': 123.0}) + + def test_sums_across_pages(self): + self.method.side_effect = [ + { + 'Items': ['a'], + 'ConsumedCapacity': { + 'TableName': 'T', + 'CapacityUnits': 100.0, + 'GlobalSecondaryIndexes': { + 'my-index': {'CapacityUnits': 100.0} + }, + }, + 'NextToken': 'tok', + }, + { + 'Items': ['b'], + 'ConsumedCapacity': { + 'TableName': 'T', + 'CapacityUnits': 102.0, + 'GlobalSecondaryIndexes': { + 'my-index': {'CapacityUnits': 102.0} + }, + }, + }, + ] + result = self.paginator.paginate().build_full_result() + self.assertEqual(result['Items'], ['a', 'b']) + cc = result['ConsumedCapacity'] + self.assertEqual(cc['CapacityUnits'], 202.0) + self.assertEqual(cc['TableName'], 'T') + self.assertEqual( + cc['GlobalSecondaryIndexes']['my-index']['CapacityUnits'], 202.0 + ) + + def test_sums_scalar_member_across_pages(self): + # A top-level scalar (non-dict) aggregate member is summed too. + self.method.side_effect = [ + {'Items': ['a'], 'ConsumedCapacity': 100.0, 'NextToken': 'tok'}, + {'Items': ['b'], 'ConsumedCapacity': 102.0}, + ] + result = self.paginator.paginate().build_full_result() + self.assertEqual(result['ConsumedCapacity'], 202.0) + + def test_does_not_sum_scalar_boolean_member(self): + # A boolean must never be summed; keep the first page's value. + self.method.side_effect = [ + {'Items': ['a'], 'ConsumedCapacity': True, 'NextToken': 'tok'}, + {'Items': ['b'], 'ConsumedCapacity': True}, + ] + result = self.paginator.paginate().build_full_result() + self.assertIs(result['ConsumedCapacity'], True) + + def test_absent_when_never_returned(self): + self.method.side_effect = [ + {'Items': ['a'], 'NextToken': 'tok'}, + {'Items': ['b']}, + ] + result = self.paginator.paginate().build_full_result() + self.assertNotIn('ConsumedCapacity', result) + + def test_does_not_mutate_source_pages(self): + page = { + 'Items': ['a'], + 'ConsumedCapacity': {'CapacityUnits': 100.0}, + 'NextToken': 'tok', + } + self.method.side_effect = [ + page, + {'Items': ['b'], 'ConsumedCapacity': {'CapacityUnits': 2.0}}, + ] + self.paginator.paginate().build_full_result() + # The first page's dict must be left untouched (deepcopy on first sight). + self.assertEqual(page['ConsumedCapacity']['CapacityUnits'], 100.0) + + if __name__ == '__main__': unittest.main()