chore(test): improve diff detection for unit tests

.github/workflows/lint.yml

@@ -57,7 +57,7 @@ jobs:
         TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
       run: |
         if [ -n "$TARGET_BRANCH" ]; then
-          git fetch origin "$TARGET_BRANCH" --depth=1 || true
+          git fetch origin "$TARGET_BRANCH" --deepen=200 || true
         fi
         python3 ci/get_package_shards.py
 

.github/workflows/unittest.yml

@@ -27,6 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      packages: ${{ steps.set-matrix.outputs.packages }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}
     env:
       MAX_SHARDS: 16
@@ -140,7 +141,7 @@ jobs:
           echo "All unit tests passed or were skipped"
 
   cover:
-    if: always() && !cancelled() && needs.all-tests.result == 'success' && needs.unit.result != 'skipped'
+    if: always() && !cancelled() && needs.all-tests.result == 'success'
     runs-on: ubuntu-latest
     needs:
         - all-tests
@@ -158,42 +159,24 @@ jobs:
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
       with:
         python-version: "3.10"
-    - name: Determine if coverage evaluation is required
-      id: packages
-      env:
-        TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
-        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
-      run: |
-        if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
-          echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT"
-        else
-          TARGET_BRANCH="${TARGET_BRANCH:-main}"
-          git fetch origin "$TARGET_BRANCH" --depth=1 || true
-          num_files_changed=$(git diff --name-only "origin/${TARGET_BRANCH}" -- ${PACKAGE_DIRS} | wc -l | tr -d ' ')
-          if [[ "${num_files_changed}" -gt 0 ]]; then
-            echo "should_evaluate_coverage=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "should_evaluate_coverage=false" >> "$GITHUB_OUTPUT"
-          fi
-        fi
     - name: Install coverage
-      if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }}
+      if: ${{ needs.initialize.outputs.packages != '' }}
       run: |
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install coverage
     - name: Download coverage results
-      if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }}
+      if: ${{ needs.initialize.outputs.packages != '' }}
       uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
       with:
         path: /dev/shm/.coverage-results/
         merge-multiple: true
     - name: Report coverage results
-      if: ${{ steps.packages.outputs.should_evaluate_coverage == 'true' }}
       env:
         # TODO: default to 100% coverage after next gapic-generator release
         # https://github.com/googleapis/google-cloud-python/issues/17459
         DEFAULT_FAIL_UNDER: 99
         TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
+        PACKAGE_LIST: ${{ needs.initialize.outputs.packages }}
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         BUILD_TYPE: presubmit
       run: |

ci/get_package_shards.py

@@ -108,7 +108,7 @@ def get_packages_to_test():
         return all_packages
 
     if build_type == 'presubmit':
-        git_diff_arg = f"origin/{target_branch}"
+        git_diff_arg = f"origin/{target_branch}..."
     elif build_type == 'continuous':
         git_diff_arg = "HEAD~1.."
     else:
@@ -227,7 +227,13 @@ def group_packages(packages_map):
     shards_json = json.dumps(shards)
     print(shards_json)
 
+    all_paths = []
+    for paths in packages.values():
+        all_paths.extend(paths)
+    packages_str = " ".join(all_paths)
+
     github_output = os.environ.get("GITHUB_OUTPUT")
     if github_output:
         with open(github_output, "a") as f:
             f.write(f"matrix={shards_json}\n")
+            f.write(f"packages={packages_str}\n")

ci/report_coverage.sh

@@ -26,31 +26,39 @@ MAX_JOBS=$(nproc)
 
 mkdir -p "${LOG_DIR}"
 
-if [ ! -d "${RESULTS_DIR}" ]; then
-    echo "Error: No coverage results found in ${RESULTS_DIR}."
-    exit 1
-fi
-
-# Unzip any zipped coverage results
-find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR"
-
-# Identify modified packages
 BUILD_TYPE="${BUILD_TYPE:-presubmit}"
 TARGET_BRANCH="${TARGET_BRANCH:-main}"
-
 PACKAGE_DIRS="${PACKAGE_DIRS:-packages preview-packages}"
 
-if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+if [ "${PACKAGE_LIST+set}" = "set" ]; then
+    # If pre-determined package list is set, use it
+    modified_packages="${PACKAGE_LIST}"
+elif [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
     # Test all packages mode: evaluate coverage for every package in the repository
     modified_packages=$(for dir in ${PACKAGE_DIRS}; do ls -d ${dir}/*/ 2>/dev/null; done | cut -d/ -f1,2 | sort -u)
 elif [[ "${BUILD_TYPE}" == "presubmit" ]]; then
     # Presubmit build: evaluate coverage only for packages modified relative to the target branch
-    modified_packages=$(git diff --name-only "origin/${TARGET_BRANCH}" -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u)
+    modified_packages=$(git diff --name-only "origin/${TARGET_BRANCH}..." -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u)
 else
     # Continuous build (post-merge on main): evaluate coverage for packages modified in the last commit
     modified_packages=$(git diff --name-only HEAD~1 -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u)
 fi
 
+if [ -z "${modified_packages}" ]; then
+    echo "============================================================"
+    echo "No modified packages to evaluate coverage for."
+    echo "============================================================"
+    exit 0
+fi
+
+if [ ! -d "${RESULTS_DIR}" ]; then
+    echo "Error: No coverage results found in ${RESULTS_DIR}."
+    exit 1
+fi
+
+# Unzip any zipped coverage results
+find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR"
+
 # Function to report coverage for a single package
 report_package_coverage() {
     local pkg=$1

ci/run_conditional_tests.sh

@@ -54,9 +54,9 @@ elif [[ ${BUILD_TYPE} == "presubmit" ]]; then
     # For presubmit build, we want to know the difference from the
     # common commit in the target branch.
     if [ -n "${TARGET_BRANCH}" ]; then
-        git fetch origin "${TARGET_BRANCH}" --depth=1 || true
+        git fetch origin "${TARGET_BRANCH}" --depth=200 || true
     fi
-    GIT_DIFF_ARG="origin/${TARGET_BRANCH}"
+    GIT_DIFF_ARG="origin/${TARGET_BRANCH}..."
 
 elif [[ ${BUILD_TYPE} == "continuous" ]]; then
     # For continuous build, we want to know the difference in the last

packages/google-apps-script-type/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-appoptimize/README.rst

@@ -198,3 +198,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-automl/README.rst

@@ -197,3 +197,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-chronicle/README.rst

@@ -198,3 +198,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-databasecenter/README.rst

@@ -202,3 +202,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-discoveryengine/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-financialservices/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-iam/README.rst

@@ -201,3 +201,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-logging/tmp/packages/google-cloud-logging

@@ -0,0 +1 @@
+../..

packages/google-cloud-managedkafka/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-monitoring-metrics-scopes/README.rst

@@ -201,3 +201,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-optimization/README.rst

@@ -198,3 +198,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-retail/README.rst

@@ -199,3 +199,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-cloud-videointelligence/README.rst

@@ -198,3 +198,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-shopping-merchant-datasources/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment

packages/google-shopping-merchant-lfp/README.rst

@@ -196,3 +196,5 @@ Logging details
 #. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get
    executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured.
    (This is the reason for 2.i. above.)
+
+.. test comment