Test librarian with new wrkflw

.github/scripts/matrix_generator.py

@@ -0,0 +1,60 @@
+import os
+import json
+import argparse
+
+VIPS = {
+    "google-cloud-spanner", 
+    "google-cloud-compute",
+    "google-cloud-compute-v1beta", 
+    "google-cloud-discoveryengine"
+}
+
+def get_valid_packages(directories):
+    return [p for p in directories if os.path.isfile(os.path.join(p, "noxfile.py"))]
+
+def distribute_packages(packages, max_buckets):
+    # VIPs jump to the front of the line
+    packages.sort(key=lambda p: os.path.basename(p) not in VIPS)
+
+    buckets = [{"weight": 0, "pkgs": []} for _ in range(min(len(packages), max_buckets))]
+
+    for pkg in packages:
+        # Find the bucket with the lowest weight, add the package, and update its weight
+        lightest = min(buckets, key=lambda b: b["weight"])
+        lightest["pkgs"].append(pkg)
+        lightest["weight"] += 9999 if os.path.basename(pkg) in VIPS else 1
+
+    return [b["pkgs"] for b in buckets]
+
+def build_github_actions_jobs(buckets):
+    jobs = []
+    for bucket in buckets:
+        # Create a clean UI label for the GitHub presubmit check
+        base_name = os.path.basename(bucket[0]).replace("google-cloud-", "")
+        job_label = f"{base_name} + {len(bucket) - 1}" if len(bucket) > 1 else base_name
+        jobs.append({"id": job_label, "packages": " ".join(bucket)})
+    return jobs
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--matrix-multiplier", type=int, required=True)
+    parser.add_argument("--max-vms", type=int, default=20)
+    args = parser.parse_args()
+
+    packages = get_valid_packages(os.environ.get("CHANGED_DIRS", "").split())
+    if not packages:
+        return
+
+    max_buckets = min(250 // args.matrix_multiplier, args.max_vms)
+    buckets = distribute_packages(packages, max_buckets)
+
+    jobs_json = json.dumps(build_github_actions_jobs(buckets))
+
+    if "GITHUB_OUTPUT" in os.environ:
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+            f.write(f"buckets={jobs_json}\n")
+    else:
+        print(jobs_json)
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci-docs.yml

@@ -0,0 +1,148 @@
+name: CI Docs
+on:
+  pull_request:
+    branches: [ main, preview ]
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # ==========================================
+  # 1. DISCOVERY & BUCKETING
+  # ==========================================
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      buckets: ${{ steps.generate-matrix.outputs.buckets }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Detect Changed Packages
+        id: changes
+        uses: tj-actions/changed-files@v44
+        with:
+          files: packages/**
+          dir_names: true               
+          dir_names_max_depth: 2        
+          matrix: false
+
+      - name: Generate Balanced Buckets
+        id: generate-matrix
+        env:
+          CHANGED_DIRS: ${{ steps.changes.outputs.all_changed_files }}
+        run: python .github/scripts/matrix_generator.py --matrix-multiplier 2 --max-vms 10
+
+  # ==========================================
+  # 2. DOCS (Py 3.10)
+  # ==========================================
+  docs:
+    needs: discover
+    if: ${{ needs.discover.outputs.buckets != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 60
+      matrix:
+        chunk: ${{ fromJSON(needs.discover.outputs.buckets) }}
+
+    name: docs (Py 3.10 - ${{ matrix.chunk.id }})
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.10"
+          enable-cache: true
+
+      - name: Execute Chunk
+        run: |
+          export NOX_DEFAULT_VENV_BACKEND=uv
+          export UV_VENV_SEED=1 
+          export UV_COMPILE_BYTECODE=1
+          FAILED=0
+
+          for pkg in ${{ matrix.chunk.packages }}; do
+            echo "=========================================================="
+            echo "🚀 TESTING DOCS: $pkg"
+            echo "=========================================================="
+            cd "$pkg"
+
+            if uvx --with 'nox[uv]' nox -l | grep -q "\bdocs\b"; then
+              uvx --with 'nox[uv]' nox -s "docs" || FAILED=1
+            else
+              echo "⏭️ Session 'docs' not defined for $pkg. Safely skipping."
+            fi
+
+            cd "$GITHUB_WORKSPACE"
+          done
+          exit $FAILED
+
+  # ==========================================
+  # 3. DOCFX (Py 3.10)
+  # ==========================================
+  docfx:
+    needs: discover
+    if: ${{ needs.discover.outputs.buckets != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 60
+      matrix:
+        chunk: ${{ fromJSON(needs.discover.outputs.buckets) }}
+
+    name: docfx (Py 3.10 - ${{ matrix.chunk.id }})
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.10"
+          enable-cache: true
+
+      - name: Execute Chunk
+        run: |
+          export NOX_DEFAULT_VENV_BACKEND=uv
+          export UV_VENV_SEED=1 
+          export UV_COMPILE_BYTECODE=1
+          FAILED=0
+
+          for pkg in ${{ matrix.chunk.packages }}; do
+            echo "=========================================================="
+            echo "🚀 TESTING DOCFX: $pkg"
+            echo "=========================================================="
+            cd "$pkg"
+
+            if uvx --with 'nox[uv]' nox -l | grep -q "\bdocfx\b"; then
+              uvx --with 'nox[uv]' nox -s "docfx" || FAILED=1
+            else
+              echo "⏭️ Session 'docfx' not defined for $pkg. Safely skipping."
+            fi
+
+            cd "$GITHUB_WORKSPACE"
+          done
+          exit $FAILED
+
+  # ==========================================
+  # 4. GATEKEEPER
+  # ==========================================
+  presubmit-passed:
+    if: always() 
+    needs: [discover, docs, docfx]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Evaluate Pipeline Status
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "::error::One or more required CI jobs failed or were cancelled."
+            exit 1
+          fi
+          echo "All dynamically generated CI jobs completed successfully."

.github/workflows/ci-lint.yml

@@ -0,0 +1,150 @@
+name: CI Lint
+on:
+  pull_request:
+    branches: [ main, preview ]
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # ==========================================
+  # 1. DISCOVERY & BUCKETING
+  # ==========================================
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      buckets: ${{ steps.generate-matrix.outputs.buckets }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Detect Changed Packages
+        id: changes
+        uses: tj-actions/changed-files@v44
+        with:
+          files: packages/**
+          dir_names: true               
+          dir_names_max_depth: 2        
+          matrix: false
+
+      - name: Generate Balanced Buckets
+        id: generate-matrix
+        env:
+          CHANGED_DIRS: ${{ steps.changes.outputs.all_changed_files }}
+        run: python .github/scripts/matrix_generator.py --matrix-multiplier 2 --max-vms 10
+
+  # ==========================================
+  # 2. LINT (Py 3.14)
+  # ==========================================
+  lint:
+    needs: discover
+    if: ${{ needs.discover.outputs.buckets != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 60
+      matrix:
+        chunk: ${{ fromJSON(needs.discover.outputs.buckets) }}
+
+    name: lint (Py 3.14 - ${{ matrix.chunk.id }})
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.14"
+          enable-cache: true
+
+      - name: Execute Chunk
+        run: |
+          export NOX_DEFAULT_VENV_BACKEND=uv
+          export UV_VENV_SEED=1 
+          export UV_COMPILE_BYTECODE=1
+          FAILED=0
+
+          for pkg in ${{ matrix.chunk.packages }}; do
+            echo "=========================================================="
+            echo "🚀 LINTING: $pkg"
+            echo "=========================================================="
+            cd "$pkg"
+
+            if uvx --with 'nox[uv]' nox -l | grep -q "\blint\b"; then
+              uvx --with 'nox[uv]' nox -s "lint" || FAILED=1
+            fi
+
+            if uvx --with 'nox[uv]' nox -l | grep -q "\blint_setup_py\b"; then
+              uvx --with 'nox[uv]' nox -s "lint_setup_py" || FAILED=1
+            fi
+
+            cd "$GITHUB_WORKSPACE"
+          done
+          exit $FAILED
+
+  # ==========================================
+  # 3. MYPY (Py 3.14)
+  # ==========================================
+  mypy:
+    needs: discover
+    if: ${{ needs.discover.outputs.buckets != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 60
+      matrix:
+        chunk: ${{ fromJSON(needs.discover.outputs.buckets) }}
+
+    name: mypy (Py 3.14 - ${{ matrix.chunk.id }})
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.14"
+          enable-cache: true
+
+      - name: Execute Chunk
+        run: |
+          export NOX_DEFAULT_VENV_BACKEND=uv
+          export UV_VENV_SEED=1 
+          export UV_COMPILE_BYTECODE=1
+          FAILED=0
+
+          for pkg in ${{ matrix.chunk.packages }}; do
+            echo "=========================================================="
+            echo "🚀 TESTING MYPY: $pkg"
+            echo "=========================================================="
+            cd "$pkg"
+
+            if uvx --with 'nox[uv]' nox -l | grep -q "\bmypy\b"; then
+              uvx --with 'nox[uv]' nox -s "mypy" || FAILED=1
+            else
+              echo "⏭️ Session 'mypy' not defined for $pkg. Safely skipping."
+            fi
+
+            cd "$GITHUB_WORKSPACE"
+          done
+          exit $FAILED
+
+  # ==========================================
+  # 4. GATEKEEPER
+  # ==========================================
+  presubmit-passed:
+    if: always() 
+    needs: [discover, lint, mypy]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Evaluate Pipeline Status
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "::error::One or more required CI jobs failed or were cancelled."
+            exit 1
+          fi
+          echo "All dynamically generated CI jobs completed successfully."