diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..c654ddb --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,100 @@ +name: GnuComment + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} + +jobs: + post-comment: + permissions: + actions: read # to list workflow runs artifacts + pull-requests: write # to comment on pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v9 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + // Update existing comment if present, otherwise create a new one + var marker = ''; + var body = marker + '\nGNU diffutils testsuite comparison:\n```\n' + content + '```'; + var comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + }); + var existing = comments.data.filter(c => c.body.includes(marker))[0]; + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: body, + }); + } + } else { + console.log('Comment content too short, skipping'); + } diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml new file mode 100644 index 0000000..6ecf9a1 --- /dev/null +++ b/.github/workflows/GnuTests.yml @@ -0,0 +1,231 @@ +name: GnuTests + +# Run GNU diffutils testsuite against the Rust diffutils implementation +# and compare results against the main branch to catch regressions + +on: + pull_request: + push: + branches: + - '*' + +permissions: + contents: write # Publish diffutils instead of discarding + +# End the current execution if there is a new changeset in the PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json' + +jobs: + native: + name: Run GNU diffutils testsuite + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@v2 + + ### Build + - name: Build Rust diffutils binary + shell: bash + run: | + ## Build Rust diffutils binary + cargo build --config=profile.release.strip=true --profile=release + zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst + + - name: Publish latest commit + uses: softprops/action-gh-release@v3 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + with: + tag_name: latest-commit + body: | + commit: ${{ github.sha }} + draft: false + prerelease: true + files: | + diffutils-x86_64-unknown-linux-gnu.zst + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ### Run tests + - name: Run GNU diffutils testsuite + shell: bash + run: | + ## Run GNU diffutils testsuite + ./tests/run-upstream-testsuite.sh release || true + env: + TERM: xterm + + - name: Upload full json results + uses: actions/upload-artifact@v4 + with: + name: diffutils-gnu-full-result + path: tests/test-results.json + if-no-files-found: warn + + aggregate: + needs: [native] + permissions: + actions: read + contents: read + pull-requests: read + name: Aggregate GNU test results + runs-on: ubuntu-24.04 + steps: + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + TEST_SUMMARY_FILE='diffutils-gnu-result.json' + outputs TEST_SUMMARY_FILE + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v20 + continue-on-error: true + with: + workflow: GnuTests.yml + branch: "${{ env.DEFAULT_BRANCH }}" + workflow_conclusion: completed + path: "reference" + if_no_artifact_found: warn + + - name: Download full json results + uses: actions/download-artifact@v4 + with: + name: diffutils-gnu-full-result + path: results + + - name: Extract/summarize testing info + id: summary + shell: bash + run: | + ## Extract/summarize testing info + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + RESULT_FILE="results/test-results.json" + if [[ ! -f "$RESULT_FILE" ]]; then + echo "::error ::Missing test results at $RESULT_FILE" + exit 1 + fi + + TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE") + PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE") + FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE") + SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE") + ERROR=0 + + output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" + echo "${output}" + + if [[ "$FAIL" -gt 0 ]]; then + echo "::warning ::${output}" + fi + + jq -n \ + --arg date "$(date --rfc-email)" \ + --arg sha "$GITHUB_SHA" \ + --arg total "$TOTAL" \ + --arg pass "$PASS" \ + --arg skip "$SKIP" \ + --arg fail "$FAIL" \ + --arg error "$ERROR" \ + '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' + + HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) + outputs HASH TOTAL PASS FAIL SKIP + + - name: Upload SHA1/ID of 'test-summary' + uses: actions/upload-artifact@v4 + with: + name: "${{ steps.summary.outputs.HASH }}" + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Upload test results summary + uses: actions/upload-artifact@v4 + with: + name: test-summary + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare test failures VS reference + REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json' + CURRENT_SUMMARY_FILE="results/test-results.json" + + IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt" + + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" + + python3 util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" + + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." + fi + else + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "::error ::Found new non-intermittent test failures" + exit 1 + else + echo "::notice ::No new test failures detected" + fi + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: comment + path: reference/comment/ + + - name: Report test results + if: success() || failure() + shell: bash + run: | + ## Report final results + echo "::notice ::GNU diffutils testsuite results:" + echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}" + echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}" + echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}" + echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}" + + if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then + PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) )) + echo "::notice :: Pass rate: ${PASS_RATE}%" + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59a9103..c72ae7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,37 +56,6 @@ jobs: - uses: actions/checkout@v4 - run: cargo clippy -- -D warnings - gnu-testsuite: - permissions: - contents: write # Publish diffutils instead of discarding - name: GNU test suite - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: | - cargo build --config=profile.release.strip=true --profile=release #-fast - zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst - # do not fail, the report is merely informative (at least until all tests pass reliably) - - run: ./tests/run-upstream-testsuite.sh release || true - env: - TERM: xterm - - uses: actions/upload-artifact@v4 - with: - name: test-results.json - path: tests/test-results.json - - run: ./tests/print-test-results.sh tests/test-results.json - - name: Publish latest commit - uses: softprops/action-gh-release@v3 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - tag_name: latest-commit - draft: false - prerelease: true - files: | - diffutils-x86_64-unknown-linux-gnu.zst - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - coverage: name: Code Coverage runs-on: ${{ matrix.job.os }} diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100644 index 0000000..0625782 --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from a diffutils test-results.json. + + Note: unlike sed, diffutils JSON has no 'summary' object — results are + computed from the 'tests' array using the 'result' and 'test' fields. + """ + tests = json_data.get("tests", []) + passed = sum(1 for t in tests if t.get("result") == "PASS") + failed = sum(1 for t in tests if t.get("result") == "FAIL") + skipped = sum(1 for t in tests if t.get("result") == "SKIP") + summary = {"total": len(tests), "passed": passed, "failed": failed, "skipped": skipped} + failed_tests = [t["test"] for t in tests if t.get("result") == "FAIL"] + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + ignore_set = load_ignore_list(ignore_file) + + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int(reference_summary.get("passed", 0)) + fail_diff = int(current_summary.get("failed", 0)) - int(reference_summary.get("failed", 0)) + total_diff = int(current_summary.get("total", 0)) - int(reference_summary.get("total", 0)) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Check if results are identical (no changes) + no_changes = ( + pass_diff == 0 + and fail_diff == 0 + and total_diff == 0 + and not new_failures + and not improvements + ) + + # If no changes, write empty output to prevent comment posting + if no_changes: + if output_file: + with open(output_file, "w") as f: + f.write("") + return 0 + + # Prepare output message + output_lines = [] + + output_lines.append("Test results comparison:") + output_lines.append( + f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}" + ) + output_lines.append( + f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}" + ) + output_lines.append("") + + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU diffutils test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main())