feat(perf,benchmark-infra): criterion bench harness + advisory CI workflows (SDD-120)
Implementation of SDD-120 §1-§8: Bench harness (crates/chanora_audio/benches/): - common.rs: deterministic synthetic audio (440 Hz sine, no RNG). - realtime_capture.rs: bench_capture_alloc_count (dhat) + bench_capture_callback_wall_clock (criterion). - opus_codec.rs: bench_opus_encode_latency + bench_opus_decode_latency (direct audiopus, not AudioHandler — SDD-120 §3 item 4). - resampler.rs: bench_resampler_throughput across 44.1->48 / 16->48 / 48->48 passthrough. CI tooling (crates/chanora_audio/examples/): - emit_baseline.rs: aggregates criterion estimates.json outputs into the SRS-217 baseline schema. - compare_baseline.rs: applies SRS-219 tolerance, renders markdown table with 🟢/🟡/🔴 markers + yellow simpler-form realization per SDD-120 §8. Deviation from SDD-120 §2 / §5 / §7 placement: these tools live under examples/, not benches/ or src/bin/. Rationale: they must consume serde_json (a dev-only dep — production builds must not pull it). Cargo only resolves dev-dependencies for [[test]], [[bench]], and [[example]] targets; [[bin]] targets under src/bin/ see only regular [dependencies]. examples/ keeps the binaries out of the production dep tree while still giving them cargo run --example invocation. An SDD-120 amendment should reflect this. Workflows (.github/workflows/): - bench-advisory.yml: PR + push triggers; runs benches; posts a sticky PR comment via actions/github-script@v7; job status is always success (SRS-218 clause 4 — non-blocking). - bench-baseline-update.yml: workflow_dispatch only; runs benches; opens PR via peter-evans/create-pull-request@v6 (sole writer of the SAD-089 baseline JSON). Cargo.toml additions ([dev-dependencies] only — verified excluded from --release builds): criterion 0.5, dhat 0.3, serde_json 1. Source-code seam: minimal pub-but-#[doc(hidden)] bench_seam module in chanora_audio (engine.rs + lib.rs re-export) so the criterion bench harness can construct a CaptureState and drive CaptureState::ingest without re-implementing the engine (SDD-120 §3). Non-iOS targets only — CaptureState itself is iOS-gated. Initial baseline seed: crates/chanora_audio/benches/baselines/ x86_64-unknown-linux-gnu.json = {}. compare_baseline handles the missing-baseline case gracefully and emits a 'no red markers' report; the first manual dispatch of bench-baseline-update.yml after merge establishes the real values. Out of scope per SDD-120 §10: production telemetry export, build-failing hard CI gate, multi-host benchmarking, IDE integration, Dart-side bridge round-trip bench. Verification: - cargo check --workspace --all-targets: PASS. - cargo bench --bench realtime_capture --no-run: PASS. - cargo bench --bench opus_codec --no-run: PASS. - cargo bench --bench resampler --no-run: PASS. - cargo build --example emit_baseline --example compare_baseline -p chanora_audio: PASS. - cargo test --workspace: 106 passed, 0 failed, 3 ignored — no regression from prior count.
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
name: bench-advisory
|
||||
|
||||
# SDD-120 §6 / SRS-218 — advisory-only realtime-audio bench workflow.
|
||||
# Runs the criterion bench harness on PR + push events, compares the
|
||||
# current results against the SAD-089 baseline JSON resolved at the
|
||||
# merge-base, and renders a markdown report posted (or updated) as a
|
||||
# single sticky PR comment. The job status is ALWAYS success — this
|
||||
# workflow never fails a check on regression (SRS-218 clause 4).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [product/scaffold-v0]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
bench-advisory:
|
||||
name: bench-advisory
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: System deps (cpal / Opus)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
libasound2-dev libpulse-dev pkg-config \
|
||||
libopus-dev
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run benchmarks
|
||||
run: |
|
||||
cargo bench -p chanora_audio \
|
||||
--bench realtime_capture \
|
||||
--bench opus_codec \
|
||||
--bench resampler
|
||||
- name: Emit current baseline JSON
|
||||
run: cargo run --example emit_baseline -p chanora_audio
|
||||
- name: Resolve merge-base baseline
|
||||
run: |
|
||||
DEFAULT_BRANCH="${{ github.event.repository.default_branch }}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
BASE_SHA="${{ github.event.pull_request.base.sha }}"
|
||||
MERGE_BASE=$(git merge-base "$BASE_SHA" HEAD || true)
|
||||
else
|
||||
MERGE_BASE=$(git merge-base "origin/${DEFAULT_BRANCH}" HEAD || true)
|
||||
fi
|
||||
if [ -n "$MERGE_BASE" ] && git show "${MERGE_BASE}:crates/chanora_audio/benches/baselines/x86_64-unknown-linux-gnu.json" > baseline.json 2>/dev/null; then
|
||||
echo "BASELINE_FOUND=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "BASELINE_FOUND=0" >> "$GITHUB_ENV"
|
||||
echo "{}" > baseline.json
|
||||
fi
|
||||
- name: Compare against baseline
|
||||
run: |
|
||||
cargo run --example compare_baseline -p chanora_audio -- \
|
||||
--current current.json \
|
||||
--baseline baseline.json \
|
||||
--output report.md
|
||||
- name: Post PR comment
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const body = fs.readFileSync('report.md', 'utf8');
|
||||
const marker = '<!-- chanora-bench-advisory -->';
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
const existing = comments.find(c => c.body && c.body.startsWith(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
- name: Upload report artifact (push events)
|
||||
if: github.event_name == 'push'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bench-report
|
||||
path: report.md
|
||||
@@ -0,0 +1,53 @@
|
||||
name: bench-baseline-update
|
||||
|
||||
# SDD-120 §7 / SAD-089 — manual-dispatch workflow that opens a PR
|
||||
# updating the committed baseline JSON. This is the SOLE writer of
|
||||
# `crates/chanora_audio/benches/baselines/<host>.json`. Triggered
|
||||
# only on operator demand via workflow_dispatch.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
bench-baseline-update:
|
||||
name: bench-baseline-update
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: System deps (cpal / Opus)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
libasound2-dev libpulse-dev pkg-config \
|
||||
libopus-dev
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run benchmarks
|
||||
run: |
|
||||
cargo bench -p chanora_audio \
|
||||
--bench realtime_capture \
|
||||
--bench opus_codec \
|
||||
--bench resampler
|
||||
- name: Emit current baseline JSON
|
||||
run: cargo run --example emit_baseline -p chanora_audio
|
||||
- name: Copy current to baseline path
|
||||
run: cp current.json crates/chanora_audio/benches/baselines/x86_64-unknown-linux-gnu.json
|
||||
- name: Create pull request
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
branch: bench/baseline-update-${{ github.run_id }}
|
||||
base: product/scaffold-v0
|
||||
title: "chore(bench): update realtime audio baselines"
|
||||
body: |
|
||||
Automated baseline update produced by manual dispatch of
|
||||
`bench-baseline-update.yml`.
|
||||
|
||||
Run: ${{ github.run_id }}
|
||||
Triggered by: ${{ github.actor }}
|
||||
commit-message: "chore(bench): update realtime audio baselines"
|
||||
add-paths: |
|
||||
crates/chanora_audio/benches/baselines/x86_64-unknown-linux-gnu.json
|
||||
Reference in New Issue
Block a user