diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 2570322d19..78f0549170 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -54,11 +54,19 @@ body: id: steps attributes: label: Steps to Reproduce - description: Step-by-step instructions to reproduce the behavior. + description: | + Step-by-step instructions to reproduce the behavior. To qualify for the + `ready-for-dev` label, include how you ran Agent Canvas using one of + these supported methods: + - `npx @openhands/agent-canvas` (or `agent-canvas` if installed globally) + - `npm run dev` / `npm run dev:minimal` (from a source checkout) + - The hosted canvas at `app.all-hands.dev/canvas` placeholder: | - 1. Go to '...' - 2. Click on '...' - 3. Scroll down to '...' + Run method: `npm run dev` + + 1. npm install && npm run dev + 2. Open http://localhost:8000 + 3. … 4. See error validations: required: true @@ -68,22 +76,14 @@ body: attributes: label: Actual Behavior description: | - What actually happened? To qualify for the `ready-for-dev` label, show - how you reproduced the bug in a live Agent Canvas session using one of - the supported run methods, and attach a screenshot or video of the bug - in this section. - Run methods (reference at least one): - - `npx @openhands/agent-canvas` (or `agent-canvas` if installed globally) - - `npm run dev` / `npm run dev:minimal` (from a source checkout) - - The hosted canvas at `app.all-hands.dev/canvas` + What actually happened? To qualify for the `ready-for-dev` label, + describe the observed behavior and attach a screenshot or video of the + bug in this section. Drag a screenshot/video into this field, or paste a link. placeholder: | - Run method: `npm run dev` + The button moved to the next step unexpectedly. - 1. npm install && npm run dev - 2. Open http://localhost:8000 - 3. … - 4. Observed the error shown in the attached screenshot. + The attached screenshot shows the observed error. validations: required: true diff --git a/.github/scripts/check_issue_readiness.py b/.github/scripts/check_issue_readiness.py index 05943d0bb0..dfd1409813 100644 --- a/.github/scripts/check_issue_readiness.py +++ b/.github/scripts/check_issue_readiness.py @@ -2,10 +2,11 @@ The criteria are type-specific: -- Bug reports (labeled `bug`): the Actual Behavior section must reference at +- Bug reports (labeled `bug`): the Steps to Reproduce section must reference at least one supported run method (`agent-canvas`, `npm run`, or - `app.all-hands.dev/canvas`) *and* embed a screenshot or video, plus a - non-empty Acceptance Criteria section with at least one checklist item. + `app.all-hands.dev/canvas`), the Actual Behavior section must embed a + screenshot or video, and there must be a non-empty Acceptance Criteria + section with at least one checklist item. - Enhancements (labeled `enhancement`): the body must contain non-empty Desired Behavior and Acceptance Criteria sections, the latter with at least @@ -153,23 +154,26 @@ def has_checklist_item(text: str) -> bool: def check_bug(sections: dict[str, str]) -> ReadinessResult: result = ReadinessResult(ready=True) - actual = visible_text(find_section(sections, "actual behavior", "actual")) - if not actual: + reproduction = visible_text(find_section(sections, "steps to reproduce", "reproduction")) + if not reproduction: result.add( - "Fill in the `### Actual Behavior` section showing how you reproduced " + "Fill in the `### Steps to Reproduce` section showing how you reproduced " "the bug in a live Agent Canvas session." ) - else: - if not references_run_method(actual): - result.add( - "The Actual Behavior section must reference a supported run method: " - "`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`." - ) - if not has_screenshot_or_video(actual): - result.add( - "The Actual Behavior section must include a screenshot or video of " - "the bug (drag a file into the field or paste a link)." - ) + elif not references_run_method(reproduction): + result.add( + "The Steps to Reproduce section must reference a supported run method: " + "`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`." + ) + + actual = visible_text(find_section(sections, "actual behavior", "actual")) + if not actual: + result.add("Fill in the `### Actual Behavior` section describing the observed bug.") + elif not has_screenshot_or_video(actual): + result.add( + "The Actual Behavior section must include a screenshot or video of " + "the bug (drag a file into the field or paste a link)." + ) acceptance = visible_text(find_section(sections, "acceptance criteria", "acceptance")) if not acceptance: diff --git a/.github/scripts/post-readiness-comment.mjs b/.github/scripts/post-readiness-comment.mjs index 02562d72f6..5fb82ef3e8 100644 --- a/.github/scripts/post-readiness-comment.mjs +++ b/.github/scripts/post-readiness-comment.mjs @@ -147,9 +147,9 @@ async function main() { "The criteria are type-specific:", "", "**Bug reports** (`bug` label):", - "- The `### Actual Behavior` section must reference a supported run method", - " (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`) **and** include", - " a screenshot or video of the bug.", + "- The `### Steps to Reproduce` section must reference a supported run method", + " (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`).", + "- The `### Actual Behavior` section must include a screenshot or video of the bug.", "- An `### Acceptance Criteria` section with at least one checklist item (`- [ ] …`).", "", "**Enhancements** (`enhancement` label):", diff --git a/.github/scripts/tests/test_issue_readiness.py b/.github/scripts/tests/test_issue_readiness.py index c84a5436cc..cc56a9b12d 100644 --- a/.github/scripts/tests/test_issue_readiness.py +++ b/.github/scripts/tests/test_issue_readiness.py @@ -22,59 +22,72 @@ from check_issue_readiness import ( # Helper builders # --------------------------------------------------------------------------- -BUG_BODY_READY = """### Actual Behavior -I ran `npm run dev` and saw this: +BUG_BODY_READY = """### Steps to Reproduce +Run `npm run dev` and click the button. + +### Actual Behavior +The button was misaligned. ![screenshot](https://github.com/user-attachments/assets/abc123) +### Acceptance Criteria +- [ ] Button is centered +""" + +BUG_BODY_NO_RUN_METHOD = """### Steps to Reproduce +Click the button. + +### Actual Behavior The button was misaligned. -### Expected Behavior -The button should be centered. +![screenshot](https://github.com/user-attachments/assets/abc123) -### Acceptance Criteria -- [ ] Button is centered -- [ ] No layout shift on resize """ -BUG_BODY_NO_RUN_METHOD = """### Actual Behavior +BUG_BODY_NO_SCREENSHOT = """### Steps to Reproduce +I ran `npm run dev` and clicked the button. + +### Actual Behavior The button was misaligned. -### Acceptance Criteria -- [ ] Button is centered """ -BUG_BODY_NO_SCREENSHOT = """### Actual Behavior -I ran `npm run dev` and saw the button was misaligned. +BUG_BODY_NO_ACCEPTANCE = """### Steps to Reproduce +I ran `npm run dev` and clicked the button. -### Acceptance Criteria -- [ ] Button is centered -""" - -BUG_BODY_NO_ACCEPTANCE = """### Actual Behavior -I ran `npm run dev` and saw this: +### Actual Behavior +The button was misaligned. ![screenshot](https://github.com/user-attachments/assets/abc123) """ -BUG_BODY_EMPTY_ACTUAL = """### Actual Behavior +BUG_BODY_EMPTY_ACTUAL = """### Steps to Reproduce +I ran `npm run dev` and clicked the button. + +### Actual Behavior _No response_ ### Acceptance Criteria - [ ] Button is centered """ -BUG_BODY_AGENT_CANVAS = """### Actual Behavior +BUG_BODY_AGENT_CANVAS = """### Steps to Reproduce I used agent-canvas to reproduce this. +### Actual Behavior +The button was misaligned. + ![screenshot](https://example.com/screenshot.png) ### Acceptance Criteria - [ ] Fixed """ -BUG_BODY_HOSTED_URL = """### Actual Behavior -Reproduced on app.all-hands.dev/canvas — see video below. +BUG_BODY_HOSTED_URL = """### Steps to Reproduce +Reproduced on app.all-hands.dev/canvas. + +### Actual Behavior +The button was misaligned. @@ -82,6 +95,15 @@ Reproduced on app.all-hands.dev/canvas — see video below. - [ ] Fixed """ +BUG_BODY_MISSING_REPRODUCTION = """### Actual Behavior +The button was misaligned. + +![screenshot](https://github.com/user-attachments/assets/abc123) + +### Acceptance Criteria +- [ ] Button is centered +""" + ENHANCEMENT_BODY_READY = """### Desired Behavior The button should animate on hover. @@ -143,6 +165,12 @@ def test_bug_not_ready_empty_actual(): assert any("Actual Behavior" in r for r in result.reasons) + +def test_bug_not_ready_missing_reproduction(): + result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [BUG_LABEL]) + assert not result.ready + assert any("Steps to Reproduce" in r for r in result.reasons) + # --------------------------------------------------------------------------- # Enhancement readiness # --------------------------------------------------------------------------- @@ -242,7 +270,10 @@ The template says: assert set(sections) == {"notes"} def test_fenced_heading_does_not_truncate_actual_behavior(): - body = """### Actual Behavior + body = """### Steps to Reproduce +Run `npm run dev` and reproduce the error. + +### Actual Behavior I ran `npm run dev` and saw: ~~~text @@ -263,7 +294,10 @@ something went wrong def test_unclosed_fence_does_not_swallow_later_sections(): """One stray marker in a log paste must not reject an otherwise-ready report.""" - body = """### Relevant Logs + body = """### Steps to Reproduce +Run `npm run dev` and reproduce the crash. + +### Relevant Logs ```shell Traceback (most recent call last): the paste was cut off before the closing fence diff --git a/.github/workflows/issue-readiness-check.yml b/.github/workflows/issue-readiness-check.yml index ad66c13d42..28b95b9a8b 100644 --- a/.github/workflows/issue-readiness-check.yml +++ b/.github/workflows/issue-readiness-check.yml @@ -3,10 +3,11 @@ name: Issue Readiness Check # Manages the `ready-for-dev` label based on type-specific readiness criteria. # -# Bug reports (label `bug`): the Actual Behavior section must reference a +# Bug reports (label `bug`): the Steps to Reproduce section must reference a # supported run method (`agent-canvas`, `npm run`, or -# `app.all-hands.dev/canvas`) and embed a screenshot or video, plus an -# Acceptance Criteria section with at least one checklist item. +# `app.all-hands.dev/canvas`), the Actual Behavior section must embed a +# screenshot or video, plus an Acceptance Criteria section with at least one +# checklist item. # # Enhancements (label `enhancement`): the body must contain Desired Behavior and # Acceptance Criteria sections, the latter with at least one checklist item.