mirror of
https://github.com/OpenHands/OpenHands.git
synced 2026-10-07 16:08:23 +08:00
fix(ci): use reproduction steps for ready-for-dev checks (#16828)
Co-authored-by: Harsha Vardhan <harshahvk2005@gmail.com> Co-authored-by: neubig <neubig@users.noreply.github.com> Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
Graham Neubig
co-authored by
Harsha Vardhan
neubig
openhands
Harsha Vardhan
neubig
openhands
parent
e25feb6468
commit
647b493170
@@ -54,11 +54,19 @@ body:
|
||||
id: steps
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: Step-by-step instructions to reproduce the behavior.
|
||||
description: |
|
||||
Step-by-step instructions to reproduce the behavior. To qualify for the
|
||||
`ready-for-dev` label, include how you ran Agent Canvas using one of
|
||||
these supported methods:
|
||||
- `npx @openhands/agent-canvas` (or `agent-canvas` if installed globally)
|
||||
- `npm run dev` / `npm run dev:minimal` (from a source checkout)
|
||||
- The hosted canvas at `app.all-hands.dev/canvas`
|
||||
placeholder: |
|
||||
1. Go to '...'
|
||||
2. Click on '...'
|
||||
3. Scroll down to '...'
|
||||
Run method: `npm run dev`
|
||||
|
||||
1. npm install && npm run dev
|
||||
2. Open http://localhost:8000
|
||||
3. …
|
||||
4. See error
|
||||
validations:
|
||||
required: true
|
||||
@@ -68,22 +76,14 @@ body:
|
||||
attributes:
|
||||
label: Actual Behavior
|
||||
description: |
|
||||
What actually happened? To qualify for the `ready-for-dev` label, show
|
||||
how you reproduced the bug in a live Agent Canvas session using one of
|
||||
the supported run methods, and attach a screenshot or video of the bug
|
||||
in this section.
|
||||
Run methods (reference at least one):
|
||||
- `npx @openhands/agent-canvas` (or `agent-canvas` if installed globally)
|
||||
- `npm run dev` / `npm run dev:minimal` (from a source checkout)
|
||||
- The hosted canvas at `app.all-hands.dev/canvas`
|
||||
What actually happened? To qualify for the `ready-for-dev` label,
|
||||
describe the observed behavior and attach a screenshot or video of the
|
||||
bug in this section.
|
||||
Drag a screenshot/video into this field, or paste a link.
|
||||
placeholder: |
|
||||
Run method: `npm run dev`
|
||||
The button moved to the next step unexpectedly.
|
||||
|
||||
1. npm install && npm run dev
|
||||
2. Open http://localhost:8000
|
||||
3. …
|
||||
4. Observed the error shown in the attached screenshot.
|
||||
The attached screenshot shows the observed error.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -2,10 +2,11 @@
|
||||
|
||||
The criteria are type-specific:
|
||||
|
||||
- Bug reports (labeled `bug`): the Actual Behavior section must reference at
|
||||
- Bug reports (labeled `bug`): the Steps to Reproduce section must reference at
|
||||
least one supported run method (`agent-canvas`, `npm run`, or
|
||||
`app.all-hands.dev/canvas`) *and* embed a screenshot or video, plus a
|
||||
non-empty Acceptance Criteria section with at least one checklist item.
|
||||
`app.all-hands.dev/canvas`), the Actual Behavior section must embed a
|
||||
screenshot or video, and there must be a non-empty Acceptance Criteria
|
||||
section with at least one checklist item.
|
||||
|
||||
- Enhancements (labeled `enhancement`): the body must contain non-empty
|
||||
Desired Behavior and Acceptance Criteria sections, the latter with at least
|
||||
@@ -153,23 +154,26 @@ def has_checklist_item(text: str) -> bool:
|
||||
def check_bug(sections: dict[str, str]) -> ReadinessResult:
|
||||
result = ReadinessResult(ready=True)
|
||||
|
||||
actual = visible_text(find_section(sections, "actual behavior", "actual"))
|
||||
if not actual:
|
||||
reproduction = visible_text(find_section(sections, "steps to reproduce", "reproduction"))
|
||||
if not reproduction:
|
||||
result.add(
|
||||
"Fill in the `### Actual Behavior` section showing how you reproduced "
|
||||
"Fill in the `### Steps to Reproduce` section showing how you reproduced "
|
||||
"the bug in a live Agent Canvas session."
|
||||
)
|
||||
else:
|
||||
if not references_run_method(actual):
|
||||
result.add(
|
||||
"The Actual Behavior section must reference a supported run method: "
|
||||
"`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`."
|
||||
)
|
||||
if not has_screenshot_or_video(actual):
|
||||
result.add(
|
||||
"The Actual Behavior section must include a screenshot or video of "
|
||||
"the bug (drag a file into the field or paste a link)."
|
||||
)
|
||||
elif not references_run_method(reproduction):
|
||||
result.add(
|
||||
"The Steps to Reproduce section must reference a supported run method: "
|
||||
"`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`."
|
||||
)
|
||||
|
||||
actual = visible_text(find_section(sections, "actual behavior", "actual"))
|
||||
if not actual:
|
||||
result.add("Fill in the `### Actual Behavior` section describing the observed bug.")
|
||||
elif not has_screenshot_or_video(actual):
|
||||
result.add(
|
||||
"The Actual Behavior section must include a screenshot or video of "
|
||||
"the bug (drag a file into the field or paste a link)."
|
||||
)
|
||||
|
||||
acceptance = visible_text(find_section(sections, "acceptance criteria", "acceptance"))
|
||||
if not acceptance:
|
||||
|
||||
@@ -147,9 +147,9 @@ async function main() {
|
||||
"The criteria are type-specific:",
|
||||
"",
|
||||
"**Bug reports** (`bug` label):",
|
||||
"- The `### Actual Behavior` section must reference a supported run method",
|
||||
" (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`) **and** include",
|
||||
" a screenshot or video of the bug.",
|
||||
"- The `### Steps to Reproduce` section must reference a supported run method",
|
||||
" (`agent-canvas`, `npm run`, or `app.all-hands.dev/canvas`).",
|
||||
"- The `### Actual Behavior` section must include a screenshot or video of the bug.",
|
||||
"- An `### Acceptance Criteria` section with at least one checklist item (`- [ ] …`).",
|
||||
"",
|
||||
"**Enhancements** (`enhancement` label):",
|
||||
|
||||
@@ -22,59 +22,72 @@ from check_issue_readiness import (
|
||||
# Helper builders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BUG_BODY_READY = """### Actual Behavior
|
||||
I ran `npm run dev` and saw this:
|
||||
BUG_BODY_READY = """### Steps to Reproduce
|
||||
Run `npm run dev` and click the button.
|
||||
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
"""
|
||||
|
||||
BUG_BODY_NO_RUN_METHOD = """### Steps to Reproduce
|
||||
Click the button.
|
||||
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
### Expected Behavior
|
||||
The button should be centered.
|
||||
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
- [ ] No layout shift on resize
|
||||
"""
|
||||
|
||||
BUG_BODY_NO_RUN_METHOD = """### Actual Behavior
|
||||
BUG_BODY_NO_SCREENSHOT = """### Steps to Reproduce
|
||||
I ran `npm run dev` and clicked the button.
|
||||
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
"""
|
||||
|
||||
BUG_BODY_NO_SCREENSHOT = """### Actual Behavior
|
||||
I ran `npm run dev` and saw the button was misaligned.
|
||||
BUG_BODY_NO_ACCEPTANCE = """### Steps to Reproduce
|
||||
I ran `npm run dev` and clicked the button.
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
"""
|
||||
|
||||
BUG_BODY_NO_ACCEPTANCE = """### Actual Behavior
|
||||
I ran `npm run dev` and saw this:
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
|
||||
"""
|
||||
|
||||
BUG_BODY_EMPTY_ACTUAL = """### Actual Behavior
|
||||
BUG_BODY_EMPTY_ACTUAL = """### Steps to Reproduce
|
||||
I ran `npm run dev` and clicked the button.
|
||||
|
||||
### Actual Behavior
|
||||
_No response_
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
"""
|
||||
|
||||
BUG_BODY_AGENT_CANVAS = """### Actual Behavior
|
||||
BUG_BODY_AGENT_CANVAS = """### Steps to Reproduce
|
||||
I used agent-canvas to reproduce this.
|
||||
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Fixed
|
||||
"""
|
||||
|
||||
BUG_BODY_HOSTED_URL = """### Actual Behavior
|
||||
Reproduced on app.all-hands.dev/canvas — see video below.
|
||||
BUG_BODY_HOSTED_URL = """### Steps to Reproduce
|
||||
Reproduced on app.all-hands.dev/canvas.
|
||||
|
||||
### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
<video src="https://example.com/bug.mp4"></video>
|
||||
|
||||
@@ -82,6 +95,15 @@ Reproduced on app.all-hands.dev/canvas — see video below.
|
||||
- [ ] Fixed
|
||||
"""
|
||||
|
||||
BUG_BODY_MISSING_REPRODUCTION = """### Actual Behavior
|
||||
The button was misaligned.
|
||||
|
||||
|
||||
|
||||
### Acceptance Criteria
|
||||
- [ ] Button is centered
|
||||
"""
|
||||
|
||||
ENHANCEMENT_BODY_READY = """### Desired Behavior
|
||||
The button should animate on hover.
|
||||
|
||||
@@ -143,6 +165,12 @@ def test_bug_not_ready_empty_actual():
|
||||
assert any("Actual Behavior" in r for r in result.reasons)
|
||||
|
||||
|
||||
|
||||
def test_bug_not_ready_missing_reproduction():
|
||||
result = evaluate_readiness(BUG_BODY_MISSING_REPRODUCTION, [BUG_LABEL])
|
||||
assert not result.ready
|
||||
assert any("Steps to Reproduce" in r for r in result.reasons)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enhancement readiness
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -242,7 +270,10 @@ The template says:
|
||||
assert set(sections) == {"notes"}
|
||||
|
||||
def test_fenced_heading_does_not_truncate_actual_behavior():
|
||||
body = """### Actual Behavior
|
||||
body = """### Steps to Reproduce
|
||||
Run `npm run dev` and reproduce the error.
|
||||
|
||||
### Actual Behavior
|
||||
I ran `npm run dev` and saw:
|
||||
|
||||
~~~text
|
||||
@@ -263,7 +294,10 @@ something went wrong
|
||||
|
||||
def test_unclosed_fence_does_not_swallow_later_sections():
|
||||
"""One stray marker in a log paste must not reject an otherwise-ready report."""
|
||||
body = """### Relevant Logs
|
||||
body = """### Steps to Reproduce
|
||||
Run `npm run dev` and reproduce the crash.
|
||||
|
||||
### Relevant Logs
|
||||
```shell
|
||||
Traceback (most recent call last):
|
||||
the paste was cut off before the closing fence
|
||||
|
||||
@@ -3,10 +3,11 @@ name: Issue Readiness Check
|
||||
|
||||
# Manages the `ready-for-dev` label based on type-specific readiness criteria.
|
||||
#
|
||||
# Bug reports (label `bug`): the Actual Behavior section must reference a
|
||||
# Bug reports (label `bug`): the Steps to Reproduce section must reference a
|
||||
# supported run method (`agent-canvas`, `npm run`, or
|
||||
# `app.all-hands.dev/canvas`) and embed a screenshot or video, plus an
|
||||
# Acceptance Criteria section with at least one checklist item.
|
||||
# `app.all-hands.dev/canvas`), the Actual Behavior section must embed a
|
||||
# screenshot or video, plus an Acceptance Criteria section with at least one
|
||||
# checklist item.
|
||||
#
|
||||
# Enhancements (label `enhancement`): the body must contain Desired Behavior and
|
||||
# Acceptance Criteria sections, the latter with at least one checklist item.
|
||||
|
||||
Reference in New Issue
Block a user