mirror of
https://github.com/OpenHands/OpenHands.git
synced 2026-10-07 12:58:49 +08:00
Automate good first issue labeling in issue triage (#14498)
Co-authored-by: openhands <openhands@all-hands.dev>
This commit is contained in:
Engel Nyst
@@ -15,10 +15,11 @@ on:
|
||||
options:
|
||||
- smoke-clone
|
||||
- issue-check
|
||||
- good-first-issue-check
|
||||
- auto-close
|
||||
default: smoke-clone
|
||||
issue_number:
|
||||
description: Existing issue number to analyze when mode is issue-check
|
||||
description: Existing issue number to analyze for manual issue-check modes
|
||||
required: false
|
||||
type: number
|
||||
close_after_days:
|
||||
@@ -57,13 +58,19 @@ jobs:
|
||||
|
||||
issue-duplicate-check:
|
||||
if: |
|
||||
github.event_name == 'issues' ||
|
||||
(github.event_name == 'issues' && github.event.action == 'opened') ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.mode == 'issue-check' && inputs.issue_number != null)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
concurrency:
|
||||
group: issue-duplicate-check-${{ github.repository }}-${{ github.event.issue.number || inputs.issue_number }}
|
||||
cancel-in-progress: false
|
||||
outputs:
|
||||
should_comment: ${{ steps.parsed_result.outputs.should_comment }}
|
||||
is_duplicate: ${{ steps.parsed_result.outputs.is_duplicate }}
|
||||
auto_close_candidate: ${{ steps.parsed_result.outputs.auto_close_candidate }}
|
||||
confidence: ${{ steps.parsed_result.outputs.confidence }}
|
||||
classification: ${{ steps.parsed_result.outputs.classification }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
@@ -430,6 +437,234 @@ jobs:
|
||||
await ensureCandidateLabelOnIssue();
|
||||
}
|
||||
|
||||
issue-good-first-issue-check:
|
||||
if: |
|
||||
(
|
||||
github.event_name == 'issues' ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.mode == 'good-first-issue-check' && inputs.issue_number != null)
|
||||
) &&
|
||||
(needs.issue-duplicate-check.result == 'success' || needs.issue-duplicate-check.result == 'skipped')
|
||||
needs:
|
||||
- issue-duplicate-check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
concurrency:
|
||||
group: issue-good-first-issue-check-${{ github.repository }}-${{ github.event.issue.number || inputs.issue_number }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: Validate good first issue check inputs
|
||||
env:
|
||||
OPENHANDS_API_KEY: ${{ secrets.OPENHANDS_API_KEY }}
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}
|
||||
run: |
|
||||
if [ -z "$OPENHANDS_API_KEY" ]; then
|
||||
echo "Error: OPENHANDS_API_KEY secret is required"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$ISSUE_NUMBER" ]; then
|
||||
echo "Error: ISSUE_NUMBER is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Determine whether good first issue auto-label check should run
|
||||
id: precheck
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}
|
||||
DUPLICATE_SHOULD_COMMENT: ${{ needs.issue-duplicate-check.outputs.should_comment }}
|
||||
DUPLICATE_CLASSIFICATION: ${{ needs.issue-duplicate-check.outputs.classification }}
|
||||
with:
|
||||
github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
script: |
|
||||
const issueNumber = Number(process.env.ISSUE_NUMBER);
|
||||
const duplicateShouldComment = process.env.DUPLICATE_SHOULD_COMMENT === 'true';
|
||||
const duplicateClassification = process.env.DUPLICATE_CLASSIFICATION || '';
|
||||
const { data: issue } = await github.rest.issues.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
});
|
||||
const labelNames = (issue.labels || []).map((label) => (
|
||||
typeof label === 'string' ? label : label.name
|
||||
)).filter(Boolean);
|
||||
|
||||
let skipReason = '';
|
||||
if (issue.pull_request) {
|
||||
skipReason = `#${issueNumber} is a pull request, not an issue.`;
|
||||
} else if (issue.state !== 'open' || issue.locked) {
|
||||
skipReason = `#${issueNumber} is not an open, unlocked issue.`;
|
||||
} else if (labelNames.includes('good first issue')) {
|
||||
skipReason = 'Issue is already labeled good first issue.';
|
||||
} else if (labelNames.includes('duplicate-candidate')) {
|
||||
skipReason = 'Issue is marked duplicate-candidate, so good first issue auto-labeling is blocked.';
|
||||
} else if (
|
||||
duplicateShouldComment ||
|
||||
['duplicate', 'overlapping-scope'].includes(duplicateClassification)
|
||||
) {
|
||||
skipReason = `Duplicate check classified the issue as ${duplicateClassification || 'overlapping-scope'} and blocked good first issue auto-labeling.`;
|
||||
}
|
||||
|
||||
core.setOutput('issue_url', issue.html_url || '');
|
||||
core.setOutput('labels_json', JSON.stringify(labelNames));
|
||||
core.setOutput('skip_reason', skipReason);
|
||||
core.setOutput('should_run', skipReason ? 'false' : 'true');
|
||||
|
||||
- name: Summarize skipped good first issue check
|
||||
if: steps.precheck.outputs.should_run != 'true'
|
||||
run: |
|
||||
{
|
||||
echo "## Good first issue check"
|
||||
echo
|
||||
echo "- Issue: #${{ github.event.issue.number || inputs.issue_number }}"
|
||||
echo "- Skipped: ${{ steps.precheck.outputs.skip_reason }}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Run OpenHands good first issue conversation
|
||||
if: steps.precheck.outputs.should_run == 'true'
|
||||
id: run_check
|
||||
env:
|
||||
OPENHANDS_API_KEY: ${{ secrets.OPENHANDS_API_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}
|
||||
OUTPUT_PATH: ${{ runner.temp }}/good-first-issue-check-result.json
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
python scripts/issue_good_first_issue_check_openhands.py \
|
||||
--repository "$REPOSITORY" \
|
||||
--issue-number "$ISSUE_NUMBER" \
|
||||
--output "$OUTPUT_PATH"
|
||||
test -f "$OUTPUT_PATH" || {
|
||||
echo "Error: Output file not created"
|
||||
exit 1
|
||||
}
|
||||
echo "result_path=$OUTPUT_PATH" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Parse good first issue check result
|
||||
if: steps.precheck.outputs.should_run == 'true'
|
||||
id: parsed_result
|
||||
env:
|
||||
RESULT_PATH: ${{ steps.run_check.outputs.result_path }}
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
result = json.loads(Path(os.environ['RESULT_PATH']).read_text())
|
||||
except (FileNotFoundError, json.JSONDecodeError) as exc:
|
||||
print(
|
||||
f"Error: Failed to read good first issue result: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1) from exc
|
||||
output_path = Path(os.environ['GITHUB_OUTPUT'])
|
||||
summary_path = Path(os.environ['GITHUB_STEP_SUMMARY'])
|
||||
|
||||
def write_multiline(name: str, value: str) -> None:
|
||||
delimiter = f"EOF_{os.urandom(8).hex()}"
|
||||
with output_path.open('a', encoding='utf-8') as fh:
|
||||
fh.write(f"{name}<<{delimiter}\n{value}\n{delimiter}\n")
|
||||
|
||||
with output_path.open('a', encoding='utf-8') as fh:
|
||||
fh.write(
|
||||
f"should_apply_label={'true' if result.get('should_apply_label') else 'false'}\n"
|
||||
)
|
||||
fh.write(f"confidence={result.get('confidence', '')}\n")
|
||||
fh.write(f"conversation_url={result.get('conversation_url', '')}\n")
|
||||
|
||||
write_multiline('summary', str(result.get('summary', '')).strip())
|
||||
write_multiline(
|
||||
'criteria_met_json',
|
||||
json.dumps(result.get('criteria_met', []), ensure_ascii=False),
|
||||
)
|
||||
write_multiline(
|
||||
'disqualifiers_json',
|
||||
json.dumps(result.get('disqualifiers', []), ensure_ascii=False),
|
||||
)
|
||||
|
||||
criteria_lines = [
|
||||
f"- {item}" for item in result.get('criteria_met', [])
|
||||
]
|
||||
disqualifier_lines = [
|
||||
f"- {item}" for item in result.get('disqualifiers', [])
|
||||
]
|
||||
|
||||
summary_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"## Good first issue check result",
|
||||
"",
|
||||
f"- Repository: {result.get('repository')}",
|
||||
f"- Issue: #{result.get('issue_number')}",
|
||||
f"- Should apply label: {result.get('should_apply_label')}",
|
||||
f"- Confidence: {result.get('confidence')}",
|
||||
f"- Conversation: {result.get('conversation_url')}",
|
||||
"",
|
||||
"### Summary",
|
||||
result.get('summary', ''),
|
||||
"",
|
||||
"### Criteria met",
|
||||
*(criteria_lines or ["- None"]),
|
||||
"",
|
||||
"### Disqualifiers",
|
||||
*(disqualifier_lines or ["- None"]),
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding='utf-8',
|
||||
)
|
||||
PY
|
||||
|
||||
- name: Apply good first issue label
|
||||
if: |
|
||||
steps.precheck.outputs.should_run == 'true' &&
|
||||
steps.parsed_result.outputs.should_apply_label == 'true'
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number || inputs.issue_number }}
|
||||
with:
|
||||
github-token: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC || github.token }}
|
||||
script: |
|
||||
const issueNumber = Number(process.env.ISSUE_NUMBER);
|
||||
const labelName = 'good first issue';
|
||||
const { data: issue } = await github.rest.issues.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
});
|
||||
const labelNames = (issue.labels || []).map((label) => (
|
||||
typeof label === 'string' ? label : label.name
|
||||
)).filter(Boolean);
|
||||
if (labelNames.includes(labelName)) {
|
||||
core.info(`Issue #${issueNumber} already has the ${labelName} label.`);
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
labels: [labelName],
|
||||
});
|
||||
|
||||
- name: Summarize good first issue label action
|
||||
if: steps.precheck.outputs.should_run == 'true'
|
||||
run: |
|
||||
if [ "${{ steps.parsed_result.outputs.should_apply_label }}" = "true" ]; then
|
||||
echo "- Applied label: good first issue" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "- Label not applied" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
auto-close-duplicates:
|
||||
if: |
|
||||
github.event_name == 'schedule' ||
|
||||
|
||||
@@ -497,3 +497,9 @@ Called by `workspace.get_llm()` in the SDK to retrieve LLM config with the API k
|
||||
- `openhands/sdk/llm/llm.py`: `LLM.api_key` accepts `SecretSource` (including `LookupSecret`)
|
||||
- `openhands/workspace/cloud/workspace.py`: `get_llm()` and `get_secrets()` return LookupSecret-backed objects
|
||||
- Tests: `tests/sdk/llm/test_llm_secret_source_api_key.py`, `tests/workspace/test_cloud_workspace_sdk_settings.py`
|
||||
|
||||
### Issue Triage Automation
|
||||
|
||||
- `.github/workflows/issue-duplicate-checker.yml` now has a second job that auto-applies `good first issue` after the duplicate check completes.
|
||||
- The duplicate check is used only as a veto/guardrail for `good first issue` automation: duplicate or overlapping-scope issues should not be auto-labeled.
|
||||
- The OpenHands classifier logic for newcomer suitability lives in `scripts/issue_good_first_issue_check_openhands.py`, with focused unit coverage in `tests/unit/test_issue_good_first_issue_check_openhands.py`.
|
||||
|
||||
+9
-1
@@ -10,7 +10,15 @@ These are the procedures and guidelines on how issues are triaged in this repo b
|
||||
* **Critical**: Affecting all users or potential security issues.
|
||||
|
||||
## Difficulty
|
||||
* Issues good for newcomers may be tagged with **good first issue**.
|
||||
* Issues good for newcomers may be tagged with **good first issue** by maintainers or by the automated triage workflow when the issue clearly meets the criteria below.
|
||||
* The `welcome-good-first-issue` workflow only posts a welcome comment after the label is present; it does not decide whether the label should be applied.
|
||||
* The automated triage workflow should be conservative and should not auto-apply **good first issue** to issues that look like duplicates or overlapping-scope reports.
|
||||
* Use **good first issue** only when all of the following are true:
|
||||
* The work is narrow in scope and should be solved as a single issue rather than a multi-step project.
|
||||
* The bug, request, or expected outcome is already clear enough that a contributor should not need major discovery work before starting.
|
||||
* The likely fix stays within a well-bounded area of the repo and does not require deep architecture knowledge, cross-repo coordination, enterprise-only context, migrations, or infrastructure work. Exclude enterprise/ directory from these.
|
||||
* The validation path is straightforward: the PR author should be able to give clear **How to Test** steps, and if the change is user-facing, screenshots or video evidence should be practical to provide.
|
||||
* Avoid **good first issue** for broad design discussions, umbrella tracking issues, items missing reproduction steps or acceptance criteria, and changes that are likely to require significant maintainer guidance.
|
||||
|
||||
## Not Enough Information
|
||||
* User is asked to provide more information (logs, how to reproduce, etc.) when the issue is not clear.
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
OPENHANDS_BASE_URL = os.environ.get('OPENHANDS_BASE_URL', 'https://app.all-hands.dev')
|
||||
REPOSITORY_PATTERN = re.compile(r'^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$')
|
||||
GITHUB_API_BASE_URL = os.environ.get('GITHUB_API_BASE_URL', 'https://api.github.com')
|
||||
FAILED_EXECUTION_STATUSES = {
|
||||
'error',
|
||||
'errored',
|
||||
'failed',
|
||||
'stopped',
|
||||
}
|
||||
SUCCESSFUL_TERMINAL_EXECUTION_STATUSES = {
|
||||
'completed',
|
||||
'finished',
|
||||
}
|
||||
TERMINAL_EXECUTION_STATUSES = (
|
||||
FAILED_EXECUTION_STATUSES | SUCCESSFUL_TERMINAL_EXECUTION_STATUSES
|
||||
)
|
||||
EVENT_SEARCH_LIMIT = 1000
|
||||
EVENT_SEARCH_LIMIT_HIT_MESSAGE = (
|
||||
f'Event search returned at least {EVENT_SEARCH_LIMIT} events; results may be '
|
||||
'incomplete'
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
'Start an OpenHands Cloud conversation that decides whether a '
|
||||
'GitHub issue should be labeled good first issue.'
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
'--repository', required=True, help='Repository in owner/repo form'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--issue-number', required=True, type=int, help='Issue number to inspect'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
default='good-first-issue-check-result.json',
|
||||
help='Path where the JSON result should be written',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--poll-interval-seconds',
|
||||
default=5,
|
||||
type=int,
|
||||
help='Polling interval while waiting for the conversation to finish',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--max-wait-seconds',
|
||||
default=900,
|
||||
type=int,
|
||||
help=(
|
||||
'Maximum time to wait per polling phase; if a start task must be awaited '
|
||||
'first, the total runtime can approach twice this value'
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def github_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'User-Agent': 'openhands-good-first-issue-check',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
}
|
||||
github_token = os.environ.get('GITHUB_TOKEN')
|
||||
if github_token:
|
||||
headers['Authorization'] = f'Bearer {github_token}'
|
||||
return headers
|
||||
|
||||
|
||||
def openhands_headers() -> dict[str, str]:
|
||||
api_key = os.environ.get('OPENHANDS_API_KEY')
|
||||
if not api_key:
|
||||
raise RuntimeError('OPENHANDS_API_KEY environment variable is required')
|
||||
return {
|
||||
'Authorization': f'Bearer {api_key}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
|
||||
|
||||
def request_json(
|
||||
base_url: str,
|
||||
path: str,
|
||||
*,
|
||||
method: str = 'GET',
|
||||
headers: dict[str, str] | None = None,
|
||||
body: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
data = json.dumps(body).encode('utf-8') if body is not None else None
|
||||
request = urllib.request.Request(
|
||||
f'{base_url}{path}',
|
||||
data=data,
|
||||
headers=headers or {},
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.load(response)
|
||||
except urllib.error.HTTPError as exc:
|
||||
error_body = exc.read().decode('utf-8', errors='replace')
|
||||
raise RuntimeError(
|
||||
f'{method} {base_url}{path} failed with HTTP {exc.code}: {error_body}'
|
||||
) from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f'Failed to parse JSON from {method} {base_url}{path}: {exc}'
|
||||
) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f'{method} {base_url}{path} failed: {exc}') from exc
|
||||
|
||||
|
||||
def fetch_issue(repository: str, issue_number: int) -> dict[str, Any]:
|
||||
if not REPOSITORY_PATTERN.fullmatch(repository):
|
||||
raise ValueError(f'Invalid repository format: {repository}')
|
||||
return request_json(
|
||||
GITHUB_API_BASE_URL,
|
||||
f'/repos/{repository}/issues/{issue_number}',
|
||||
headers=github_headers(),
|
||||
)
|
||||
|
||||
|
||||
def escape_json_text(value: str | None) -> str:
|
||||
return json.dumps(value or '', ensure_ascii=False)
|
||||
|
||||
|
||||
def extract_label_names(issue: dict[str, Any]) -> list[str]:
|
||||
labels = issue.get('labels')
|
||||
if not isinstance(labels, list):
|
||||
return []
|
||||
|
||||
label_names: list[str] = []
|
||||
for label in labels:
|
||||
if isinstance(label, str):
|
||||
candidate = label.strip()
|
||||
elif isinstance(label, dict):
|
||||
candidate = str(label.get('name') or '').strip()
|
||||
else:
|
||||
candidate = ''
|
||||
if candidate:
|
||||
label_names.append(candidate)
|
||||
return sorted(set(label_names), key=str.lower)
|
||||
|
||||
|
||||
def build_prompt(repository: str, issue: dict[str, Any]) -> str:
|
||||
issue_number = issue['number']
|
||||
issue_title = issue.get('title', '')
|
||||
issue_body = issue.get('body') or ''
|
||||
issue_url = issue.get('html_url', '')
|
||||
issue_labels = extract_label_names(issue)
|
||||
issue_title_json = escape_json_text(issue_title)
|
||||
issue_body_json = escape_json_text(issue_body)
|
||||
issue_labels_json = json.dumps(issue_labels, ensure_ascii=False)
|
||||
|
||||
return '\n'.join(
|
||||
[
|
||||
'You are deciding whether a GitHub issue should be labeled `good first issue` for newcomers to this repository.',
|
||||
'',
|
||||
'Be conservative. This label is auto-applied by automation, so false positives are more harmful than false negatives.',
|
||||
'',
|
||||
f'Repository: {repository}',
|
||||
f'Issue number: #{issue_number}',
|
||||
f'Issue URL: {issue_url}',
|
||||
f'Issue labels (JSON array): {issue_labels_json}',
|
||||
f'Issue title (JSON-escaped string): {issue_title_json}',
|
||||
f'Issue body (JSON-escaped string): {issue_body_json}',
|
||||
'',
|
||||
'Use the repository, issue details, labels, and any code/documentation context you inspect to decide whether this issue is truly appropriate for a newcomer.',
|
||||
'',
|
||||
'All of the following must be true to apply `good first issue`:',
|
||||
'1. The work is narrow in scope and should be solved as a single issue rather than a multi-step project.',
|
||||
'2. The bug, request, or expected outcome is already clear enough that a contributor should not need major discovery work before starting.',
|
||||
'3. The likely fix stays within a well-bounded area of the repo and does not require deep architecture knowledge, cross-repo coordination, enterprise-only context, migrations, security-sensitive changes, or infrastructure work.',
|
||||
'4. The validation path is straightforward: the PR author should be able to give clear How to Test steps, and if the change is user-facing, screenshots or video evidence should be practical to provide.',
|
||||
'5. A reviewer should be able to evaluate the change with the normal PR review workflow, without special credentials, paid services, or hard-to-reproduce environments.',
|
||||
'',
|
||||
'Do NOT apply `good first issue` for broad design discussions, umbrella tracking issues, items missing reproduction steps or acceptance criteria, or changes likely to require significant maintainer guidance.',
|
||||
'Do NOT treat “not a duplicate” as positive evidence on its own. Only use the criteria above.',
|
||||
'Do not post comments, do not modify files, and do not change repository state.',
|
||||
'Return exactly one JSON object and nothing else. Do not wrap it in markdown fences.',
|
||||
'',
|
||||
'Return schema:',
|
||||
'{',
|
||||
f' "issue_number": {issue_number},',
|
||||
' "should_apply_label": true or false,',
|
||||
' "confidence": "high" | "medium" | "low",',
|
||||
' "summary": "short explanation",',
|
||||
' "criteria_met": ["criterion that is satisfied"],',
|
||||
' "disqualifiers": ["reason the issue should not be auto-labeled"]',
|
||||
'}',
|
||||
'',
|
||||
'Rules:',
|
||||
'- `should_apply_label` must be true only when all required criteria are satisfied.',
|
||||
'- If any material uncertainty remains, return `should_apply_label: false`.',
|
||||
'- `should_apply_label` must be false unless confidence is `high`.',
|
||||
'- `criteria_met` and `disqualifiers` should each contain at most 5 short strings.',
|
||||
'- If the issue is missing enough detail to judge newcomer suitability confidently, return `should_apply_label: false`.',
|
||||
'- If the issue appears enterprise-only, security-sensitive, infrastructure-heavy, or broadly architectural, return `should_apply_label: false`.',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def start_conversation(
|
||||
prompt: str, repository: str, issue_number: int
|
||||
) -> dict[str, Any]:
|
||||
body = {
|
||||
'title': f'Good first issue check #{issue_number}',
|
||||
'selected_repository': repository,
|
||||
'initial_message': {
|
||||
'content': [
|
||||
{
|
||||
'type': 'text',
|
||||
'text': prompt,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
return request_json(
|
||||
OPENHANDS_BASE_URL,
|
||||
'/api/v1/app-conversations',
|
||||
method='POST',
|
||||
headers=openhands_headers(),
|
||||
body=body,
|
||||
)
|
||||
|
||||
|
||||
def extract_first_item(payload: Any) -> dict[str, Any] | None:
|
||||
if isinstance(payload, list):
|
||||
first_item = payload[0] if payload else None
|
||||
return first_item if isinstance(first_item, dict) else None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
items = payload.get('items')
|
||||
if isinstance(items, list):
|
||||
first_item = items[0] if items else None
|
||||
return first_item if isinstance(first_item, dict) else None
|
||||
return payload
|
||||
|
||||
|
||||
def poll_start_task(
|
||||
start_task_id: str, poll_interval_seconds: int, max_wait_seconds: int
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.time() + max_wait_seconds
|
||||
while time.time() < deadline:
|
||||
payload = request_json(
|
||||
OPENHANDS_BASE_URL,
|
||||
f'/api/v1/app-conversations/start-tasks?ids={urllib.parse.quote(start_task_id)}',
|
||||
headers={'Authorization': openhands_headers()['Authorization']},
|
||||
)
|
||||
item = extract_first_item(payload)
|
||||
if item is None:
|
||||
time.sleep(poll_interval_seconds)
|
||||
continue
|
||||
status = item.get('status')
|
||||
if status == 'READY' and item.get('app_conversation_id'):
|
||||
return item
|
||||
if status in {'ERROR', 'FAILED'}:
|
||||
raise RuntimeError(f'OpenHands start task failed: {json.dumps(item)}')
|
||||
time.sleep(poll_interval_seconds)
|
||||
raise TimeoutError(
|
||||
f'Timed out waiting for start task {start_task_id} to become ready'
|
||||
)
|
||||
|
||||
|
||||
def poll_conversation(
|
||||
app_conversation_id: str, poll_interval_seconds: int, max_wait_seconds: int
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.time() + max_wait_seconds
|
||||
while time.time() < deadline:
|
||||
payload = request_json(
|
||||
OPENHANDS_BASE_URL,
|
||||
f'/api/v1/app-conversations?ids={urllib.parse.quote(app_conversation_id)}',
|
||||
headers={'Authorization': openhands_headers()['Authorization']},
|
||||
)
|
||||
item = extract_first_item(payload)
|
||||
if item is None:
|
||||
time.sleep(poll_interval_seconds)
|
||||
continue
|
||||
execution_status = str(item.get('execution_status', '')).lower()
|
||||
if execution_status in FAILED_EXECUTION_STATUSES:
|
||||
raise RuntimeError(
|
||||
'OpenHands conversation ended with '
|
||||
f'{execution_status}: {json.dumps(item)}'
|
||||
)
|
||||
if execution_status in SUCCESSFUL_TERMINAL_EXECUTION_STATUSES:
|
||||
return item
|
||||
time.sleep(poll_interval_seconds)
|
||||
raise TimeoutError(
|
||||
f'Timed out waiting for conversation {app_conversation_id} to finish running'
|
||||
)
|
||||
|
||||
|
||||
def validate_event_search_results(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if len(events) >= EVENT_SEARCH_LIMIT:
|
||||
raise RuntimeError(EVENT_SEARCH_LIMIT_HIT_MESSAGE)
|
||||
return events
|
||||
|
||||
|
||||
def fetch_app_server_events(app_conversation_id: str) -> list[dict[str, Any]]:
|
||||
payload = request_json(
|
||||
OPENHANDS_BASE_URL,
|
||||
f'/api/v1/conversation/{urllib.parse.quote(app_conversation_id)}/events/search?limit={EVENT_SEARCH_LIMIT}',
|
||||
headers={'Authorization': openhands_headers()['Authorization']},
|
||||
)
|
||||
if isinstance(payload, dict):
|
||||
items = payload.get('items')
|
||||
return validate_event_search_results(items) if isinstance(items, list) else []
|
||||
if isinstance(payload, list):
|
||||
return validate_event_search_results(payload)
|
||||
return []
|
||||
|
||||
|
||||
def fetch_agent_server_events(
|
||||
app_conversation_id: str, agent_server_url: str, session_api_key: str
|
||||
) -> list[dict[str, Any]]:
|
||||
payload = request_json(
|
||||
agent_server_url,
|
||||
f'/api/conversations/{urllib.parse.quote(app_conversation_id)}/events/search?limit={EVENT_SEARCH_LIMIT}',
|
||||
headers={'X-Session-API-Key': session_api_key},
|
||||
)
|
||||
if isinstance(payload, dict):
|
||||
items = payload.get('items')
|
||||
return validate_event_search_results(items) if isinstance(items, list) else []
|
||||
if isinstance(payload, list):
|
||||
return validate_event_search_results(payload)
|
||||
return []
|
||||
|
||||
|
||||
def fetch_agent_server_final_response(
|
||||
app_conversation_id: str, agent_server_url: str, session_api_key: str
|
||||
) -> str:
|
||||
payload = request_json(
|
||||
agent_server_url,
|
||||
f'/api/conversations/{urllib.parse.quote(app_conversation_id)}/agent_final_response',
|
||||
headers={'X-Session-API-Key': session_api_key},
|
||||
)
|
||||
if not isinstance(payload, dict):
|
||||
return ''
|
||||
return str(payload.get('response') or '').strip()
|
||||
|
||||
|
||||
def extract_agent_server_url(conversation_url: str) -> str | None:
|
||||
marker = '/api/conversations/'
|
||||
if marker not in conversation_url:
|
||||
return None
|
||||
return conversation_url.rsplit(marker, 1)[0]
|
||||
|
||||
|
||||
def extract_last_agent_text(events: list[dict[str, Any]]) -> str:
|
||||
agent_events = [
|
||||
event
|
||||
for event in events
|
||||
if event.get('kind') == 'MessageEvent' and event.get('source') == 'agent'
|
||||
]
|
||||
if not agent_events:
|
||||
raise RuntimeError(
|
||||
'No assistant text message was found in the conversation events'
|
||||
)
|
||||
|
||||
llm_message = agent_events[-1].get('llm_message')
|
||||
if not isinstance(llm_message, dict):
|
||||
raise RuntimeError('Last agent message has no llm_message field')
|
||||
content = llm_message.get('content')
|
||||
if not isinstance(content, list):
|
||||
raise RuntimeError('Last agent message content is not a list')
|
||||
|
||||
text_parts: list[str] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if part.get('type') == 'text' and part.get('text'):
|
||||
text_parts.append(str(part['text']))
|
||||
if not text_parts:
|
||||
raise RuntimeError('Last agent message contains no text content')
|
||||
return ''.join(text_parts).strip()
|
||||
|
||||
|
||||
def parse_agent_json(text: str) -> dict[str, Any]:
|
||||
cleaned = text.strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except json.JSONDecodeError:
|
||||
decoder = json.JSONDecoder()
|
||||
for start, character in enumerate(cleaned):
|
||||
if character != '{':
|
||||
continue
|
||||
try:
|
||||
candidate, end = decoder.raw_decode(cleaned[start:])
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
trailing = cleaned[start + end :].strip()
|
||||
if trailing not in {'', '```'}:
|
||||
continue
|
||||
if isinstance(candidate, dict):
|
||||
return candidate
|
||||
raise ValueError('No valid JSON object found in the agent response')
|
||||
|
||||
|
||||
def as_bool(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {'true', '1', 'yes'}
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
return False
|
||||
|
||||
|
||||
def normalize_string_list(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
|
||||
normalized: list[str] = []
|
||||
for item in value:
|
||||
text = str(item).strip() if item is not None else ''
|
||||
if text:
|
||||
normalized.append(text)
|
||||
if len(normalized) == 5:
|
||||
break
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_result(result: dict[str, Any]) -> dict[str, Any]:
|
||||
normalized = dict(result)
|
||||
normalized['should_apply_label'] = as_bool(normalized.get('should_apply_label'))
|
||||
|
||||
confidence = str(normalized.get('confidence') or 'low').strip().lower()
|
||||
if confidence not in {'high', 'medium', 'low'}:
|
||||
confidence = 'low'
|
||||
normalized['confidence'] = confidence
|
||||
|
||||
normalized['summary'] = str(normalized.get('summary') or '').strip()
|
||||
normalized['criteria_met'] = normalize_string_list(normalized.get('criteria_met'))
|
||||
normalized['disqualifiers'] = normalize_string_list(normalized.get('disqualifiers'))
|
||||
|
||||
if confidence != 'high':
|
||||
normalized['should_apply_label'] = False
|
||||
if normalized['disqualifiers']:
|
||||
normalized['should_apply_label'] = False
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
issue = fetch_issue(args.repository, args.issue_number)
|
||||
if issue.get('pull_request'):
|
||||
raise RuntimeError(f'#{args.issue_number} is a pull request, not an issue')
|
||||
|
||||
prompt = build_prompt(args.repository, issue)
|
||||
start_task = start_conversation(prompt, args.repository, args.issue_number)
|
||||
app_conversation_id = start_task.get('app_conversation_id')
|
||||
conversation_url = ''
|
||||
|
||||
if not app_conversation_id:
|
||||
task_id = start_task.get('id')
|
||||
if not task_id:
|
||||
raise RuntimeError(f'Missing id in start task response: {start_task}')
|
||||
ready_task = poll_start_task(
|
||||
task_id,
|
||||
args.poll_interval_seconds,
|
||||
args.max_wait_seconds,
|
||||
)
|
||||
app_conversation_id = ready_task.get('app_conversation_id')
|
||||
if not app_conversation_id:
|
||||
raise RuntimeError(f'Missing app_conversation_id in response: {ready_task}')
|
||||
|
||||
conversation = poll_conversation(
|
||||
app_conversation_id,
|
||||
args.poll_interval_seconds,
|
||||
args.max_wait_seconds,
|
||||
)
|
||||
conversation_url = (
|
||||
conversation.get('conversation_url')
|
||||
or f'{OPENHANDS_BASE_URL}/conversations/{app_conversation_id}'
|
||||
)
|
||||
session_api_key_value = conversation.get('session_api_key')
|
||||
if session_api_key_value and not isinstance(session_api_key_value, str):
|
||||
raise RuntimeError(
|
||||
'session_api_key had unexpected type in the OpenHands conversation: '
|
||||
f'{type(session_api_key_value).__name__}'
|
||||
)
|
||||
session_api_key = session_api_key_value or ''
|
||||
agent_server_url = extract_agent_server_url(conversation_url)
|
||||
|
||||
agent_text = ''
|
||||
if agent_server_url and session_api_key:
|
||||
try:
|
||||
agent_text = fetch_agent_server_final_response(
|
||||
app_conversation_id,
|
||||
agent_server_url,
|
||||
session_api_key,
|
||||
)
|
||||
except RuntimeError:
|
||||
agent_text = ''
|
||||
if not agent_text:
|
||||
events = fetch_app_server_events(app_conversation_id)
|
||||
try:
|
||||
agent_text = extract_last_agent_text(events)
|
||||
except RuntimeError as exc:
|
||||
if not session_api_key:
|
||||
raise RuntimeError(
|
||||
'App server events did not contain assistant text and '
|
||||
'session_api_key was missing from the OpenHands conversation'
|
||||
) from exc
|
||||
if not agent_server_url:
|
||||
raise RuntimeError(
|
||||
'App server events did not contain assistant text and cannot '
|
||||
'extract agent server URL from conversation URL: '
|
||||
f'{conversation_url}'
|
||||
) from exc
|
||||
events = fetch_agent_server_events(
|
||||
app_conversation_id,
|
||||
agent_server_url,
|
||||
session_api_key,
|
||||
)
|
||||
agent_text = extract_last_agent_text(events)
|
||||
result = normalize_result(parse_agent_json(agent_text))
|
||||
|
||||
result['issue_number'] = args.issue_number
|
||||
result['repository'] = args.repository
|
||||
result['app_conversation_id'] = app_conversation_id
|
||||
result['conversation_url'] = conversation_url
|
||||
result['agent_response'] = agent_text
|
||||
|
||||
output_path = Path(args.output)
|
||||
try:
|
||||
output_path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + '\n')
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f'Failed to write output to {output_path}: {exc}') from exc
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
'issue_number': result.get('issue_number'),
|
||||
'should_apply_label': result.get('should_apply_label'),
|
||||
'confidence': result.get('confidence'),
|
||||
'conversation_url': result.get('conversation_url'),
|
||||
'output': str(output_path),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc: # pragma: no cover - CLI boundary
|
||||
print(f'Error: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,291 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import itertools
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MODULE_COUNTER = itertools.count()
|
||||
|
||||
|
||||
def load_module(script_name: str):
|
||||
path = ROOT / 'scripts' / script_name
|
||||
module_name = f'test_{path.stem}_{next(MODULE_COUNTER)}'
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise AssertionError(f'Unable to load module from {path}')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def make_agent_message(text: str) -> dict:
|
||||
return {
|
||||
'kind': 'MessageEvent',
|
||||
'source': 'agent',
|
||||
'llm_message': {'content': [{'type': 'text', 'text': text}]},
|
||||
}
|
||||
|
||||
|
||||
def test_extract_label_names_normalizes_issue_labels():
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
|
||||
label_names = module.extract_label_names(
|
||||
{
|
||||
'labels': [
|
||||
{'name': 'bug'},
|
||||
'good first issue',
|
||||
{'name': 'Bug'},
|
||||
{'name': ''},
|
||||
None,
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert label_names == ['Bug', 'bug', 'good first issue']
|
||||
|
||||
|
||||
def test_build_prompt_includes_issue_labels_and_criteria():
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
|
||||
prompt = module.build_prompt(
|
||||
'OpenHands/OpenHands',
|
||||
{
|
||||
'number': 123,
|
||||
'title': 'Clarify triage behavior',
|
||||
'body': 'The issue needs a clear automated path.',
|
||||
'html_url': 'https://github.com/OpenHands/OpenHands/issues/123',
|
||||
'labels': [{'name': 'bug'}, {'name': 'frontend'}],
|
||||
},
|
||||
)
|
||||
|
||||
assert 'Issue labels (JSON array): ["bug", "frontend"]' in prompt
|
||||
assert 'Do NOT treat “not a duplicate” as positive evidence on its own.' in prompt
|
||||
assert 'All of the following must be true to apply `good first issue`:' in prompt
|
||||
|
||||
|
||||
def test_normalize_result_requires_high_confidence_and_no_disqualifiers():
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
|
||||
normalized = module.normalize_result(
|
||||
{
|
||||
'should_apply_label': True,
|
||||
'confidence': 'medium',
|
||||
'summary': 'Looks promising',
|
||||
'criteria_met': ['narrow scope', 'clear request'],
|
||||
'disqualifiers': [],
|
||||
}
|
||||
)
|
||||
|
||||
assert normalized['should_apply_label'] is False
|
||||
assert normalized['confidence'] == 'medium'
|
||||
|
||||
normalized = module.normalize_result(
|
||||
{
|
||||
'should_apply_label': True,
|
||||
'confidence': 'high',
|
||||
'summary': 'Looks promising',
|
||||
'criteria_met': ['narrow scope', 'clear request'],
|
||||
'disqualifiers': ['needs enterprise context'],
|
||||
}
|
||||
)
|
||||
|
||||
assert normalized['should_apply_label'] is False
|
||||
assert normalized['disqualifiers'] == ['needs enterprise context']
|
||||
|
||||
|
||||
def test_normalize_result_limits_string_lists():
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
|
||||
normalized = module.normalize_result(
|
||||
{
|
||||
'should_apply_label': True,
|
||||
'confidence': 'high',
|
||||
'summary': 'Looks promising',
|
||||
'criteria_met': ['1', '2', '3', '4', '5', '6'],
|
||||
'disqualifiers': [None, '', 'too broad', 42, ''],
|
||||
}
|
||||
)
|
||||
|
||||
assert normalized['criteria_met'] == ['1', '2', '3', '4', '5']
|
||||
assert normalized['disqualifiers'] == ['too broad', '42']
|
||||
assert normalized['should_apply_label'] is False
|
||||
|
||||
|
||||
def test_good_first_issue_main_writes_result_from_final_response(monkeypatch, tmp_path):
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
output_path = tmp_path / 'result.json'
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'parse_args',
|
||||
lambda: argparse.Namespace(
|
||||
repository='OpenHands/OpenHands',
|
||||
issue_number=123,
|
||||
output=str(output_path),
|
||||
poll_interval_seconds=1,
|
||||
max_wait_seconds=10,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_issue',
|
||||
lambda repository, issue_number: {
|
||||
'number': issue_number,
|
||||
'title': 'Issue title',
|
||||
'body': 'Issue body',
|
||||
'html_url': f'https://github.com/{repository}/issues/{issue_number}',
|
||||
'labels': [{'name': 'bug'}],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'start_conversation',
|
||||
lambda *args, **kwargs: {'app_conversation_id': 'conv-123'},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'poll_conversation',
|
||||
lambda app_conversation_id, poll_interval_seconds, max_wait_seconds: {
|
||||
'conversation_url': 'https://runtime.example/api/conversations/conv-123',
|
||||
'session_api_key': 'session-key',
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_agent_server_final_response',
|
||||
lambda app_conversation_id, agent_server_url, session_api_key: json.dumps(
|
||||
{
|
||||
'should_apply_label': True,
|
||||
'confidence': 'high',
|
||||
'summary': 'Narrow, clear, and easy to validate.',
|
||||
'criteria_met': ['narrow scope', 'clear expected outcome'],
|
||||
'disqualifiers': [],
|
||||
}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_app_server_events',
|
||||
lambda app_conversation_id: pytest.fail(
|
||||
'fetch_app_server_events should not run'
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_agent_server_events',
|
||||
lambda *args, **kwargs: pytest.fail('fetch_agent_server_events should not run'),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
|
||||
result = json.loads(output_path.read_text())
|
||||
assert result['should_apply_label'] is True
|
||||
assert result['confidence'] == 'high'
|
||||
assert result['criteria_met'] == ['narrow scope', 'clear expected outcome']
|
||||
assert result['repository'] == 'OpenHands/OpenHands'
|
||||
|
||||
|
||||
def test_good_first_issue_main_falls_back_to_app_server_events(monkeypatch, tmp_path):
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
output_path = tmp_path / 'result.json'
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'parse_args',
|
||||
lambda: argparse.Namespace(
|
||||
repository='OpenHands/OpenHands',
|
||||
issue_number=123,
|
||||
output=str(output_path),
|
||||
poll_interval_seconds=1,
|
||||
max_wait_seconds=10,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_issue',
|
||||
lambda repository, issue_number: {
|
||||
'number': issue_number,
|
||||
'title': 'Issue title',
|
||||
'body': 'Issue body',
|
||||
'html_url': f'https://github.com/{repository}/issues/{issue_number}',
|
||||
'labels': [{'name': 'enhancement'}],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'start_conversation',
|
||||
lambda *args, **kwargs: {'app_conversation_id': 'conv-123'},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'poll_conversation',
|
||||
lambda app_conversation_id, poll_interval_seconds, max_wait_seconds: {
|
||||
'conversation_url': 'https://runtime.example/api/conversations/conv-123',
|
||||
'session_api_key': 'session-key',
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_agent_server_final_response',
|
||||
lambda *args, **kwargs: '',
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_app_server_events',
|
||||
lambda app_conversation_id: [
|
||||
make_agent_message(
|
||||
json.dumps(
|
||||
{
|
||||
'should_apply_label': False,
|
||||
'confidence': 'medium',
|
||||
'summary': 'Needs more discovery before a newcomer can pick it up.',
|
||||
'criteria_met': ['clear user-facing outcome'],
|
||||
'disqualifiers': ['scope is still too broad'],
|
||||
}
|
||||
)
|
||||
)
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_agent_server_events',
|
||||
lambda *args, **kwargs: pytest.fail('fetch_agent_server_events should not run'),
|
||||
)
|
||||
|
||||
assert module.main() == 0
|
||||
|
||||
result = json.loads(output_path.read_text())
|
||||
assert result['should_apply_label'] is False
|
||||
assert result['disqualifiers'] == ['scope is still too broad']
|
||||
|
||||
|
||||
def test_good_first_issue_main_rejects_pull_requests(monkeypatch, tmp_path):
|
||||
module = load_module('issue_good_first_issue_check_openhands.py')
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'parse_args',
|
||||
lambda: argparse.Namespace(
|
||||
repository='OpenHands/OpenHands',
|
||||
issue_number=123,
|
||||
output=str(tmp_path / 'result.json'),
|
||||
poll_interval_seconds=1,
|
||||
max_wait_seconds=10,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
'fetch_issue',
|
||||
lambda repository, issue_number: {
|
||||
'number': issue_number,
|
||||
'pull_request': {'url': 'https://example.test/pr/123'},
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match='is a pull request, not an issue'):
|
||||
module.main()
|
||||
Reference in New Issue
Block a user