Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
326c3efc70 | ||
|
|
152a0506c9 | ||
|
|
e1f0fe7eed | ||
|
|
248cb1e634 | ||
|
|
f26a35b17f | ||
|
|
b5627f27d8 | ||
|
|
3daf8c9984 | ||
|
|
d91428ad9d | ||
|
|
32b5c0e3fc | ||
|
|
30e0c7c726 | ||
|
|
98addbd6c4 | ||
|
|
f29147864e | ||
|
|
b89ec5b5df | ||
|
|
5bfe0c5222 | ||
|
|
5497079ab2 | ||
|
|
1d46200c47 | ||
|
|
8ca9cb1a4d | ||
|
|
927a17264d | ||
|
|
c8a1ecf69d | ||
|
|
9d015164a5 | ||
|
|
296a571263 | ||
|
|
0824b96d15 | ||
|
|
d3a7ce95a5 | ||
|
|
4cd3ee3832 | ||
|
|
f40973a1ca | ||
|
|
4e362ba70a | ||
|
|
48f15a3bca | ||
|
|
8e76729750 | ||
|
|
fd4d4a3fee | ||
|
|
c8683d58fc | ||
|
|
de63418f7e | ||
|
|
7639308f65 | ||
|
|
68e4a5aece | ||
|
|
84564e09b8 | ||
|
|
bc98239fb4 | ||
|
|
608be7655d | ||
|
|
b486d04d75 | ||
|
|
28df98c997 | ||
|
|
96578aa4a8 | ||
|
|
a418c47e29 | ||
|
|
05ca80ea30 | ||
|
|
e55256ba53 | ||
|
|
9d91530f94 | ||
|
|
46e4c979c4 | ||
|
|
8fc32e6af9 | ||
|
|
e60e62f193 | ||
|
|
816ae5e66e | ||
|
|
4048f53e35 | ||
|
|
027340292f |
@@ -32,6 +32,7 @@ import pathlib
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
@@ -190,7 +191,17 @@ def fetch_text(url: str, timeout: int = 8) -> str | None:
|
||||
set (raises the rate limit from 60 to 5 000 requests/hour).
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url):
|
||||
# Parse the URL and check the hostname rather than substring-matching
|
||||
# on the full URL string (CodeQL py/incomplete-url-substring-sanitization).
|
||||
# `https://evil.com/?u=github.com` would have passed the substring check.
|
||||
try:
|
||||
parsed_host = urllib.parse.urlparse(url).hostname or ""
|
||||
except ValueError:
|
||||
parsed_host = ""
|
||||
is_github_host = parsed_host == "github.com" or parsed_host.endswith(
|
||||
(".github.com", ".githubusercontent.com")
|
||||
) or parsed_host == "githubusercontent.com"
|
||||
if _GITHUB_TOKEN and is_github_host:
|
||||
headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
||||
@@ -95,31 +95,33 @@ jobs:
|
||||
|
||||
# Validate PR number is a positive integer (artifact comes from
|
||||
# untrusted fork code, so treat contents defensively).
|
||||
PR_NUM=$(cat "$DIR/pr_number" | tr -d '[:space:]')
|
||||
PR_NUM=$(tr -d '[:space:]' < "$DIR/pr_number")
|
||||
if ! [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::error::Invalid PR number in artifact: '$PR_NUM'"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
echo "pr_number=$PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
# Validate job-result strings against known GitHub Actions values.
|
||||
# Artifact contents come from the PR workflow (potentially untrusted
|
||||
# fork code), so we whitelist to prevent newline injection into
|
||||
# GITHUB_OUTPUT.
|
||||
validate_result() {
|
||||
local val
|
||||
val=$(cat "$1" | tr -d '[:space:]')
|
||||
val=$(tr -d '[:space:]' < "$1")
|
||||
case "$val" in
|
||||
success|failure|cancelled|skipped) echo "$val" ;;
|
||||
*) echo "unknown" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
echo "quality=$(validate_result "$DIR/quality_result")" >> "$GITHUB_OUTPUT"
|
||||
echo "tests=$(validate_result "$DIR/tests_result")" >> "$GITHUB_OUTPUT"
|
||||
echo "e2e=$(validate_result "$DIR/e2e_result")" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "skip=false"
|
||||
echo "pr_number=$PR_NUM"
|
||||
echo "quality=$(validate_result "$DIR/quality_result")"
|
||||
echo "tests=$(validate_result "$DIR/tests_result")"
|
||||
echo "e2e=$(validate_result "$DIR/e2e_result")"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout (for vitest config)
|
||||
if: steps.meta.outputs.skip != 'true'
|
||||
@@ -138,14 +140,15 @@ jobs:
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Find the latest successful CI run on main
|
||||
// Find recent successful CI runs on main (check several in case
|
||||
// the most recent artifact has expired).
|
||||
const runs = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: 'ci.yml',
|
||||
branch: 'main',
|
||||
status: 'success',
|
||||
per_page: 1,
|
||||
per_page: 5,
|
||||
});
|
||||
|
||||
if (runs.data.workflow_runs.length === 0) {
|
||||
@@ -154,32 +157,47 @@ jobs:
|
||||
return;
|
||||
}
|
||||
|
||||
const mainRunId = runs.data.workflow_runs[0].id;
|
||||
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: mainRunId,
|
||||
});
|
||||
// Try each run until we find a downloadable test-reports artifact
|
||||
for (const run of runs.data.workflow_runs) {
|
||||
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: run.id,
|
||||
});
|
||||
|
||||
const testReports = artifacts.data.artifacts.find(a => a.name === 'test-reports');
|
||||
if (!testReports) {
|
||||
core.setOutput('found', 'false');
|
||||
core.info('No test-reports artifact on main branch');
|
||||
return;
|
||||
const testReports = artifacts.data.artifacts.find(a => a.name === 'test-reports');
|
||||
if (!testReports) {
|
||||
core.info(`Run ${run.id}: no test-reports artifact, trying next`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const zip = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: testReports.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
|
||||
const dest = path.join(process.env.RUNNER_TEMP, 'base-coverage');
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
fs.writeFileSync(path.join(dest, 'base.zip'), Buffer.from(zip.data));
|
||||
core.setOutput('found', 'true');
|
||||
core.setOutput('dir', dest);
|
||||
return;
|
||||
} catch (err) {
|
||||
// 410 Gone means the artifact expired; try the next run
|
||||
if (err.status === 410 || err.response?.status === 410) {
|
||||
core.info(`Run ${run.id}: artifact expired, trying next`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const zip = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: testReports.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
|
||||
const dest = path.join(process.env.RUNNER_TEMP, 'base-coverage');
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
fs.writeFileSync(path.join(dest, 'base.zip'), Buffer.from(zip.data));
|
||||
core.setOutput('found', 'true');
|
||||
core.setOutput('dir', dest);
|
||||
// All attempts exhausted — no usable base coverage
|
||||
core.setOutput('found', 'false');
|
||||
core.info('No downloadable test-reports artifact found on main (all expired or missing)');
|
||||
|
||||
- name: Extract base coverage
|
||||
if: steps.meta.outputs.skip != 'true' && steps.base-coverage.outputs.found == 'true'
|
||||
@@ -234,7 +252,7 @@ jobs:
|
||||
printf -v "${prefix}_BRANCH_COV" '%s' ""
|
||||
printf -v "${prefix}_FUNCS_COV" '%s' ""
|
||||
printf -v "${prefix}_LINES_COV" '%s' ""
|
||||
return 1
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -263,14 +281,17 @@ jobs:
|
||||
fi
|
||||
}
|
||||
|
||||
read CLI_T CLI_P CLI_F CLI_S CLI_SU CLI_D <<< "$(sum_results "$RESULTS_FILE")"
|
||||
read WEB_T WEB_P WEB_F WEB_S WEB_SU WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")"
|
||||
# `_` placeholder for the suite-count column — positional
|
||||
# readability for sum_results' 6-field output, but the value
|
||||
# isn't surfaced in the report (suites are tracked per-test
|
||||
# framework, not as a top-line metric).
|
||||
read -r CLI_T CLI_P CLI_F CLI_S _ CLI_D <<< "$(sum_results "$RESULTS_FILE")"
|
||||
read -r WEB_T WEB_P WEB_F WEB_S _ WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")"
|
||||
|
||||
TOTAL=$((CLI_T + WEB_T))
|
||||
PASSED=$((CLI_P + WEB_P))
|
||||
FAILED=$((CLI_F + WEB_F))
|
||||
SKIPPED=$((CLI_S + WEB_S))
|
||||
SUITES=$((CLI_SU + WEB_SU))
|
||||
DURATION=$((CLI_D > WEB_D ? CLI_D : WEB_D))
|
||||
|
||||
# ── Status helpers ──
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
|
||||
uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
queries: security-and-quality
|
||||
@@ -66,6 +66,6 @@ jobs:
|
||||
- '**/test/fixtures/**'
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
|
||||
uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
|
||||
with:
|
||||
category: '/language:${{ matrix.language }}'
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
name: PR Autofix (publish)
|
||||
|
||||
# TRUSTED HALF of the autofix pipeline.
|
||||
#
|
||||
# Triggered by `pr-autofix.yml` completing on a PR (including fork PRs).
|
||||
# Downloads the diff artifact produced by the untrusted job and posts
|
||||
# inline review-comment suggestions to the PR using `reviewdog`. This
|
||||
# job NEVER checks out fork code — it only consumes the diff (data) and
|
||||
# calls the GitHub API. That isolation is what makes it safe to run
|
||||
# under `pull-requests: write` on fork-triggered events.
|
||||
#
|
||||
# Also posts (or edits) a single sticky summary comment so contributors
|
||||
# and AI agents have one stable, machine-readable signal that says
|
||||
# whether autofix had anything to suggest. Look for the heading
|
||||
# "## :sparkles: PR Autofix" in the PR's top-level comments.
|
||||
#
|
||||
# Reviewdog reporter: `github-pr-review` reads $REVIEWDOG_GITHUB_API_TOKEN
|
||||
# and posts via the GraphQL/REST PR-review API. It does not need a
|
||||
# checkout because the diff itself encodes file paths + line numbers.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['PR Autofix']
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
# Key on PR identity, NOT workflow_run.id — workflow_run.id is per-run
|
||||
# unique, which would defeat serialization and let two parallel
|
||||
# publishes both POST a sticky summary comment. CONTRIBUTING.md
|
||||
# § GitHub Actions — Concurrency Convention names this anti-pattern
|
||||
# explicitly. For fork PRs, `pull_requests[]` is empty in the
|
||||
# workflow_run payload, so we fall back to head-repo + head-branch.
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: publish-autofix
|
||||
if: >-
|
||||
github.event.workflow_run.event == 'pull_request'
|
||||
&& github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
pull-requests: write
|
||||
# Required by actions/download-artifact to fetch artifacts produced
|
||||
# by a different workflow run.
|
||||
actions: read
|
||||
# Required to create the `gitnexus/autofix` Check Run that reports
|
||||
# the outcome (clean / suggestions-posted / skipped-too-large) to
|
||||
# the PR's Checks tab. Branch protection or agents can grep the
|
||||
# conclusion + output title without parsing the sticky comment.
|
||||
checks: write
|
||||
steps:
|
||||
# Pinned to v8.0.1. Verify SHA via:
|
||||
# gh api repos/actions/download-artifact/git/refs/tags/v8.0.1
|
||||
- name: Download autofix artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: autofix
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
path: autofix-in
|
||||
|
||||
- name: Read and validate metadata
|
||||
id: meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -f autofix-in/metadata.json
|
||||
jq . autofix-in/metadata.json
|
||||
|
||||
# The artifact comes from the untrusted half running fork code.
|
||||
# Every field is allowlist-validated before it can flow into
|
||||
# $GITHUB_OUTPUT. A newline in head_ref would otherwise let a
|
||||
# malicious branch name inject a second `pr_number=N` line and
|
||||
# redirect this job's reviewdog suggestions / sticky summary
|
||||
# comment onto a victim PR under github-actions[bot] with
|
||||
# pull-requests: write.
|
||||
assert_field() {
|
||||
local key="$1" pattern="$2" value
|
||||
value=$(jq -r ".${key} // empty" autofix-in/metadata.json)
|
||||
if [ -z "$value" ] || ! [[ "$value" =~ $pattern ]]; then
|
||||
echo "::error::metadata.${key} failed allowlist (got: $(printf '%q' "$value"))"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
SCHEMA=$(assert_field schema '^gitnexus\.pr-autofix/v[0-9]+$')
|
||||
PR_NUMBER=$(assert_field pr_number '^[0-9]+$')
|
||||
HEAD_SHA=$(assert_field head_sha '^[0-9a-f]{40}$')
|
||||
HEAD_REF=$(assert_field head_ref '^[A-Za-z0-9._/-]+$')
|
||||
HEAD_REPO=$(assert_field head_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$')
|
||||
BASE_REPO=$(assert_field base_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$')
|
||||
CHANGED=$(assert_field changed_lines '^[0-9]+$')
|
||||
|
||||
# Defence-in-depth: refuse to act if the artifact claims to
|
||||
# belong to a different repo than the one that triggered us.
|
||||
if [ "$BASE_REPO" != "${GITHUB_REPOSITORY}" ]; then
|
||||
echo "::error::Artifact base_repo does not match \$GITHUB_REPOSITORY — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{
|
||||
echo "schema=${SCHEMA}"
|
||||
echo "pr_number=${PR_NUMBER}"
|
||||
echo "head_sha=${HEAD_SHA}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "head_repo=${HEAD_REPO}"
|
||||
echo "base_repo=${BASE_REPO}"
|
||||
echo "changed_lines=${CHANGED}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Pinned to v1.5.0. Verify SHA via:
|
||||
# gh api repos/reviewdog/action-setup/git/refs/tags/v1.5.0
|
||||
# (annotated tag — resolve via .../git/tags/<sha> --jq .object)
|
||||
- name: Install reviewdog
|
||||
if: steps.meta.outputs.changed_lines != '0'
|
||||
uses: reviewdog/action-setup@d8a7baabd7f3e8544ee4dbde3ee41d0011c3a93f # v1.5.0
|
||||
with:
|
||||
# Pin the binary, not just the action SHA — a bad reviewdog
|
||||
# release otherwise breaks every PR with no rollback. Bump
|
||||
# this knob deliberately when validating a new release.
|
||||
reviewdog_version: v0.21.0
|
||||
|
||||
- name: Post inline suggestions
|
||||
id: suggest
|
||||
if: steps.meta.outputs.changed_lines != '0'
|
||||
env:
|
||||
REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
CI_REPO_OWNER: ${{ github.repository_owner }}
|
||||
CI_REPO_NAME: ${{ github.event.repository.name }}
|
||||
CI_PULL_REQUEST: ${{ steps.meta.outputs.pr_number }}
|
||||
CI_COMMIT: ${{ steps.meta.outputs.head_sha }}
|
||||
PR: ${{ steps.meta.outputs.pr_number }}
|
||||
# Pull `changed_lines` through env so bash gets a real
|
||||
# variable (and shellcheck SC2170 doesn't fire on `-gt` against
|
||||
# a `${{ }}`-interpolated literal).
|
||||
CHANGED_LINES: ${{ steps.meta.outputs.changed_lines }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
patch=autofix-in/autofix.patch
|
||||
if [ ! -s "$patch" ]; then
|
||||
echo "Empty patch — nothing to suggest."
|
||||
echo "posted=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# GitHub's review-comment API returns 406 on diffs above ~3k
|
||||
# changed lines. Bail out gracefully and let the summary
|
||||
# comment carry the signal instead.
|
||||
if [ "$CHANGED_LINES" -gt 3000 ]; then
|
||||
echo "Diff too large ($CHANGED_LINES lines) — skipping inline suggestions."
|
||||
echo "posted=skipped-too-large" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Snapshot the count of bot-authored review comments BEFORE
|
||||
# reviewdog. Reviewdog with `-filter-mode=added` silently posts
|
||||
# nothing when the patch's lines don't overlap the PR's added
|
||||
# range (typical case: formatter touched code that wasn't part
|
||||
# of this PR). It exits 0 either way, so the only reliable
|
||||
# signal is "did new review comments actually show up?".
|
||||
before=$(gh api "repos/${GH_REPO}/pulls/${PR}/comments" \
|
||||
--paginate --jq '[.[] | select(.user.login == "github-actions[bot]")] | length')
|
||||
|
||||
# `-f.diff.strip=1` matches `git diff` output (a/foo b/foo).
|
||||
# `-filter-mode=added` only suggests on lines the PR added,
|
||||
# which avoids re-suggesting on already-resolved threads on
|
||||
# subsequent pushes.
|
||||
reviewdog \
|
||||
-f=diff -f.diff.strip=1 \
|
||||
-name="prettier+eslint" \
|
||||
-reporter=github-pr-review \
|
||||
-filter-mode=added \
|
||||
-level=warning \
|
||||
-fail-on-error=false < "$patch"
|
||||
|
||||
# Verify reviewdog actually posted suggestions. Without this
|
||||
# check, a "0 review comments" run still emits a sticky claiming
|
||||
# "Click Apply suggestion" — confusing because there's nothing
|
||||
# to click.
|
||||
after=$(gh api "repos/${GH_REPO}/pulls/${PR}/comments" \
|
||||
--paginate --jq '[.[] | select(.user.login == "github-actions[bot]")] | length')
|
||||
delta=$((after - before))
|
||||
echo "Bot review comments: before=$before after=$after delta=$delta"
|
||||
|
||||
if [ "$delta" -gt 0 ]; then
|
||||
echo "posted=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
# Patch had fixes but reviewdog couldn't surface them as
|
||||
# suggestions — almost always because the formatter touched
|
||||
# lines outside the PR's added range, which `-filter-mode=added`
|
||||
# correctly filters out. Sticky tells the user to apply locally.
|
||||
echo "posted=no-overlap" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Upsert sticky summary comment
|
||||
# Only post when ci-quality found something fixable (= the
|
||||
# autofix patch is non-empty). When prettier/eslint are clean
|
||||
# the patch is zero bytes and the sticky comment is pure noise,
|
||||
# so we skip it. When the diff was too large for inline
|
||||
# suggestions, the sticky is the only signal the contributor
|
||||
# gets, so we still post in that case.
|
||||
if: >-
|
||||
always()
|
||||
&& steps.meta.outputs.pr_number != ''
|
||||
&& steps.meta.outputs.changed_lines != '0'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR: ${{ steps.meta.outputs.pr_number }}
|
||||
CHANGED: ${{ steps.meta.outputs.changed_lines }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
SCHEMA: ${{ steps.meta.outputs.schema }}
|
||||
POSTED: ${{ steps.suggest.outputs.posted }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Stable heading + marker — agents grep for these exact strings.
|
||||
marker="<!-- gitnexus:pr-autofix-summary -->"
|
||||
heading="## :sparkles: PR Autofix"
|
||||
|
||||
if [ "${POSTED}" = "skipped-too-large" ]; then
|
||||
ui_state="skipped-too-large"
|
||||
prose="Diff is **${CHANGED}** lines — too large for inline suggestions (GitHub caps the review-comment API at ~3000). Run locally: \`npm run lint:fix && npm run format\`."
|
||||
elif [ "${POSTED}" = "no-overlap" ]; then
|
||||
# Reviewdog ran but couldn't post any inline suggestions —
|
||||
# the formatter touched lines outside this PR's added range,
|
||||
# which \`-filter-mode=added\` correctly filters out. There's
|
||||
# nothing for **Apply suggestion** to click; the user has to
|
||||
# apply locally.
|
||||
ui_state="diff-no-overlap"
|
||||
prose="Formatter found fixable issues, but they're on lines outside this PR's added range — there's nothing to click here. Run locally: \`npm run lint:fix && npm run format\`."
|
||||
else
|
||||
ui_state="suggestions-posted"
|
||||
prose="Posted formatting / unused-import suggestions inline. Click **Apply suggestion** on each, or run locally: \`npm run lint:fix && npm run format\`."
|
||||
fi
|
||||
|
||||
# Machine-readable JSON block — agents parse this instead of
|
||||
# regexing English. Fenced code-block info string is
|
||||
# `gitnexus-autofix` so agents can locate it without ambiguity.
|
||||
json=$(jq -n -c \
|
||||
--arg schema "${SCHEMA}" \
|
||||
--arg state "${ui_state}" \
|
||||
--argjson pr_number "${PR}" \
|
||||
--argjson changed_lines "${CHANGED}" \
|
||||
--arg head_sha "${HEAD_SHA}" \
|
||||
--arg run_id "${RUN_ID}" \
|
||||
'{schema:$schema, state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id}')
|
||||
|
||||
# Multi-line quoted string instead of a column-0 heredoc — YAML's
|
||||
# `run: |` block ends as soon as a content line dedents below the
|
||||
# block's first-line indent, which would mis-parse the workflow.
|
||||
body="${marker}
|
||||
${heading}
|
||||
|
||||
${prose}
|
||||
|
||||
\`\`\`gitnexus-autofix
|
||||
${json}
|
||||
\`\`\`"
|
||||
# Strip the leading 10-space indent that the YAML block requires
|
||||
# so the rendered comment body starts at column 0.
|
||||
body="$(printf '%s\n' "$body" | sed 's/^ //')"
|
||||
|
||||
# Small retry wrapper for transient 5xx / rate-limit responses
|
||||
# on the GitHub REST API. Three tries with linear backoff. We
|
||||
# only retry GET (idempotent) and PATCH on a known comment id
|
||||
# (idempotent). POST is NOT wrapped — retrying a comment-create
|
||||
# would create duplicates if the first attempt actually landed.
|
||||
gh_retry() {
|
||||
local n=0 max=3
|
||||
while true; do
|
||||
if gh "$@"; then return 0; fi
|
||||
n=$((n+1))
|
||||
if [ "$n" -ge "$max" ]; then return 1; fi
|
||||
sleep $((n * 2))
|
||||
done
|
||||
}
|
||||
|
||||
# Find existing bot comment by the marker and edit-in-place; else create.
|
||||
# CRITICAL: filter by `.user.login == "github-actions[bot]"`. A regular
|
||||
# user posting a comment containing the marker would otherwise be the
|
||||
# `head -n1` match; PATCH on someone else's comment 403s, `set -e`
|
||||
# aborts, and the bot is permanently DoS'd for that PR.
|
||||
existing=$(gh_retry api "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
--paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"${marker}\"))) | .id" \
|
||||
| head -n1 || true)
|
||||
|
||||
if [ -n "${existing}" ]; then
|
||||
gh_retry api -X PATCH "repos/${GH_REPO}/issues/comments/${existing}" \
|
||||
-f body="${body}" >/dev/null
|
||||
echo "Updated comment ${existing}."
|
||||
else
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="${body}" >/dev/null
|
||||
echo "Created summary comment."
|
||||
fi
|
||||
|
||||
- name: Emit gitnexus/autofix Check Run
|
||||
# Stable check name `gitnexus/autofix` so PR-watching agents can
|
||||
# `gh pr checks <pr>` and read the conclusion + title without
|
||||
# parsing the sticky comment. Three outcomes:
|
||||
# clean → conclusion: success
|
||||
# suggestions-posted → conclusion: neutral (review suggestions)
|
||||
# skipped-too-large → conclusion: neutral (diff > 3000 lines)
|
||||
# `neutral` does not block branch-protection required-checks but
|
||||
# is visually distinct from a green pass.
|
||||
if: always() && steps.meta.outputs.head_sha != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
CHANGED: ${{ steps.meta.outputs.changed_lines }}
|
||||
POSTED: ${{ steps.suggest.outputs.posted }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${CHANGED}" = "0" ]; then
|
||||
conclusion="success"
|
||||
title="Formatting clean"
|
||||
summary="Prettier and ESLint --fix produced no changes."
|
||||
elif [ "${POSTED}" = "skipped-too-large" ]; then
|
||||
conclusion="neutral"
|
||||
title="Diff too large for inline suggestions (${CHANGED} lines)"
|
||||
summary="GitHub caps the review-comment API at ~3000 lines. Run \`npm run lint:fix && npm run format\` locally."
|
||||
elif [ "${POSTED}" = "no-overlap" ]; then
|
||||
conclusion="neutral"
|
||||
title="Formatter changes don't overlap PR's added range"
|
||||
summary="Reviewdog couldn't post inline suggestions because the formatter touched lines outside this PR's added range. Run \`npm run lint:fix && npm run format\` locally."
|
||||
else
|
||||
conclusion="neutral"
|
||||
title="Suggestions posted"
|
||||
summary="Inline review-comment suggestions posted. Click **Apply suggestion** on each, or run \`npm run lint:fix && npm run format\` locally."
|
||||
fi
|
||||
|
||||
gh api -X POST "repos/${GH_REPO}/check-runs" \
|
||||
-f name="gitnexus/autofix" \
|
||||
-f head_sha="${HEAD_SHA}" \
|
||||
-f status="completed" \
|
||||
-f conclusion="${conclusion}" \
|
||||
-f "output[title]=${title}" \
|
||||
-f "output[summary]=${summary}" \
|
||||
>/dev/null
|
||||
echo "Posted check-run gitnexus/autofix=${conclusion} (${title})"
|
||||
@@ -0,0 +1,146 @@
|
||||
name: PR Autofix
|
||||
|
||||
# UNTRUSTED HALF of the autofix pipeline.
|
||||
#
|
||||
# Runs `npm run lint:fix` + `npm run format` against the PR head
|
||||
# (including fork heads) and uploads the resulting diff as an artifact.
|
||||
# This job has NO privileged token and CANNOT post to the PR. The trusted
|
||||
# `pr-autofix-publish.yml` workflow downloads the artifact via
|
||||
# `workflow_run` and posts the inline review-comment suggestions.
|
||||
#
|
||||
# Why the split:
|
||||
# ESLint loads plugins from fork-controlled `node_modules`, so running
|
||||
# it in a job with `pull-requests: write` would let a malicious fork PR
|
||||
# ship a poisoned eslint plugin and execute arbitrary code under that
|
||||
# token. By keeping fork code execution in this job (token: read-only)
|
||||
# and posting from a separate trusted job that never touches fork
|
||||
# code, we get the inline-suggestion UX for fork PRs without the
|
||||
# supply-chain hole. (See autofix.ci for the same pattern.)
|
||||
#
|
||||
# Removes unused imports via `eslint-plugin-unused-imports`, already in
|
||||
# devDependencies and wired into the `lint` config.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
# Skip lockfile / generated-file PRs entirely — `action-suggester`
|
||||
# cannot post on diffs > ~3k lines (GitHub returns 406) and these
|
||||
# paths produce massive diffs no human wants suggested back inline.
|
||||
paths-ignore:
|
||||
- '**/package-lock.json'
|
||||
- '**/*.snap'
|
||||
- '**/dist/**'
|
||||
- '**/node_modules/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
# Don't cancel in-flight runs; the publish workflow may already be
|
||||
# downloading the artifact and a cancelled untrusted run produces no
|
||||
# signal at all (worse DX than waiting).
|
||||
cancel-in-progress: false
|
||||
|
||||
# This workflow runs untrusted fork code. Top-level deny-all and NO
|
||||
# job-level grants — the job can only read its own checkout.
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
autofix:
|
||||
name: autofix
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# PR head commit (not the synthetic merge ref) — we need the
|
||||
# exact tree the contributor pushed so suggestions line up.
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
# `--ignore-scripts` blocks pre/postinstall lifecycle hooks. ESLint
|
||||
# plugins still load from node_modules (that is the actual escape
|
||||
# hatch on a typical fork), but this job has no token to abuse —
|
||||
# which is the whole point of the split.
|
||||
- run: npm ci --ignore-scripts
|
||||
|
||||
- name: ESLint --fix (removes unused imports)
|
||||
run: npm run lint:fix
|
||||
# Lint errors that --fix can't auto-resolve must not block the
|
||||
# diff artifact — partial fixes are still useful as suggestions.
|
||||
continue-on-error: true
|
||||
|
||||
- name: Prettier --write
|
||||
run: npm run format
|
||||
continue-on-error: true
|
||||
|
||||
- name: Capture diff and metadata
|
||||
id: capture
|
||||
# Pass GitHub-context values via env: rather than `${{ }}`
|
||||
# interpolated directly into the bash body. `head.ref` and
|
||||
# `head.repo.full_name` are fork-controlled strings; expanding
|
||||
# them into shell source is the canonical template-injection
|
||||
# vector zizmor flags. Even though this job has `permissions: {}`,
|
||||
# routing through env: makes it impossible for a future scope
|
||||
# grant to turn into RCE. Inside bash, reference as `$HEAD_REF`
|
||||
# etc. — the values are then plain strings, not code.
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
BASE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p autofix-out
|
||||
|
||||
# Produce a unified diff of the working tree vs. the PR head.
|
||||
# Empty diff => nothing to suggest; the publish job short-circuits.
|
||||
git diff --no-color > autofix-out/autofix.patch
|
||||
|
||||
# NOTE: `changed_lines` is the line-count of the patch file,
|
||||
# which includes hunk headers and context lines — NOT the
|
||||
# added/removed source-line count. The 3000-line cap in
|
||||
# pr-autofix-publish.yml is therefore conservative (fires
|
||||
# before reviewdog hits GitHub's ~3k review-comment API
|
||||
# ceiling). That bias is intentional.
|
||||
changed_lines=$(wc -l < autofix-out/autofix.patch | tr -d ' ')
|
||||
echo "changed_lines=${changed_lines}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Carry PR identity over to the trusted job. workflow_run
|
||||
# context is base-repo-only, so the publish job needs these
|
||||
# to call the GitHub PR API on the right resource.
|
||||
# CONTRACT: keep this schema in sync with pr-autofix-publish.yml's
|
||||
# `assert_field` validators and the agent-facing JSON block in
|
||||
# the sticky comment. Bump `schema` when changing field names.
|
||||
jq -n \
|
||||
--arg schema 'gitnexus.pr-autofix/v1' \
|
||||
--argjson pr_number "${PR_NUMBER}" \
|
||||
--arg head_sha "${HEAD_SHA}" \
|
||||
--arg head_ref "${HEAD_REF}" \
|
||||
--arg head_repo "${HEAD_REPO}" \
|
||||
--arg base_repo "${BASE_REPO}" \
|
||||
--argjson changed_lines "${changed_lines}" \
|
||||
'{schema:$schema, pr_number:$pr_number, head_sha:$head_sha, head_ref:$head_ref, head_repo:$head_repo, base_repo:$base_repo, changed_lines:$changed_lines}' \
|
||||
> autofix-out/metadata.json
|
||||
|
||||
echo "--- metadata ---"
|
||||
cat autofix-out/metadata.json
|
||||
echo "--- diff (head) ---"
|
||||
head -c 2000 autofix-out/autofix.patch || true
|
||||
|
||||
# Pinned to v7.0.1. Verify SHA via:
|
||||
# gh api repos/actions/upload-artifact/git/refs/tags/v7.0.1
|
||||
- name: Upload autofix artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: autofix
|
||||
path: autofix-out/
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
@@ -123,7 +123,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: write # push rc tag + marker
|
||||
# The default GITHUB_TOKEN cannot be granted `workflows: write`, so
|
||||
# tag pushes that reach a commit which modified `.github/workflows/**`
|
||||
# are rejected with: "refusing to allow a GitHub App to create or
|
||||
# update workflow ... without `workflows` permission". We pass a
|
||||
# fine-grained PAT (RELEASE_PUSH_TOKEN, scoped to this repo with
|
||||
# Contents: write + Workflows: write) to `actions/checkout` so that
|
||||
# the subsequent `git push --atomic` of the v-tag and rc marker
|
||||
# carries the PAT's identity. Job-level GITHUB_TOKEN keeps its
|
||||
# scoped permissions for everything else (npm provenance, etc.).
|
||||
contents: write # push rc tag + marker (via PAT)
|
||||
id-token: write # npm provenance
|
||||
outputs:
|
||||
vtag: ${{ steps.reltag.outputs.vtag }}
|
||||
@@ -132,6 +141,11 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
# Use the PAT so `origin` is preauthed for `git push`. Without
|
||||
# this the default GITHUB_TOKEN is wired into the remote, and a
|
||||
# workflows-touching tag push is rejected — see the permissions
|
||||
# block above.
|
||||
token: ${{ secrets.RELEASE_PUSH_TOKEN }}
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
@@ -280,9 +294,11 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "base=$BASE" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "base=$BASE"
|
||||
echo "rc_n=$NEXT_N"
|
||||
echo "rc_version=$RC_VERSION"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply rc version in-CI
|
||||
shell: bash
|
||||
@@ -340,9 +356,11 @@ jobs:
|
||||
# remote ref, the push fails and we stop before npm publish.
|
||||
git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER"
|
||||
|
||||
echo "vtag=$VTAG" >> "$GITHUB_OUTPUT"
|
||||
echo "marker=$MARKER" >> "$GITHUB_OUTPUT"
|
||||
echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "vtag=$VTAG"
|
||||
echo "marker=$MARKER"
|
||||
echo "release_sha=$RELEASE_SHA"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Publish to npm (rc dist-tag)
|
||||
run: npm publish --provenance --access public --tag rc
|
||||
|
||||
@@ -53,6 +53,6 @@ jobs:
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload to Security tab
|
||||
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
|
||||
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build image (load locally for scan)
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.image.dockerfile }}
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload to Security tab
|
||||
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
|
||||
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
|
||||
with:
|
||||
sarif_file: trivy-${{ matrix.image.name }}.sarif
|
||||
category: trivy-${{ matrix.image.name }}
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
name: Workflow Lint (zizmor)
|
||||
name: Workflow Lint
|
||||
|
||||
# Lints .github/workflows/** for known GitHub Actions security misconfigurations:
|
||||
# unpinned Actions, dangerous ${{ ... }} interpolation in run: blocks,
|
||||
# missing per-job permissions:, etc.
|
||||
# Lints .github/workflows/** for both:
|
||||
# - actionlint: YAML syntax, expression typing, shellcheck inside `run:`
|
||||
# blocks, unknown contexts, deprecated runner labels.
|
||||
# - zizmor: security misconfigurations — unpinned actions, dangerous
|
||||
# `${{ }}` interpolation, missing per-job permissions, etc.
|
||||
#
|
||||
# Scoped to PRs that touch .github/** only — keeps off the typical PR critical path.
|
||||
# Scoped to PRs that touch .github/** only — keeps off the typical PR
|
||||
# critical path.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -17,6 +20,27 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
actionlint:
|
||||
name: actionlint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Pinned to v2.1.2. Verify SHA via:
|
||||
# gh api repos/raven-actions/actionlint/git/refs/tags/v2.1.2
|
||||
# The action wraps the upstream `rhysd/actionlint` binary and emits
|
||||
# GitHub-annotation-formatted findings on PRs.
|
||||
- name: Run actionlint
|
||||
uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2
|
||||
with:
|
||||
fail-on-error: true
|
||||
|
||||
zizmor:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
@@ -49,7 +73,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload SARIF
|
||||
uses: github/codeql-action/upload-sarif@0daab03d71ff584ef619d027a3fd9146679c5d84 # v3.35.3
|
||||
uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
|
||||
with:
|
||||
sarif_file: zizmor.sarif
|
||||
category: zizmor
|
||||
|
||||
@@ -14,6 +14,15 @@ rules:
|
||||
# no checkout of fork code occurs. Header comment in the file documents.
|
||||
- ci-report.yml
|
||||
|
||||
# workflow_run is the trusted half of the autofix pipeline. The
|
||||
# untrusted half (pr-autofix.yml) runs fork code with permissions:{}
|
||||
# and produces only a diff artifact (data, not executable code). The
|
||||
# publish job consumes the artifact, allowlist-validates every field
|
||||
# of metadata.json before exporting to $GITHUB_OUTPUT, never checks
|
||||
# out fork code, and never executes anything fork-controlled. Header
|
||||
# comment in the file documents the split.
|
||||
- pr-autofix-publish.yml
|
||||
|
||||
# pull_request_target needed by claude-code-action to access secrets
|
||||
# and post review comments on fork PRs. Mitigated by: PR checkouts pin
|
||||
# the fork's HEAD SHA (not the branch ref) to prevent TOCTOU races,
|
||||
|
||||
@@ -103,6 +103,24 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:
|
||||
|
||||
- When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape.
|
||||
|
||||
## CI automation contracts
|
||||
|
||||
Two workflows produce machine-readable signals on every PR. Coding agents and humans alike can rely on the names and shapes below — change them with intent.
|
||||
|
||||
### `gitnexus/autofix`
|
||||
|
||||
`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface the diff as inline review-comment suggestions. Three signals are emitted:
|
||||
|
||||
| Surface | Where | Notes |
|
||||
|---|---|---|
|
||||
| Sticky PR comment | Top-level comment with the HTML marker `<!-- gitnexus:pr-autofix-summary -->` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. |
|
||||
| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v1` with fields `state` (`suggestions-posted` \| `skipped-too-large`), `pr_number`, `head_sha`, `changed_lines`, `run_id`. | Parseable signal — preferred over regexing prose. |
|
||||
| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (suggestions-posted / skipped-too-large). The output title disambiguates the two `neutral` cases. | Surfaced under PR Checks; readable via `gh pr checks <pr>`. |
|
||||
|
||||
To detect outcome from an agent: `gh pr checks <pr> --json name,conclusion,output | jq '.[] | select(.name == "gitnexus/autofix")'`.
|
||||
|
||||
Forks are supported. The untrusted half runs fork code with `permissions: {}` and ships the diff as an artifact; the trusted publish job consumes only the diff (data, not code) and posts the comment + check run.
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes.
|
||||
|
||||
@@ -109,6 +109,8 @@ That's it. This indexes the codebase, installs agent skills, registers Claude Co
|
||||
|
||||
To configure MCP for your editor, run `npx gitnexus setup` once — or set it up manually below.
|
||||
|
||||
> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the native `tree-sitter-dart` and `tree-sitter-proto` builds. Dart/Proto files won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild.
|
||||
|
||||
### MCP Setup
|
||||
|
||||
`gitnexus setup` auto-detects your editors and writes the correct global MCP config. You only need to run it once.
|
||||
@@ -138,6 +140,8 @@ Built by the community — not officially maintained, but worth checking out.
|
||||
|
||||
If you prefer manual configuration:
|
||||
|
||||
> **Recommended for fastest startup:** install gitnexus globally (`npm i -g gitnexus`) and run `gitnexus setup` — this writes an absolute-path MCP config that bypasses `npx` entirely. The pinned-`npx` snippets below are a quickstart fallback; on a cold cache the `npx` install can exceed Claude Code's `MCP_TIMEOUT` default (~30s).
|
||||
|
||||
**Claude Code** (full support — MCP + skills + hooks):
|
||||
|
||||
```bash
|
||||
@@ -210,6 +214,7 @@ gitnexus clean --all --force # Delete all indexes
|
||||
gitnexus wiki [path] # Generate repository wiki from knowledge graph
|
||||
gitnexus wiki --model <model> # Wiki with custom LLM model (default: gpt-4o-mini)
|
||||
gitnexus wiki --base-url <url> # Wiki with custom LLM API base URL
|
||||
gitnexus publish # Notify the understand-quickly registry (opt-in, see below)
|
||||
|
||||
# Repository groups (multi-repo / monorepo service tracking)
|
||||
gitnexus group create <name> # Create a repository group
|
||||
@@ -224,6 +229,12 @@ gitnexus group status <name> # Check staleness of repos in a group
|
||||
|
||||
If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget.
|
||||
|
||||
#### Publishing to understand-quickly (opt-in)
|
||||
|
||||
[`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) is a public registry of code-knowledge graphs that lists `gitnexus@1` as a first-class format. After registering your repo once (`npx @understand-quickly/cli add` or the [wizard](https://looptech-ai.github.io/understand-quickly/add.html)), `gitnexus publish` fires a single `repository_dispatch` event so the registry resyncs your entry on demand instead of waiting for the nightly job.
|
||||
|
||||
It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained GitHub PAT with `Repository dispatches: write` on the registry repo. Nothing else happens; no graph file is uploaded. See the [protocol spec](https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md) for the full contract.
|
||||
|
||||
### What Your AI Agent Gets
|
||||
|
||||
**16 tools** exposed via MCP (11 per-repo + 5 group):
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ services:
|
||||
- ${WORKSPACE_DIR:-./workspace}:/workspace:ro
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ['CMD', 'curl', '-fsSI', 'http://localhost:4747/api/heartbeat']
|
||||
test: ['CMD', 'curl', '-f', 'http://localhost:4747/api/health']
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
# Using GitNexus across Apache Thrift microservices
|
||||
|
||||
## When to use this guide
|
||||
|
||||
Use this guide when several repositories communicate through Apache Thrift and you want GitNexus to trace impact across provider and consumer boundaries. The walkthrough assumes each service is indexed on its own, then joined through a GitNexus group.
|
||||
|
||||
This is not a framework integration guide. GitNexus reads portable Thrift IDL and common Java generated-code shapes. Framework-specific wiring, service discovery, deployment metadata, and private annotations belong outside the open-source core.
|
||||
|
||||
## Mental model
|
||||
|
||||
- `.thrift` files define the canonical service contract. A method in an IDL service becomes a stable contract id in the form `thrift::<namespace>.<Service>/<Method>`.
|
||||
- Service wildcard ids in the form `thrift::<namespace>.<Service>/*` are supported as manifest and matching fallback forms when a service-level link is needed.
|
||||
- Java generated-code usage points GitNexus toward implementation and call sites. Providers commonly implement generated `Service.Iface`; consumers commonly hold or construct generated service interfaces or clients.
|
||||
- Group sync matches provider and consumer contracts with the same id, then cross-repo impact can hop through those links.
|
||||
- Framework-specific wiring should be modeled by extractor plugins, manifest links, or downstream integrations rather than hard-coded into core Thrift support.
|
||||
|
||||
## Fictional IDL
|
||||
|
||||
```thrift
|
||||
namespace java billing.v1
|
||||
|
||||
struct PlaceOrderRequest {
|
||||
1: string orderId
|
||||
2: double amount
|
||||
}
|
||||
|
||||
struct PlaceOrderResponse {
|
||||
1: bool accepted
|
||||
}
|
||||
|
||||
struct GetOrderRequest {
|
||||
1: string orderId
|
||||
}
|
||||
|
||||
struct GetOrderResponse {
|
||||
1: string orderId
|
||||
2: string status
|
||||
}
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
GetOrderResponse GetOrder(1: GetOrderRequest request)
|
||||
}
|
||||
```
|
||||
|
||||
The service methods above produce canonical ids:
|
||||
|
||||
- `thrift::billing.v1.OrderService/PlaceOrder`
|
||||
- `thrift::billing.v1.OrderService/GetOrder`
|
||||
- `thrift::billing.v1.OrderService/*` as a service-level manifest or matching fallback form
|
||||
|
||||
## Java provider example
|
||||
|
||||
Generated Java code usually exposes an `Iface` interface for the service. A provider implementation can be detected when it implements that generated interface.
|
||||
|
||||
```java
|
||||
package example.billing;
|
||||
|
||||
import billing.v1.GetOrderRequest;
|
||||
import billing.v1.GetOrderResponse;
|
||||
import billing.v1.OrderService;
|
||||
import billing.v1.PlaceOrderRequest;
|
||||
import billing.v1.PlaceOrderResponse;
|
||||
|
||||
public final class OrderServiceHandler implements OrderService.Iface {
|
||||
@Override
|
||||
public PlaceOrderResponse PlaceOrder(PlaceOrderRequest request) {
|
||||
return new PlaceOrderResponse(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetOrderResponse GetOrder(GetOrderRequest request) {
|
||||
return new GetOrderResponse(request.getOrderId(), "CREATED");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
With the IDL available, GitNexus can connect the implementation to `thrift::billing.v1.OrderService/PlaceOrder` and `thrift::billing.v1.OrderService/GetOrder`.
|
||||
|
||||
## Java consumer examples
|
||||
|
||||
Consumers are strongest when Java usage can be tied back to the IDL namespace and service.
|
||||
|
||||
```java
|
||||
package example.checkout;
|
||||
|
||||
import billing.v1.OrderService;
|
||||
import billing.v1.PlaceOrderRequest;
|
||||
|
||||
public final class CheckoutWorkflow {
|
||||
private final OrderService.Iface orders;
|
||||
|
||||
public CheckoutWorkflow(OrderService.Iface orders) {
|
||||
this.orders = orders;
|
||||
}
|
||||
|
||||
public void submit(String orderId) throws Exception {
|
||||
orders.PlaceOrder(new PlaceOrderRequest(orderId, 42.0));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Some generated-code styles use the generated service type directly while keeping enough IDL context through imports and method calls.
|
||||
|
||||
```java
|
||||
package example.reporting;
|
||||
|
||||
import billing.v1.GetOrderRequest;
|
||||
import billing.v1.OrderService;
|
||||
|
||||
public final class OrderLookup {
|
||||
private final OrderService.Client client;
|
||||
|
||||
public OrderLookup(OrderService.Client client) {
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
public String status(String orderId) throws Exception {
|
||||
return client.GetOrder(new GetOrderRequest(orderId)).getStatus();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
When IDL context is missing, GitNexus may still emit a weaker consumer signal for generated `Iface` or `Client` shapes, but confidence is lower.
|
||||
|
||||
## Group configuration
|
||||
|
||||
New group configs enable Thrift contract detection by default. Keep `detect.thrift: true`
|
||||
when a group should scan for Thrift contracts, or set it to `false` to skip Thrift
|
||||
extraction for that group.
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
name: billing-platform
|
||||
description: Fictional services connected by Apache Thrift
|
||||
|
||||
repos:
|
||||
checkout: checkout-service
|
||||
billing: billing-service
|
||||
|
||||
links: []
|
||||
|
||||
detect:
|
||||
http: true
|
||||
grpc: false
|
||||
thrift: true
|
||||
topics: false
|
||||
shared_libs: true
|
||||
```
|
||||
|
||||
To disable Thrift extraction explicitly:
|
||||
|
||||
```yaml
|
||||
detect:
|
||||
thrift: false
|
||||
```
|
||||
|
||||
After indexing each member repository, run group sync to extract contracts and write cross-repo links:
|
||||
|
||||
```bash
|
||||
npx gitnexus group sync billing-platform
|
||||
```
|
||||
|
||||
## Manifest escape hatch
|
||||
|
||||
Use manifest links when automatic extraction cannot see a provider or consumer, or when generated code is wrapped behind an abstraction. Write the contract without the `thrift::` prefix; GitNexus canonicalizes it to the full Thrift contract id.
|
||||
|
||||
```yaml
|
||||
links:
|
||||
- from: checkout
|
||||
to: billing
|
||||
type: thrift
|
||||
contract: billing.v1.OrderService/PlaceOrder
|
||||
role: consumer
|
||||
```
|
||||
|
||||
GitNexus canonicalizes that manifest entry to `thrift::billing.v1.OrderService/PlaceOrder` and uses it to connect the two repositories.
|
||||
|
||||
## Known limitations
|
||||
|
||||
- Java detection currently targets v1 generated-code patterns.
|
||||
- Maven and POM dependency coordinates are not used for inference.
|
||||
- Framework-specific annotations and service discovery metadata are ignored by open-source Thrift extraction.
|
||||
- Ambiguous same-name services are skipped instead of guessed.
|
||||
- Java consumers without IDL context are lower confidence and limited to generated `Iface` and `Client` shapes.
|
||||
+98
-2
@@ -4,6 +4,38 @@ import unusedImports from 'eslint-plugin-unused-imports';
|
||||
import reactHooks from 'eslint-plugin-react-hooks';
|
||||
import prettierConfig from 'eslint-config-prettier';
|
||||
|
||||
// Selectors that protect MCP-reachable code from corrupting the JSON-RPC
|
||||
// stdio frame stream. The MCP-reachable block below uses these directly;
|
||||
// the lbug-adapter file-specific block must spread them in too because
|
||||
// ESLint flat config REPLACES (not merges) `no-restricted-syntax` when
|
||||
// multiple matching configs target the same file. Extracting to a const
|
||||
// makes the dependency mechanical instead of documentation-enforced.
|
||||
const mcpStdoutWriteSelectors = [
|
||||
{
|
||||
selector:
|
||||
"MemberExpression[object.type='MemberExpression'][object.object.name='process'][object.property.name='stdout'][property.name='write']",
|
||||
message:
|
||||
'Direct process.stdout.write is forbidden in MCP-reachable code. Route diagnostics through console.error or process.stderr.write — the MCP stdio transport owns stdout for JSON-RPC frames.',
|
||||
},
|
||||
{
|
||||
selector:
|
||||
"CallExpression[callee.type='MemberExpression'][callee.object.type='MemberExpression'][callee.object.object.name='process'][callee.object.property.name='stdout'][callee.property.name='write']",
|
||||
message:
|
||||
'Direct process.stdout.write is forbidden in MCP-reachable code. Route diagnostics through console.error or process.stderr.write — the MCP stdio transport owns stdout for JSON-RPC frames.',
|
||||
},
|
||||
{
|
||||
// Catches the canonical destructuring shape:
|
||||
// const { write } = process.stdout;
|
||||
// (and any other ObjectPattern destructure rooted at process.stdout)
|
||||
// which would otherwise capture a reference to the original write
|
||||
// and bypass the sentinel.
|
||||
selector:
|
||||
"VariableDeclarator[init.type='MemberExpression'][init.object.name='process'][init.property.name='stdout'] > ObjectPattern",
|
||||
message:
|
||||
'Destructuring process.stdout is forbidden in MCP-reachable code — bypasses the sentinel. Use process.stderr.write for diagnostics.',
|
||||
},
|
||||
];
|
||||
|
||||
export default [
|
||||
// Global ignores
|
||||
{
|
||||
@@ -59,11 +91,47 @@ export default [
|
||||
},
|
||||
},
|
||||
|
||||
// CLI package — allow console.log (it's a CLI tool)
|
||||
// CLI/server packages — `console.log` IS the contract (CLI tool data output
|
||||
// on stdout, e.g. `gitnexus query | jq`; server pretty-printed banners).
|
||||
// Diagnostic logging (`warn`/`error`/`debug`/`info`) goes through pino like
|
||||
// the rest of the codebase.
|
||||
{
|
||||
files: ['gitnexus/src/cli/**/*.ts', 'gitnexus/src/server/**/*.ts'],
|
||||
rules: {
|
||||
'no-console': 'off',
|
||||
'no-console': ['error', { allow: ['log'] }],
|
||||
},
|
||||
},
|
||||
|
||||
// Forcing function for the pino migration. Severity is `error` — the
|
||||
// codebase-wide migration is complete; new `console.*` in core source
|
||||
// must fail lint. CLI/server are exempt above (legitimate stdout output).
|
||||
// Tests, bin scripts, and the logger module itself remain exempt.
|
||||
{
|
||||
files: ['gitnexus/src/**/*.ts'],
|
||||
ignores: ['gitnexus/src/cli/**', 'gitnexus/src/server/**', 'gitnexus/src/core/logger.ts'],
|
||||
rules: {
|
||||
'no-console': 'error',
|
||||
},
|
||||
},
|
||||
|
||||
// MCP-reachable code: forbid stdout-corrupting writes. The MCP stdio
|
||||
// transport writes JSON-RPC frames to stdout; per the spec, the server
|
||||
// MUST NOT write anything to stdout that is not a valid MCP message.
|
||||
// Diagnostics must go to stderr (console.error). Direct process.stdout.write
|
||||
// bypasses the gate and is also forbidden in these dirs.
|
||||
// cli/mcp.ts is included here even though it lives under cli/ — it is the
|
||||
// MCP entrypoint and inherits stricter discipline than the rest of cli/.
|
||||
{
|
||||
files: [
|
||||
'gitnexus/src/mcp/**/*.ts',
|
||||
'gitnexus/src/core/lbug/**/*.ts',
|
||||
'gitnexus/src/core/embeddings/**/*.ts',
|
||||
'gitnexus/src/core/tree-sitter/**/*.ts',
|
||||
'gitnexus/src/cli/mcp.ts',
|
||||
],
|
||||
rules: {
|
||||
'no-console': ['error', { allow: ['error'] }],
|
||||
'no-restricted-syntax': ['error', ...mcpStdoutWriteSelectors],
|
||||
},
|
||||
},
|
||||
|
||||
@@ -79,6 +147,34 @@ export default [
|
||||
},
|
||||
},
|
||||
|
||||
// Prevent direct conn.close() / db.close() in the LadybugDB adapter (#1376).
|
||||
// All close operations must go through safeClose() so the WAL is always
|
||||
// flushed before the connection is released. The sole authorised call site
|
||||
// inside safeClose itself uses an eslint-disable-next-line override.
|
||||
//
|
||||
// ESLint flat config REPLACES (not merges) `no-restricted-syntax` when
|
||||
// multiple matching configs target the same file. lbug-adapter.ts is also
|
||||
// covered by the MCP-reachable block above, so we spread the shared
|
||||
// mcpStdoutWriteSelectors here alongside the safeClose selectors. Without
|
||||
// this, lbug-adapter would silently lose its MCP stdout-write protection.
|
||||
{
|
||||
files: ['gitnexus/src/core/lbug/lbug-adapter.ts'],
|
||||
rules: {
|
||||
'no-restricted-syntax': [
|
||||
'error',
|
||||
...mcpStdoutWriteSelectors,
|
||||
{
|
||||
selector: "CallExpression[callee.object.name='conn'][callee.property.name='close']",
|
||||
message: 'Use safeClose() instead of calling conn.close() directly (#1376).',
|
||||
},
|
||||
{
|
||||
selector: "CallExpression[callee.object.name='db'][callee.property.name='close']",
|
||||
message: 'Use safeClose() instead of calling db.close() directly (#1376).',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
// Disable formatting rules (prettier handles those)
|
||||
prettierConfig,
|
||||
];
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"./test-helpers": {
|
||||
"types": "./dist/test-helpers.d.ts",
|
||||
"default": "./dist/test-helpers.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@@ -143,6 +143,34 @@ export type { ScopeTree } from './scope-resolution/scope-tree.js';
|
||||
export { buildPositionIndex } from './scope-resolution/position-index.js';
|
||||
export type { PositionIndex } from './scope-resolution/position-index.js';
|
||||
|
||||
// Resilient fetch primitives — bounded retries + per-process circuit breaker.
|
||||
// Test-only helpers (`__resetBreakerRegistry__`, `classifyOutcome`) are
|
||||
// reachable via the separate `gitnexus-shared/test-helpers` subpath; do
|
||||
// NOT add them here. Production consumers must not call them.
|
||||
export { withRetry, computeBackoffMs } from './integrations/retry.js';
|
||||
export type { RetryOptions, RetryDecision } from './integrations/retry.js';
|
||||
export { CircuitBreaker, CircuitOpenError, getBreaker } from './integrations/circuit-breaker.js';
|
||||
export type { CircuitBreakerOptions } from './integrations/circuit-breaker.js';
|
||||
export {
|
||||
resilientFetch,
|
||||
ResilientFetchExhaustedError,
|
||||
RETRY_AFTER_CAP_MS,
|
||||
parseRetryAfter,
|
||||
} from './integrations/resilient-fetch.js';
|
||||
export type { ResilientFetchOptions } from './integrations/resilient-fetch.js';
|
||||
|
||||
// Understand-Quickly registry integration (opt-in)
|
||||
export {
|
||||
UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
UNDERSTAND_QUICKLY_EVENT_TYPE,
|
||||
UNDERSTAND_QUICKLY_TOKEN_ENV,
|
||||
buildUqDispatchPayload,
|
||||
isValidOwnerRepo,
|
||||
parseOwnerRepoFromRemote,
|
||||
stripGitSuffix,
|
||||
} from './integrations/understand-quickly.js';
|
||||
export type { UqDispatchPayload } from './integrations/understand-quickly.js';
|
||||
|
||||
// Shadow-mode diff + aggregation (RFC §6.3; Ring 2 SHARED #918)
|
||||
export { diffResolutions } from './scope-resolution/shadow/diff.js';
|
||||
export type {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* Per-process circuit breaker.
|
||||
*
|
||||
* Closed -> Open transition fires after `failureThreshold` consecutive
|
||||
* failures. While Open, `check` throws `CircuitOpenError` until
|
||||
* `cooldownMs` has elapsed since the breaker tripped. The first call
|
||||
* after the cooldown enters Half-Open and consumes the *probe permit*:
|
||||
* a recorded success returns to Closed; a recorded failure flips back
|
||||
* to Open with a fresh timestamp.
|
||||
*
|
||||
* Half-open admits exactly one in-flight probe at a time. Concurrent
|
||||
* callers attempting `check()` while a probe is outstanding receive
|
||||
* `CircuitOpenError` with `retryAfterMs = halfOpenRetryAfterMs` (default
|
||||
* 1000ms; configurable). This prevents the recovery-time thundering
|
||||
* herd that defeats the breaker's "fail fast" promise.
|
||||
*
|
||||
* Outcome reporting splits permit-release from state-resolution:
|
||||
* - `recordSuccess` — releases the probe permit, resets the failure
|
||||
* counter, transitions to Closed. Reserved for true 2xx/3xx outcomes.
|
||||
* - `recordFailure` — releases the probe permit, increments the
|
||||
* consecutive-failure counter, transitions to Open with a fresh
|
||||
* `openedAt` (when called from Half-Open or when the threshold
|
||||
* trips from Closed).
|
||||
* - `recordNeutral` — releases the probe permit, BUT leaves state and
|
||||
* counter untouched. Used for outcomes that are neither evidence of
|
||||
* backend health nor evidence of backend failure (caller-driven
|
||||
* cancellation, local timeout, terminal 4xx client errors). Critical
|
||||
* design point: if `recordNeutral` did not release the permit, a
|
||||
* single `TimeoutError` from per-attempt `AbortSignal.timeout` would
|
||||
* route through `recordNeutral` and permanently park the breaker in
|
||||
* half-open until process restart. Releasing the permit while leaving
|
||||
* state half-open keeps the "neutral doesn't claim health" semantic
|
||||
* without creating that wedge.
|
||||
*
|
||||
* Pairing invariant: every successful `check()` MUST be paired with
|
||||
* exactly one `record*()` on every code path including throws. Direct
|
||||
* consumers should wrap the protected operation in `try/finally`:
|
||||
*
|
||||
* breaker.check();
|
||||
* try {
|
||||
* const result = await operation();
|
||||
* breaker.recordSuccess();
|
||||
* return result;
|
||||
* } catch (err) {
|
||||
* // classify err and call recordFailure / recordNeutral / etc.
|
||||
* throw err;
|
||||
* }
|
||||
*
|
||||
* `resilientFetch`'s catch-all on `fetchImpl` already satisfies this
|
||||
* for that consumer.
|
||||
*
|
||||
* Atomicity model: the half-open gate relies on JavaScript event-loop
|
||||
* single-threadedness within a synchronous `check()` body. There is no
|
||||
* `await` inside `check()`; concurrent callers serialize on microtask
|
||||
* order, and exactly one observes `probeInFlight === false`. Do not
|
||||
* introduce `await` inside `check()` without revisiting the gate. If
|
||||
* this code is ever ported to a runtime with shared-memory threads
|
||||
* (Node `worker_threads` with `SharedArrayBuffer`, Web Workers with
|
||||
* shared registries), the boolean must become an atomic CAS — Resilience4j
|
||||
* and Hystrix use atomic permits *because* they run in JVM thread pools.
|
||||
*
|
||||
* Runtime-agnostic: depends only on a `now()` clock and standard JS —
|
||||
* no Node-only imports. Tests inject `now` to advance the clock
|
||||
* deterministically without `vi.useFakeTimers()`.
|
||||
*/
|
||||
|
||||
export class CircuitOpenError extends Error {
|
||||
override readonly name = 'CircuitOpenError';
|
||||
/** Approximate wait time before the breaker may transition to Half-Open
|
||||
* (or before the in-flight probe is expected to resolve). */
|
||||
readonly retryAfterMs: number;
|
||||
|
||||
constructor(retryAfterMs: number, key?: string) {
|
||||
super(
|
||||
key
|
||||
? `Circuit '${key}' is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`
|
||||
: `Circuit is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`,
|
||||
);
|
||||
this.retryAfterMs = retryAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CircuitBreakerOptions {
|
||||
/** Consecutive failures required to trip Closed -> Open. */
|
||||
failureThreshold?: number;
|
||||
/** Milliseconds Open before the next call may probe (Half-Open). */
|
||||
cooldownMs?: number;
|
||||
/**
|
||||
* Milliseconds to suggest in `CircuitOpenError.retryAfterMs` when the
|
||||
* breaker is Half-Open with the probe permit consumed. Default 1000ms.
|
||||
* Consumers with long-running protected ops (LLM streaming, large
|
||||
* uploads) should raise this — the cooldown clock is no longer the
|
||||
* right answer because cooldown has elapsed. Returning 0 invites
|
||||
* retry storms; returning the full cooldown misleads about wait.
|
||||
*/
|
||||
halfOpenRetryAfterMs?: number;
|
||||
/** Optional key for error messages and registry lookups. */
|
||||
key?: string;
|
||||
/** Clock override — defaults to `Date.now`. Tests inject deterministic time. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
type State = 'closed' | 'open' | 'half-open';
|
||||
|
||||
export class CircuitBreaker {
|
||||
private readonly failureThreshold: number;
|
||||
private readonly cooldownMs: number;
|
||||
private readonly halfOpenRetryAfterMs: number;
|
||||
private readonly key: string | undefined;
|
||||
private readonly now: () => number;
|
||||
|
||||
private state: State = 'closed';
|
||||
private consecutiveFailures = 0;
|
||||
private openedAt: number | null = null;
|
||||
/**
|
||||
* True between a successful `check()` and the next `record*()` call
|
||||
* during Half-Open. Gates concurrent callers from stampeding a still-
|
||||
* recovering dependency. Boolean rather than counter — single-permit
|
||||
* is the conservative end of the Hystrix/Resilience4j spectrum.
|
||||
*/
|
||||
private probeInFlight = false;
|
||||
|
||||
constructor(opts: CircuitBreakerOptions = {}) {
|
||||
this.failureThreshold = opts.failureThreshold ?? 3;
|
||||
this.cooldownMs = opts.cooldownMs ?? 30_000;
|
||||
this.halfOpenRetryAfterMs = opts.halfOpenRetryAfterMs ?? 1_000;
|
||||
this.key = opts.key;
|
||||
this.now = opts.now ?? (() => Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw `CircuitOpenError` if the breaker won't admit this call.
|
||||
* Otherwise consume the half-open probe permit (if applicable) and
|
||||
* return so the caller can attempt the protected work.
|
||||
*
|
||||
* Three rejection paths:
|
||||
* 1. Open and still in cooldown → throws with `retryAfterMs` =
|
||||
* remaining cooldown.
|
||||
* 2. Open with cooldown elapsed AND a probe is already in flight
|
||||
* (race: another caller transitioned to half-open and grabbed
|
||||
* the permit on a microtask before us) → throws with
|
||||
* `halfOpenRetryAfterMs`.
|
||||
* 3. Half-Open with probe in flight → throws with `halfOpenRetryAfterMs`.
|
||||
*
|
||||
* **Pairing invariant**: every successful return from `check()` MUST
|
||||
* be paired with exactly one `recordSuccess` / `recordFailure` /
|
||||
* `recordNeutral` on every code path including thrown exceptions.
|
||||
* Failing to pair leaves the probe permit consumed forever and
|
||||
* wedges the breaker. See file-header JSDoc for the canonical
|
||||
* try/finally pattern.
|
||||
*/
|
||||
check(): void {
|
||||
if (this.state === 'open' && this.openedAt !== null) {
|
||||
const elapsed = this.now() - this.openedAt;
|
||||
if (elapsed < this.cooldownMs) {
|
||||
throw new CircuitOpenError(this.cooldownMs - elapsed, this.key);
|
||||
}
|
||||
// Cooldown elapsed — transition to Half-Open. The very next
|
||||
// `probeInFlight` check below decides whether THIS caller gets
|
||||
// the permit or hits the gate.
|
||||
this.state = 'half-open';
|
||||
}
|
||||
|
||||
if (this.state === 'half-open') {
|
||||
if (this.probeInFlight) {
|
||||
throw new CircuitOpenError(this.halfOpenRetryAfterMs, this.key);
|
||||
}
|
||||
this.probeInFlight = true;
|
||||
}
|
||||
// Closed state falls through silently.
|
||||
}
|
||||
|
||||
recordSuccess(): void {
|
||||
this.probeInFlight = false;
|
||||
this.consecutiveFailures = 0;
|
||||
this.state = 'closed';
|
||||
this.openedAt = null;
|
||||
}
|
||||
|
||||
recordFailure(): void {
|
||||
this.probeInFlight = false;
|
||||
this.consecutiveFailures += 1;
|
||||
if (this.state === 'half-open' || this.consecutiveFailures >= this.failureThreshold) {
|
||||
this.state = 'open';
|
||||
this.openedAt = this.now();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the probe permit BUT leaves state and counter untouched.
|
||||
* Use when an attempt produced a response or error that should not
|
||||
* influence breaker health in either direction — caller-driven aborts,
|
||||
* local AbortSignal timeouts, terminal 4xx client errors.
|
||||
*
|
||||
* Why permit-release-without-state-resolution: if `recordNeutral` did
|
||||
* not clear `probeInFlight`, a single `TimeoutError` from per-attempt
|
||||
* `AbortSignal.timeout` (which routes through neutral classification)
|
||||
* would permanently park the breaker in half-open. Since timeouts are
|
||||
* an *expected* outcome under flaky-dependency conditions, the cited
|
||||
* "per-attempt timeout bounds the stuck state" mitigation would itself
|
||||
* be the trigger for a permanent wedge. Releasing the permit closes
|
||||
* that loop while keeping the "neutral doesn't claim dependency
|
||||
* health" semantic.
|
||||
*
|
||||
* Calling `recordSuccess` for these would erase legitimate prior
|
||||
* failure signal; calling `recordFailure` would trip the breaker for
|
||||
* outcomes the backend isn't responsible for.
|
||||
*/
|
||||
recordNeutral(): void {
|
||||
this.probeInFlight = false;
|
||||
// State and consecutiveFailures are preserved by design.
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure read — no state mutation, no permit accounting. Returns the
|
||||
* *would-be* state at the current instant: 'half-open' if the breaker
|
||||
* is open with cooldown elapsed (regardless of whether a probe is in
|
||||
* flight), 'open' if open and still in cooldown, 'closed' otherwise.
|
||||
*
|
||||
* Inspection-only; safe to call from tests without consuming a probe
|
||||
* permit. The implicit Open -> Half-Open transition that mutates
|
||||
* `state` lives in `check()` only.
|
||||
*/
|
||||
getState(): State {
|
||||
if (this.state === 'open' && this.openedAt !== null) {
|
||||
const elapsed = this.now() - this.openedAt;
|
||||
if (elapsed >= this.cooldownMs) return 'half-open';
|
||||
}
|
||||
return this.state;
|
||||
}
|
||||
getConsecutiveFailures(): number {
|
||||
return this.consecutiveFailures;
|
||||
}
|
||||
/** Inspection-only test accessor for the half-open probe permit. */
|
||||
isProbeInFlight(): boolean {
|
||||
return this.probeInFlight;
|
||||
}
|
||||
/** Timestamp (ms since epoch) when the breaker last transitioned to Open,
|
||||
* or `null` if it's currently Closed. Useful for computing remaining
|
||||
* cooldown without consuming a probe permit via `check()`. */
|
||||
getOpenedAt(): number | null {
|
||||
return this.openedAt;
|
||||
}
|
||||
/** Configured cooldown duration in milliseconds. */
|
||||
getCooldownMs(): number {
|
||||
return this.cooldownMs;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Per-process registry ────────────────────────────────────────────
|
||||
//
|
||||
// Single shared map keyed on caller-chosen strings. Used by
|
||||
// `resilient-fetch.ts` so multiple call sites targeting the same logical
|
||||
// endpoint share breaker state. Per-process only — not persisted.
|
||||
|
||||
const registry = new Map<string, CircuitBreaker>();
|
||||
|
||||
export function getBreaker(key: string, opts?: CircuitBreakerOptions): CircuitBreaker {
|
||||
let breaker = registry.get(key);
|
||||
if (!breaker) {
|
||||
breaker = new CircuitBreaker({ ...opts, key });
|
||||
registry.set(key, breaker);
|
||||
}
|
||||
return breaker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only: clear all registered breakers. Tests must call this in
|
||||
* `beforeEach` to prevent breaker state from leaking across test cases.
|
||||
*/
|
||||
export function __resetBreakerRegistry__(): void {
|
||||
registry.clear();
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* `resilientFetch` — fetch wrapped in retry + circuit breaker, with
|
||||
* GitHub-flavoured retry classification baked in (Retry-After parsing,
|
||||
* 401/403/404/422 treated as terminal client errors).
|
||||
*
|
||||
* Designed for the `gitnexus publish` GitHub `repository_dispatch`
|
||||
* call, but the classification rules apply to any GitHub REST endpoint.
|
||||
* Runtime-agnostic — no Node-only imports.
|
||||
*/
|
||||
|
||||
import {
|
||||
CircuitBreaker,
|
||||
CircuitOpenError,
|
||||
getBreaker,
|
||||
type CircuitBreakerOptions,
|
||||
} from './circuit-breaker.js';
|
||||
import { computeBackoffMs, type RetryOptions } from './retry.js';
|
||||
|
||||
export { CircuitOpenError };
|
||||
|
||||
export interface ResilientFetchOptions {
|
||||
/** Optional fetch implementation override. Defaults to `globalThis.fetch`. */
|
||||
fetchImpl?: typeof fetch;
|
||||
/**
|
||||
* Logical key for the breaker. Defaults to `<host><pathname>` of the
|
||||
* request URL — call sites targeting the same endpoint share breaker
|
||||
* state regardless of query-string differences.
|
||||
*/
|
||||
breakerKey?: string;
|
||||
/** Per-call breaker override. Used for tests and one-off configuration. */
|
||||
breaker?: CircuitBreaker;
|
||||
/** Tuning knobs for the breaker registered under `breakerKey`. */
|
||||
breakerOptions?: CircuitBreakerOptions;
|
||||
/** Tuning knobs for the retry helper. */
|
||||
retry?: Partial<Pick<RetryOptions, 'maxAttempts' | 'baseDelayMs' | 'capDelayMs'>> & {
|
||||
sleep?: RetryOptions['sleep'];
|
||||
random?: RetryOptions['random'];
|
||||
};
|
||||
/** Clock override propagated into Retry-After HTTP-date math and breaker. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Cap on any single Retry-After wait — protects CLI from a buggy registry. */
|
||||
export const RETRY_AFTER_CAP_MS = 30_000;
|
||||
|
||||
const DEFAULT_RETRY = {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
capDelayMs: 5_000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a `Retry-After` header value into milliseconds.
|
||||
* Accepts either a delta-seconds integer (`"30"`) or an HTTP-date.
|
||||
* Returns null on parse failure or negative deltas.
|
||||
*/
|
||||
export function parseRetryAfter(value: string | null, now: () => number = Date.now): number | null {
|
||||
if (!value) return null;
|
||||
const trimmed = value.trim();
|
||||
if (trimmed === '') return null;
|
||||
|
||||
if (/^[0-9]+$/.test(trimmed)) {
|
||||
const seconds = parseInt(trimmed, 10);
|
||||
if (Number.isNaN(seconds) || seconds < 0) return null;
|
||||
return seconds * 1000;
|
||||
}
|
||||
|
||||
const target = Date.parse(trimmed);
|
||||
if (Number.isNaN(target)) return null;
|
||||
const delta = target - now();
|
||||
return delta >= 0 ? delta : 0;
|
||||
}
|
||||
|
||||
/** Internal: outcome classification used by the resilientFetch loop. */
|
||||
type Outcome =
|
||||
| { kind: 'success'; resp: Response }
|
||||
| { kind: 'terminal-client'; resp: Response } // 4xx other than 429: no retry, breaker neutral
|
||||
| { kind: 'retryable-status'; resp: Response; afterMs: number | undefined } // 5xx, 429
|
||||
| { kind: 'terminal-network'; err: unknown } // TimeoutError or AbortError: no retry, breaker neutral
|
||||
| { kind: 'retryable-network'; err: unknown }; // DNS, ECONNRESET, etc.
|
||||
|
||||
/** Exported for unit tests. */
|
||||
export function classifyOutcome(
|
||||
result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response },
|
||||
now: () => number,
|
||||
): Outcome {
|
||||
if (result.kind === 'error') {
|
||||
// Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`)
|
||||
// and caller-driven aborts (`AbortController.abort()` → `AbortError`)
|
||||
// are terminal: retrying against an already-aborted signal would
|
||||
// fail again immediately, and neither outcome reflects backend
|
||||
// health. They route through the breaker's neutral path.
|
||||
if (
|
||||
result.err instanceof DOMException &&
|
||||
(result.err.name === 'TimeoutError' || result.err.name === 'AbortError')
|
||||
) {
|
||||
return { kind: 'terminal-network', err: result.err };
|
||||
}
|
||||
return { kind: 'retryable-network', err: result.err };
|
||||
}
|
||||
const resp = result.resp;
|
||||
if (resp.status >= 200 && resp.status < 400) return { kind: 'success', resp };
|
||||
if (resp.status === 429) {
|
||||
// `resp.headers` is always present on a real `Response`, but tests
|
||||
// sometimes stub `fetch` with a plain `{ ok, status }` object. Be
|
||||
// defensive — a missing `Retry-After` falls through to exponential
|
||||
// backoff, which is the correct behaviour anyway.
|
||||
const retryAfterHeader =
|
||||
typeof resp.headers?.get === 'function' ? resp.headers.get('Retry-After') : null;
|
||||
const parsed = parseRetryAfter(retryAfterHeader, now);
|
||||
return {
|
||||
kind: 'retryable-status',
|
||||
resp,
|
||||
afterMs: parsed !== null ? Math.min(parsed, RETRY_AFTER_CAP_MS) : undefined,
|
||||
};
|
||||
}
|
||||
if (resp.status >= 500) return { kind: 'retryable-status', resp, afterMs: undefined };
|
||||
return { kind: 'terminal-client', resp };
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function defaultBreakerKey(input: string | URL): string {
|
||||
try {
|
||||
const url = typeof input === 'string' ? new URL(input) : input;
|
||||
return `${url.host}${url.pathname}`;
|
||||
} catch {
|
||||
return String(input);
|
||||
}
|
||||
}
|
||||
|
||||
/** Final error thrown when retries are exhausted on a 5xx / 429. */
|
||||
export class ResilientFetchExhaustedError extends Error {
|
||||
override readonly name = 'ResilientFetchExhaustedError';
|
||||
constructor(public readonly response: Response) {
|
||||
super(`Request failed after retries (HTTP ${response.status})`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap `fetch` with bounded retries and a per-process circuit breaker.
|
||||
*
|
||||
* Semantics:
|
||||
* - 5xx and 429 responses are retried; 429 honors `Retry-After` (capped).
|
||||
* - Network throws are retried unless they are `TimeoutError` DOMExceptions.
|
||||
* - Timeouts and 4xx (other than 429) are returned/thrown without retry
|
||||
* AND without incrementing the breaker — they reflect caller config
|
||||
* or local network state, not registry health.
|
||||
* - Each `fetch` call carries the caller-supplied `signal` (e.g. an
|
||||
* `AbortSignal.timeout()`) — that timeout bounds each individual
|
||||
* attempt, not the whole retry sequence.
|
||||
* - When the breaker is open, throws `CircuitOpenError` synchronously
|
||||
* without invoking `fetch`.
|
||||
* - When retries are exhausted on a 5xx / 429, throws
|
||||
* `ResilientFetchExhaustedError` carrying the last response.
|
||||
*
|
||||
* Cumulative wall-clock budget:
|
||||
* maxAttempts × (per-attempt-timeout + capDelayMs)
|
||||
* With defaults (3, 500ms base, 5000ms cap) and a typical 15s per-attempt
|
||||
* timeout from the caller's signal, worst case is ~3 × (15s + 5s) = 60s.
|
||||
* Callers that want a tighter total bound should reduce `maxAttempts` or
|
||||
* wrap `resilientFetch` in their own outer `AbortSignal.timeout()`.
|
||||
*/
|
||||
export async function resilientFetch(
|
||||
input: string | URL,
|
||||
init: RequestInit | undefined,
|
||||
opts: ResilientFetchOptions = {},
|
||||
): Promise<Response> {
|
||||
const fetchImpl = opts.fetchImpl ?? globalThis.fetch;
|
||||
const now = opts.now ?? (() => Date.now());
|
||||
const breaker =
|
||||
opts.breaker ?? getBreaker(opts.breakerKey ?? defaultBreakerKey(input), opts.breakerOptions);
|
||||
|
||||
const retryConfig = {
|
||||
maxAttempts: opts.retry?.maxAttempts ?? DEFAULT_RETRY.maxAttempts,
|
||||
baseDelayMs: opts.retry?.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs,
|
||||
capDelayMs: opts.retry?.capDelayMs ?? DEFAULT_RETRY.capDelayMs,
|
||||
};
|
||||
const sleep = opts.retry?.sleep ?? defaultSleep;
|
||||
const random = opts.retry?.random ?? Math.random;
|
||||
|
||||
// Fail fast on an open breaker, before invoking fetch.
|
||||
breaker.check();
|
||||
|
||||
for (let attempt = 0; attempt < retryConfig.maxAttempts; attempt++) {
|
||||
let result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response };
|
||||
try {
|
||||
// CodeQL js/server-side-request-forgery — flagged because `input`
|
||||
// is caller-supplied. Suppressed: every concrete caller passes
|
||||
// either a hardcoded URL constant (UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
// OpenRouter base URL) or a value derived from configuration
|
||||
// (env vars, saved settings, the local backend URL). User-input
|
||||
// request fields (e.g. PR title, repo name) never flow into
|
||||
// `input`. Validating URL shape here would push false-positive
|
||||
// rejection onto every caller — wrong layer for the check.
|
||||
// lgtm[js/server-side-request-forgery]
|
||||
// codeql[js/server-side-request-forgery]
|
||||
const resp = await fetchImpl(input, init);
|
||||
result = { kind: 'response', resp };
|
||||
} catch (err) {
|
||||
result = { kind: 'error', err };
|
||||
}
|
||||
|
||||
const outcome = classifyOutcome(result, now);
|
||||
|
||||
switch (outcome.kind) {
|
||||
case 'success':
|
||||
breaker.recordSuccess();
|
||||
return outcome.resp;
|
||||
|
||||
case 'terminal-client':
|
||||
// 4xx: do not count as breaker failure (the server is healthy
|
||||
// and rejecting our request — auth, scope, or routing). But
|
||||
// also do NOT call recordSuccess: a 401 sandwiched between
|
||||
// 5xx responses would otherwise erase the running outage
|
||||
// signal. The breaker's neutral path leaves state untouched.
|
||||
breaker.recordNeutral();
|
||||
return outcome.resp;
|
||||
|
||||
case 'terminal-network':
|
||||
// Either `AbortSignal.timeout()` fired locally OR an external
|
||||
// caller cancelled the request via AbortController. The server
|
||||
// never had a chance to answer; this reflects the user's
|
||||
// network or an explicit cancel, not registry health. Don't
|
||||
// punish the breaker AND don't reset its outage signal.
|
||||
breaker.recordNeutral();
|
||||
throw outcome.err;
|
||||
|
||||
case 'retryable-status':
|
||||
if (attempt + 1 >= retryConfig.maxAttempts) {
|
||||
breaker.recordFailure();
|
||||
throw new ResilientFetchExhaustedError(outcome.resp);
|
||||
}
|
||||
await sleep(
|
||||
computeBackoffMs(
|
||||
attempt,
|
||||
retryConfig.baseDelayMs,
|
||||
retryConfig.capDelayMs,
|
||||
outcome.afterMs,
|
||||
random,
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
case 'retryable-network':
|
||||
if (attempt + 1 >= retryConfig.maxAttempts) {
|
||||
breaker.recordFailure();
|
||||
throw outcome.err;
|
||||
}
|
||||
await sleep(
|
||||
computeBackoffMs(
|
||||
attempt,
|
||||
retryConfig.baseDelayMs,
|
||||
retryConfig.capDelayMs,
|
||||
undefined,
|
||||
random,
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
default: {
|
||||
// Exhaustiveness guard. If a sixth `Outcome` kind is added in
|
||||
// future, TypeScript will refuse to assign it to `never` and
|
||||
// this line forces the maintainer to add an explicit arm
|
||||
// rather than silently fall through to retry/no-retry behaviour.
|
||||
const _exhaustive: never = outcome;
|
||||
throw new Error(`resilientFetch: unhandled outcome ${JSON.stringify(_exhaustive)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable: every iteration of the loop either returns (success
|
||||
// / terminal-client) or throws (terminal-network / retry exhaustion).
|
||||
// The throw is here purely so TypeScript's control-flow analysis sees
|
||||
// the function never falls off the end without producing `Promise<Response>`.
|
||||
/* c8 ignore next 2 */
|
||||
throw new Error('resilientFetch: retry loop terminated unexpectedly');
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Bounded retry helper with full-jitter exponential backoff.
|
||||
*
|
||||
* Runtime-agnostic: depends only on `setTimeout`, `Math.random`, and the
|
||||
* Promise machinery — no Node-only imports. Safe to consume from CLI,
|
||||
* server, or browser callers.
|
||||
*
|
||||
* Pattern reference: gitnexus/src/core/embeddings/http-client.ts. This
|
||||
* helper is the upgraded form: classification is caller-supplied (so
|
||||
* 4xx-vs-5xx-vs-timeout decisions live with the protocol that knows
|
||||
* them), backoff is exponential with full jitter, and an optional
|
||||
* `afterMs` lets callers honor `Retry-After` headers.
|
||||
*/
|
||||
|
||||
export interface RetryOptions {
|
||||
/** Initial delay before the first retry attempt, in milliseconds. */
|
||||
baseDelayMs: number;
|
||||
/** Upper bound on any single delay, in milliseconds. */
|
||||
capDelayMs: number;
|
||||
/** Total attempts including the first call. Must be >= 1. */
|
||||
maxAttempts: number;
|
||||
/**
|
||||
* Decide whether to retry after a thrown error.
|
||||
* Return `{retry:false}` to terminate immediately and rethrow.
|
||||
* Return `{retry:true}` to retry with exponential-backoff jitter.
|
||||
* Return `{retry:true, afterMs}` to wait at least `afterMs` (still
|
||||
* subject to `capDelayMs`) — used by callers parsing `Retry-After`.
|
||||
*/
|
||||
isRetryable: (err: unknown, attempt: number) => RetryDecision;
|
||||
/** Sleep override — defaults to `setTimeout`. Tests inject fake timers. */
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
/** Random override — defaults to `Math.random`. Tests inject seeded values. */
|
||||
random?: () => number;
|
||||
}
|
||||
|
||||
export type RetryDecision = { retry: false } | { retry: true; afterMs?: number };
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Compute the delay before the next retry attempt.
|
||||
*
|
||||
* - When the caller specifies `afterMs` (e.g., from `Retry-After`), use
|
||||
* `min(afterMs, capDelayMs)` so a misbehaving server can't pin the
|
||||
* client for an arbitrarily long wait.
|
||||
* - Otherwise compute full-jitter exponential backoff:
|
||||
* `random() * min(cap, base * 2^attempt)`. Full jitter (rather than
|
||||
* "equal jitter") avoids retry-storm thundering herd, per AWS
|
||||
* guidance on backoff strategies.
|
||||
*/
|
||||
export function computeBackoffMs(
|
||||
attempt: number,
|
||||
baseDelayMs: number,
|
||||
capDelayMs: number,
|
||||
afterMs: number | undefined,
|
||||
random: () => number,
|
||||
): number {
|
||||
if (afterMs !== undefined) {
|
||||
return Math.min(Math.max(0, afterMs), capDelayMs);
|
||||
}
|
||||
const exponential = baseDelayMs * Math.pow(2, attempt);
|
||||
const upper = Math.min(capDelayMs, exponential);
|
||||
return Math.floor(random() * upper);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute `fn` with bounded retries.
|
||||
*
|
||||
* The classification of "retryable" is the caller's responsibility — see
|
||||
* `resilient-fetch.ts` for the GitHub-dispatch-specific rules. This
|
||||
* helper is the mechanical retry loop only.
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
fn: (attempt: number) => Promise<T>,
|
||||
opts: RetryOptions,
|
||||
): Promise<T> {
|
||||
if (opts.maxAttempts < 1) {
|
||||
throw new Error(`withRetry: maxAttempts must be >= 1, got ${opts.maxAttempts}`);
|
||||
}
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
const random = opts.random ?? Math.random;
|
||||
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < opts.maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fn(attempt);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
const decision = opts.isRetryable(err, attempt);
|
||||
if (!decision.retry) throw err;
|
||||
// Don't sleep after the final attempt.
|
||||
if (attempt + 1 >= opts.maxAttempts) break;
|
||||
const delayMs = computeBackoffMs(
|
||||
attempt,
|
||||
opts.baseDelayMs,
|
||||
opts.capDelayMs,
|
||||
decision.afterMs,
|
||||
random,
|
||||
);
|
||||
if (delayMs > 0) await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Understand-Quickly registry integration helpers.
|
||||
*
|
||||
* Pure, runtime-agnostic logic for opting in to publishing a GitNexus
|
||||
* index to the [`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly)
|
||||
* registry. Lives in `gitnexus-shared` so both the Node CLI and any
|
||||
* future browser-side surface can construct identical dispatch payloads.
|
||||
*
|
||||
* Network I/O lives in the CLI command (`gitnexus/src/cli/publish.ts`)
|
||||
* to keep this module free of Node-only imports — see the comment at
|
||||
* the top of `gitnexus-shared/src/graph/types.ts`.
|
||||
*
|
||||
* The protocol contract (single dispatch event, no graph upload) is
|
||||
* documented at:
|
||||
* https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md
|
||||
*/
|
||||
|
||||
/**
|
||||
* URL of the registry repo's repository_dispatch endpoint. Hardcoded
|
||||
* because the registry is the canonical home for this integration —
|
||||
* users who want a private registry can fork and patch.
|
||||
*/
|
||||
export const UNDERSTAND_QUICKLY_DISPATCH_URL =
|
||||
'https://api.github.com/repos/looptech-ai/understand-quickly/dispatches';
|
||||
|
||||
/**
|
||||
* Event type the registry's sync workflow listens for.
|
||||
* See `looptech-ai/understand-quickly/.github/workflows/sync.yml`.
|
||||
*/
|
||||
export const UNDERSTAND_QUICKLY_EVENT_TYPE = 'sync-entry';
|
||||
|
||||
/** Environment variable that gates the dispatch. */
|
||||
export const UNDERSTAND_QUICKLY_TOKEN_ENV = 'UNDERSTAND_QUICKLY_TOKEN';
|
||||
|
||||
export interface UqDispatchPayload {
|
||||
event_type: typeof UNDERSTAND_QUICKLY_EVENT_TYPE;
|
||||
client_payload: {
|
||||
/** `<owner>/<repo>` shape — must match the registered entry. */
|
||||
id: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the JSON body for the `repository_dispatch` ping. Pure — no
|
||||
* env reads, no network. Validates that `id` looks like `owner/repo`
|
||||
* (one slash, no whitespace, both halves non-empty) so a misconfigured
|
||||
* caller fails loudly before the round-trip.
|
||||
*/
|
||||
export function buildUqDispatchPayload(id: string): UqDispatchPayload {
|
||||
if (!isValidOwnerRepo(id)) {
|
||||
throw new Error(
|
||||
`[understand-quickly] expected id of the form "owner/repo", got "${id}". ` +
|
||||
`The registry uses this string to look up your entry in registry.json — ` +
|
||||
`it must match the GitHub owner/repo of the source code, not a local path.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
event_type: UNDERSTAND_QUICKLY_EVENT_TYPE,
|
||||
client_payload: { id },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* `owner/repo` validation. Conservative on purpose: GitHub's actual
|
||||
* naming rules are looser, but we want to catch local paths
|
||||
* (`/Users/...`), bare slugs (`my-repo`), and accidental whitespace.
|
||||
*
|
||||
* Matches GitHub's published slug rules:
|
||||
* owner: starts with alnum, then alnum/hyphen only, must end with
|
||||
* alnum (no trailing hyphen — GitHub rejects this at account
|
||||
* creation, so a `my-org-/repo` input would otherwise pass us
|
||||
* and 422 from GitHub). No underscore, no dot. Length cap 39.
|
||||
* repo: any of alnum/dot/hyphen/underscore. Length cap 100.
|
||||
*/
|
||||
export function isValidOwnerRepo(id: string): boolean {
|
||||
return /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?\/[A-Za-z0-9._-]{1,100}$/.test(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a single trailing `.git` (case-insensitive) and any trailing
|
||||
* slashes from a URL-ish string. Bounded linear: each character is
|
||||
* visited at most twice, no backtracking.
|
||||
*
|
||||
* Replaces `s.replace(/\.git\/*$/i, '').replace(/\/+$/, '')` which
|
||||
* CodeQL's polynomial-regex check (codeql/js/polynomial-redos) flags as
|
||||
* a worst-case O(n²) on adversarial input like "////.../x".
|
||||
*/
|
||||
export function stripGitSuffix(input: string): string {
|
||||
let end = input.length;
|
||||
// Trim trailing '/'.
|
||||
while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--;
|
||||
// Drop one trailing '.git' (case-insensitive).
|
||||
if (end >= 4) {
|
||||
const tail = input.slice(end - 4, end).toLowerCase();
|
||||
if (tail === '.git') end -= 4;
|
||||
}
|
||||
// Trim trailing '/' that may have sat between '.git' and the rest.
|
||||
while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--;
|
||||
return input.slice(0, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `owner/repo` out of a git remote URL. Mirrors the heuristic in
|
||||
* `gitnexus/src/storage/git.ts:parseRepoNameFromUrl` but keeps both
|
||||
* halves so we can build a registry id. Returns `null` on shapes we
|
||||
* don't recognise.
|
||||
*
|
||||
* Examples:
|
||||
* git@github.com:looptech-ai/understand-quickly.git
|
||||
* https://github.com/looptech-ai/understand-quickly
|
||||
* ssh://git@github.com/looptech-ai/understand-quickly.git
|
||||
*/
|
||||
export function parseOwnerRepoFromRemote(url: string | null | undefined): string | null {
|
||||
if (!url) return null;
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed) return null;
|
||||
// Strip a trailing `.git` (case-insensitive) and any trailing slashes
|
||||
// so https://h/o/r and https://h/o/r.git collapse to the same id.
|
||||
// Bounded-linear helper avoids the polynomial-regex CodeQL alert.
|
||||
const stripped = stripGitSuffix(trimmed);
|
||||
|
||||
// SCP-form SSH (`git@host:owner/repo`). Capture host so we can reject
|
||||
// non-GitHub remotes — a GitLab origin like
|
||||
// `https://gitlab.example.com/group/sub/project.git` would otherwise
|
||||
// silently dispatch the wrong id (LOW 9).
|
||||
const ssh = stripped.match(/^[^@]+@([^:]+):([^/]+)\/([^/]+)$/);
|
||||
if (ssh) {
|
||||
const host = ssh[1].toLowerCase();
|
||||
if (host !== 'github.com' && host !== 'www.github.com') return null;
|
||||
return `${ssh[2]}/${ssh[3]}`;
|
||||
}
|
||||
|
||||
// URL forms (https://, ssh://, git://, file://) — last two path segments.
|
||||
const url2 = stripped.match(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/([^/]+)\/(.+)$/);
|
||||
if (url2) {
|
||||
// Strip optional `userinfo@` (e.g. `ssh://git@github.com/...`).
|
||||
const authority = url2[1];
|
||||
const atIdx = authority.lastIndexOf('@');
|
||||
const hostAndPort = atIdx >= 0 ? authority.slice(atIdx + 1) : authority;
|
||||
// Strip `:port` suffix if present.
|
||||
const colonIdx = hostAndPort.indexOf(':');
|
||||
const host = (colonIdx >= 0 ? hostAndPort.slice(0, colonIdx) : hostAndPort).toLowerCase();
|
||||
if (host !== 'github.com' && host !== 'www.github.com') return null;
|
||||
const segments = url2[2].split('/').filter(Boolean);
|
||||
if (segments.length >= 2) {
|
||||
const [owner, repo] = segments.slice(-2);
|
||||
return `${owner}/${repo}`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Test-only helpers.
|
||||
*
|
||||
* Symbols here are reachable from `gitnexus-shared/test-helpers` so test
|
||||
* suites can reset shared registries or exercise internal classifiers,
|
||||
* but they are deliberately NOT re-exported from the main `gitnexus-shared`
|
||||
* barrel. Production consumers should never import this module — calling
|
||||
* `__resetBreakerRegistry__()` from a tool implementation would silently
|
||||
* nuke every circuit breaker process-wide.
|
||||
*/
|
||||
|
||||
export { __resetBreakerRegistry__ } from './integrations/circuit-breaker.js';
|
||||
export { classifyOutcome } from './integrations/resilient-fetch.js';
|
||||
Generated
+53
-53
@@ -8,15 +8,15 @@
|
||||
"name": "gitnexus",
|
||||
"version": "0.0.0",
|
||||
"dependencies": {
|
||||
"@langchain/anthropic": "^1.3.27",
|
||||
"@langchain/core": "^1.1.41",
|
||||
"@langchain/anthropic": "^1.3.29",
|
||||
"@langchain/core": "^1.1.44",
|
||||
"@langchain/google-genai": "^2.1.28",
|
||||
"@langchain/langgraph": "^1.2.9",
|
||||
"@langchain/ollama": "^1.2.6",
|
||||
"@langchain/openai": "^1.4.5",
|
||||
"@sigma/edge-curve": "^3.1.0",
|
||||
"@tailwindcss/vite": "^4.2.4",
|
||||
"axios": "^1.13.2",
|
||||
"axios": "^1.16.0",
|
||||
"d3": "^7.9.0",
|
||||
"dompurify": "^3.4.2",
|
||||
"gitnexus-shared": "file:../gitnexus-shared",
|
||||
@@ -33,7 +33,7 @@
|
||||
"mnemonist": "^0.39.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"react": "^19.2.5",
|
||||
"react-dom": "^19.2.5",
|
||||
"react-dom": "^19.2.6",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-syntax-highlighter": "^16.1.0",
|
||||
"react-zoom-pan-pinch": "^4.0.3",
|
||||
@@ -57,7 +57,7 @@
|
||||
"@vercel/node": "^5.5.16",
|
||||
"@vitejs/plugin-react": "^5.1.4",
|
||||
"@vitest/coverage-v8": "^4.1.5",
|
||||
"jsdom": "^29.0.2",
|
||||
"jsdom": "^29.1.1",
|
||||
"tree-sitter-wasms": "^0.1.13",
|
||||
"typescript": "^5.4.5",
|
||||
"vite": "^8.0.10",
|
||||
@@ -95,9 +95,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk": {
|
||||
"version": "0.90.0",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.90.0.tgz",
|
||||
"integrity": "sha512-MzZtPabJF1b0FTDl6Z6H5ljphPwACLGP13lu8MTiB8jXaW/YXlpOp+Po2cVou3MPM5+f5toyLnul9whKCy7fBg==",
|
||||
"version": "0.91.1",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz",
|
||||
"integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"json-schema-to-ts": "^3.1.1"
|
||||
@@ -132,9 +132,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/dom-selector": {
|
||||
"version": "7.0.10",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.0.10.tgz",
|
||||
"integrity": "sha512-KyOb19eytNSELkmdqzZZUXWCU25byIlOld5qVFg0RYdS0T3tt7jeDByxk9hIAC73frclD8GKrHttr0SUjKCCdQ==",
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz",
|
||||
"integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -685,9 +685,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@csstools/css-syntax-patches-for-csstree": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.1.tgz",
|
||||
"integrity": "sha512-BvqN0AMWNAnLk9G8jnUT77D+mUbY/H2b3uDTvg2isJkHaOufUE2R3AOwxWo7VBQKT1lOdwdvorddo2B/lk64+w==",
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz",
|
||||
"integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -1396,25 +1396,25 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@langchain/anthropic": {
|
||||
"version": "1.3.27",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.27.tgz",
|
||||
"integrity": "sha512-A0pWKIMIhgF01z3ILA8uAbZ6ZR2H8UQP2Ww8Ofq5DtHp36uJkQgrNCdS+q9pUVJJ87eq5dEdFkpjrjmH4fVkfQ==",
|
||||
"version": "1.3.29",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.29.tgz",
|
||||
"integrity": "sha512-ep1qBIcV07bajsg3fDqMd39rYwoRLOEK/6lk+MCxlm1YB5SRoKKJAZANrblQ/4RYhZJnxf95c6BSQu8VoNbVAQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.90.0",
|
||||
"@anthropic-ai/sdk": "^0.91.1",
|
||||
"zod": "^3.25.76 || ^4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@langchain/core": "^1.1.41"
|
||||
"@langchain/core": "^1.1.45"
|
||||
}
|
||||
},
|
||||
"node_modules/@langchain/core": {
|
||||
"version": "1.1.42",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.42.tgz",
|
||||
"integrity": "sha512-d0tN96BrwPMryYyWR9VfyAntSivn7EQrZCe5Kpxum93tcjTXbKKmKvItFec8AluQt88iTcmAJrahUZUNfzGwTA==",
|
||||
"version": "1.1.45",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.45.tgz",
|
||||
"integrity": "sha512-Y/wvuglLTMKJahkl4QD9dBIdF/z/CxZJWdTfHJF/q2jtlJtoFf6Mb5JpGxZfsi3mBY6NSG941FSLTcqhCKrhBA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@cfworker/json-schema": "^4.0.2",
|
||||
@@ -3401,12 +3401,12 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.15.0.tgz",
|
||||
"integrity": "sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==",
|
||||
"version": "1.16.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz",
|
||||
"integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.11",
|
||||
"follow-redirects": "^1.16.0",
|
||||
"form-data": "^4.0.5",
|
||||
"proxy-from-env": "^2.1.0"
|
||||
}
|
||||
@@ -4563,13 +4563,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz",
|
||||
"integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
@@ -5478,28 +5478,28 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/jsdom": {
|
||||
"version": "29.0.2",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.2.tgz",
|
||||
"integrity": "sha512-9VnGEBosc/ZpwyOsJBCQ/3I5p7Q5ngOY14a9bf5btenAORmZfDse1ZEheMiWcJ3h81+Fv7HmJFdS0szo/waF2w==",
|
||||
"version": "29.1.1",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz",
|
||||
"integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@asamuzakjp/css-color": "^5.1.5",
|
||||
"@asamuzakjp/dom-selector": "^7.0.6",
|
||||
"@asamuzakjp/css-color": "^5.1.11",
|
||||
"@asamuzakjp/dom-selector": "^7.1.1",
|
||||
"@bramus/specificity": "^2.4.2",
|
||||
"@csstools/css-syntax-patches-for-csstree": "^1.1.1",
|
||||
"@csstools/css-syntax-patches-for-csstree": "^1.1.3",
|
||||
"@exodus/bytes": "^1.15.0",
|
||||
"css-tree": "^3.2.1",
|
||||
"data-urls": "^7.0.0",
|
||||
"decimal.js": "^10.6.0",
|
||||
"html-encoding-sniffer": "^6.0.0",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"lru-cache": "^11.2.7",
|
||||
"parse5": "^8.0.0",
|
||||
"lru-cache": "^11.3.5",
|
||||
"parse5": "^8.0.1",
|
||||
"saxes": "^6.0.0",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^6.0.1",
|
||||
"undici": "^7.24.5",
|
||||
"undici": "^7.25.0",
|
||||
"w3c-xmlserializer": "^5.0.0",
|
||||
"webidl-conversions": "^8.0.1",
|
||||
"whatwg-mimetype": "^5.0.0",
|
||||
@@ -6032,9 +6032,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "11.2.7",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz",
|
||||
"integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==",
|
||||
"version": "11.3.6",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.6.tgz",
|
||||
"integrity": "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": "20 || >=22"
|
||||
@@ -7452,13 +7452,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/parse5": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz",
|
||||
"integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==",
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",
|
||||
"integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
"entities": "^8.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
@@ -7727,24 +7727,24 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/react": {
|
||||
"version": "19.2.5",
|
||||
"resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz",
|
||||
"integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==",
|
||||
"version": "19.2.6",
|
||||
"resolved": "https://registry.npmjs.org/react/-/react-19.2.6.tgz",
|
||||
"integrity": "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-dom": {
|
||||
"version": "19.2.5",
|
||||
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.5.tgz",
|
||||
"integrity": "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag==",
|
||||
"version": "19.2.6",
|
||||
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.6.tgz",
|
||||
"integrity": "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"scheduler": "^0.27.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^19.2.5"
|
||||
"react": "^19.2.6"
|
||||
}
|
||||
},
|
||||
"node_modules/react-is": {
|
||||
|
||||
@@ -19,15 +19,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"gitnexus-shared": "file:../gitnexus-shared",
|
||||
"@langchain/anthropic": "^1.3.27",
|
||||
"@langchain/core": "^1.1.41",
|
||||
"@langchain/anthropic": "^1.3.29",
|
||||
"@langchain/core": "^1.1.44",
|
||||
"@langchain/google-genai": "^2.1.28",
|
||||
"@langchain/langgraph": "^1.2.9",
|
||||
"@langchain/ollama": "^1.2.6",
|
||||
"@langchain/openai": "^1.4.5",
|
||||
"@sigma/edge-curve": "^3.1.0",
|
||||
"@tailwindcss/vite": "^4.2.4",
|
||||
"axios": "^1.13.2",
|
||||
"axios": "^1.16.0",
|
||||
"d3": "^7.9.0",
|
||||
"dompurify": "^3.4.2",
|
||||
"graphology": "^0.26.0",
|
||||
@@ -43,7 +43,7 @@
|
||||
"mnemonist": "^0.39.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"react": "^19.2.5",
|
||||
"react-dom": "^19.2.5",
|
||||
"react-dom": "^19.2.6",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-syntax-highlighter": "^16.1.0",
|
||||
"react-zoom-pan-pinch": "^4.0.3",
|
||||
@@ -67,7 +67,7 @@
|
||||
"@vercel/node": "^5.5.16",
|
||||
"@vitejs/plugin-react": "^5.1.4",
|
||||
"@vitest/coverage-v8": "^4.1.5",
|
||||
"jsdom": "^29.0.2",
|
||||
"jsdom": "^29.1.1",
|
||||
"tree-sitter-wasms": "^0.1.13",
|
||||
"typescript": "^5.4.5",
|
||||
"vite": "^8.0.10",
|
||||
|
||||
@@ -277,8 +277,10 @@ const extractInstanceName = (endpoint: string): string => {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
const hostname = url.hostname;
|
||||
// Extract the first part before .openai.azure.com
|
||||
const match = hostname.match(/^([^.]+)\.openai\.azure\.com/);
|
||||
// Extract the first part before .openai.azure.com. The trailing `$`
|
||||
// anchor is required (CodeQL js/regex/missing-regexp-anchor): without
|
||||
// it `evil.openai.azure.com.attacker.tld` would match.
|
||||
const match = hostname.match(/^([^.]+)\.openai\.azure\.com$/);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
ProviderConfig,
|
||||
} from './types';
|
||||
import { DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OLLAMA_BASE_URL } from '../../config/ui-constants';
|
||||
import { resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
const STORAGE_KEY = 'gitnexus-llm-settings';
|
||||
|
||||
@@ -407,7 +408,10 @@ export const getAvailableModels = (provider: LLMProvider): string[] => {
|
||||
*/
|
||||
export const fetchOpenRouterModels = async (): Promise<Array<{ id: string; name: string }>> => {
|
||||
try {
|
||||
const response = await fetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`);
|
||||
const response = await resilientFetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`, undefined, {
|
||||
breakerKey: 'openrouter-models',
|
||||
retry: { maxAttempts: 2, baseDelayMs: 500, capDelayMs: 2_000 },
|
||||
});
|
||||
if (!response.ok) throw new Error('Failed to fetch models');
|
||||
const data = await response.json();
|
||||
return data.data.map((model: any) => ({
|
||||
|
||||
@@ -278,8 +278,11 @@ export const createGraphRAGTools = (backend: GraphRAGBackend) => {
|
||||
const val = row[col];
|
||||
if (val === null || val === undefined) return '';
|
||||
if (typeof val === 'object') return JSON.stringify(val);
|
||||
// Truncate long values and escape pipe characters
|
||||
const str = String(val).replace(/\|/g, '\\|');
|
||||
// Truncate long values and escape pipe characters. Escape
|
||||
// backslashes FIRST so the subsequent pipe escape isn't
|
||||
// unescaped by a trailing backslash (CodeQL
|
||||
// js/incomplete-sanitization).
|
||||
const str = String(val).replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
|
||||
return str.length > 60 ? str.slice(0, 57) + '...' : str;
|
||||
});
|
||||
return `| ${values.join(' | ')} |`;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
*/
|
||||
|
||||
import type { GraphNode, GraphRelationship } from 'gitnexus-shared';
|
||||
import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -237,29 +238,91 @@ export function normalizeServerUrl(input: string): string {
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const PROBE_TIMEOUT_MS = 2_000;
|
||||
|
||||
/** Idempotent HTTP methods. Other verbs (POST, PATCH, PUT, DELETE) get
|
||||
* a single-attempt retry budget by default to avoid duplicate side
|
||||
* effects on retry — a POST that 5xx'd may have already executed
|
||||
* server-side. Callers that have idempotency keys or otherwise know
|
||||
* their mutation is safe to retry can opt in via `forceRetry`. */
|
||||
const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
|
||||
|
||||
const fetchWithTimeout = async (
|
||||
url: string,
|
||||
init: RequestInit = {},
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
/**
|
||||
* Force a retry budget on non-idempotent methods. Default false.
|
||||
* Pass true only when the endpoint is known-idempotent (e.g. DELETE
|
||||
* of a known-deleted resource — second call is a 404 / no-op) AND
|
||||
* the duplicate-side-effect window is acceptable.
|
||||
*/
|
||||
forceRetry = false,
|
||||
): Promise<Response> => {
|
||||
const controller = new AbortController();
|
||||
// Merge external signal if provided
|
||||
// Merge the external caller signal (if any) with an
|
||||
// `AbortSignal.timeout()` so a timer-fired abort produces a
|
||||
// `DOMException` with `name === 'TimeoutError'` — which
|
||||
// `resilientFetch` correctly classifies as terminal-network (no
|
||||
// retry, no breaker hit). A manual `AbortController.abort()` would
|
||||
// produce `name === 'AbortError'` and route through the
|
||||
// retryable-network branch, which mis-penalizes the breaker for
|
||||
// user-side network slowness.
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const externalSignal = init.signal;
|
||||
if (externalSignal) {
|
||||
externalSignal.addEventListener('abort', () => controller.abort());
|
||||
const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal;
|
||||
|
||||
const method = (init.method ?? 'GET').toUpperCase();
|
||||
const isIdempotent = IDEMPOTENT_METHODS.has(method);
|
||||
const maxAttempts = isIdempotent || forceRetry ? 2 : 1;
|
||||
|
||||
// Key the breaker by the current backend origin so switching backend
|
||||
// URLs (e.g. recovering from a flapping local server by pointing at
|
||||
// a different host) gives the new origin a fresh breaker state. A
|
||||
// single shared `'web-backend'` key would otherwise leave a user
|
||||
// locked out for the full cooldown after one bad host trips the
|
||||
// circuit. The malformed-URL fallback is defensive — `setBackendUrl`
|
||||
// normalizes input, so this branch shouldn't fire in practice.
|
||||
let breakerKey: string;
|
||||
try {
|
||||
breakerKey = `web-backend:${new URL(_backendUrl).origin}`;
|
||||
} catch {
|
||||
breakerKey = 'web-backend:invalid';
|
||||
}
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, { ...init, signal: controller.signal });
|
||||
// Bounded retries + 5xx/429 handling are delegated to resilientFetch.
|
||||
// Method-aware budget: idempotent verbs retry once on transient
|
||||
// backend failures; mutations (POST/PATCH/PUT/DELETE) default to
|
||||
// single-attempt to avoid duplicate side effects.
|
||||
const response = await resilientFetch(
|
||||
url,
|
||||
{ ...init, signal },
|
||||
{
|
||||
breakerKey,
|
||||
retry: { maxAttempts, baseDelayMs: 250, capDelayMs: 1500 },
|
||||
},
|
||||
);
|
||||
return response;
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
if (externalSignal?.aborted) {
|
||||
throw new BackendError('Request aborted', 0, 'network');
|
||||
}
|
||||
if (error instanceof CircuitOpenError) {
|
||||
throw new BackendError(
|
||||
`GitNexus backend at ${_backendUrl} is unhealthy; retry in ${Math.ceil(error.retryAfterMs / 1000)}s`,
|
||||
0,
|
||||
'network',
|
||||
);
|
||||
}
|
||||
if (error instanceof ResilientFetchExhaustedError) {
|
||||
// Fall through to caller — surface the raw response so assertOk
|
||||
// can craft the BackendError with the right code.
|
||||
return error.response;
|
||||
}
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
throw new BackendError(`Request to ${url} timed out after ${timeoutMs}ms`, 0, 'timeout');
|
||||
}
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
// External caller-driven cancellation — `timeoutSignal` would
|
||||
// have surfaced as TimeoutError above, so this branch covers
|
||||
// only the externally-aborted case.
|
||||
throw new BackendError('Request aborted', 0, 'network');
|
||||
}
|
||||
if (error instanceof TypeError) {
|
||||
throw new BackendError(
|
||||
`Network error reaching GitNexus backend at ${_backendUrl}: ${error.message}`,
|
||||
@@ -268,8 +331,6 @@ const fetchWithTimeout = async (
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Method-aware retry budget + timeout-as-TimeoutError verification for
|
||||
* backend-client's `fetchWithTimeout`.
|
||||
*
|
||||
* Closes review findings on PR #1448:
|
||||
* - Non-idempotent POST/DELETE must NOT be retried by default —
|
||||
* a 5xx on `startAnalyze` could otherwise start a duplicate job.
|
||||
* - Timer-fired timeout must surface as `DOMException(name='TimeoutError')`,
|
||||
* not `AbortError`, so resilientFetch routes it through the
|
||||
* terminal-network branch (no retry, no breaker hit).
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getBreaker } from 'gitnexus-shared';
|
||||
import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers';
|
||||
import { fetchRepos, setBackendUrl, startAnalyze } from '../../src/services/backend-client';
|
||||
|
||||
const BASE = 'http://localhost:4747';
|
||||
|
||||
describe('backend-client retry budget (method-aware)', () => {
|
||||
beforeEach(() => {
|
||||
__resetBreakerRegistry__();
|
||||
setBackendUrl(BASE);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('GET retries once on transient 503 (idempotent verb)', async () => {
|
||||
let n = 0;
|
||||
const fetchMock = vi.fn(async () => {
|
||||
n += 1;
|
||||
if (n === 1) return new Response('boom', { status: 503 });
|
||||
return new Response('[]', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const repos = await fetchRepos();
|
||||
expect(repos).toEqual([]);
|
||||
// 1 retry budget on idempotent GET → 2 total fetch calls.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('POST does NOT retry on 503 by default (non-idempotent verb)', async () => {
|
||||
const fetchMock = vi.fn(async () => new Response('boom', { status: 503 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(startAnalyze({ path: '/tmp/repo' })).rejects.toBeTruthy();
|
||||
// Single attempt — never duplicates a job-start POST.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('switching backend URL after a circuit opens reaches a fresh breaker (U3)', async () => {
|
||||
// Pre-open the breaker for host-A by directly recording 3 failures.
|
||||
setBackendUrl('http://host-a.test:4747');
|
||||
const aKey = 'web-backend:http://host-a.test:4747';
|
||||
const breakerA = getBreaker(aKey);
|
||||
breakerA.recordFailure();
|
||||
breakerA.recordFailure();
|
||||
breakerA.recordFailure();
|
||||
expect(breakerA.getState()).toBe('open');
|
||||
|
||||
// Switch to host-B and make a request — must succeed against the
|
||||
// new origin without tripping the host-A circuit. Under the old
|
||||
// single-key behaviour the call would throw CircuitOpenError.
|
||||
setBackendUrl('http://host-b.test:4747');
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response('[]', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const repos = await fetchRepos();
|
||||
expect(repos).toEqual([]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Host-A's breaker is still open in cooldown.
|
||||
expect(breakerA.getState()).toBe('open');
|
||||
// Host-B has its own (fresh) breaker.
|
||||
const bKey = 'web-backend:http://host-b.test:4747';
|
||||
expect(getBreaker(bKey).getState()).toBe('closed');
|
||||
expect(getBreaker(bKey).getConsecutiveFailures()).toBe(0);
|
||||
});
|
||||
|
||||
it('breaker not incremented when timeout fires (TimeoutError, not AbortError)', async () => {
|
||||
// Reject directly with a TimeoutError DOMException, mimicking what
|
||||
// `fetch` produces when its `AbortSignal.timeout()`-wired signal
|
||||
// fires. The real-fetch path goes signal.reason → reject(reason);
|
||||
// we shortcut that here so the test doesn't have to wait the
|
||||
// 30-second default timeout.
|
||||
const fetchMock = vi.fn(async () => {
|
||||
throw new DOMException('aborted by timeout', 'TimeoutError');
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(fetchRepos()).rejects.toMatchObject({ code: 'timeout' });
|
||||
|
||||
// The breaker must not have been penalized for a local timeout.
|
||||
expect(getBreaker(`web-backend:${BASE}`).getConsecutiveFailures()).toBe(0);
|
||||
// Timeout is terminal — no retry attempted.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
Generated
+269
-23
@@ -30,6 +30,8 @@
|
||||
"mnemonist": "^0.40.3",
|
||||
"onnxruntime-node": "^1.24.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-c": "0.21.4",
|
||||
"tree-sitter-c-sharp": "0.23.1",
|
||||
@@ -1579,6 +1581,12 @@
|
||||
"url": "https://github.com/sponsors/Boshen"
|
||||
}
|
||||
},
|
||||
"node_modules/@pinojs/redact": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
|
||||
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@protobufjs/aspromise": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
||||
@@ -2057,9 +2065,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.6.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz",
|
||||
"integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==",
|
||||
"version": "25.6.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
||||
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.19.0"
|
||||
@@ -2380,6 +2388,15 @@
|
||||
"js-tokens": "^10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/atomic-sleep": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
|
||||
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
|
||||
@@ -2579,6 +2596,12 @@
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/colorette": {
|
||||
"version": "2.0.20",
|
||||
"resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
|
||||
"integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "14.0.3",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz",
|
||||
@@ -2683,6 +2706,15 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/dateformat": {
|
||||
"version": "4.6.3",
|
||||
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
|
||||
"integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
@@ -2806,6 +2838,15 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/end-of-stream": {
|
||||
"version": "1.4.5",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
|
||||
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/es-define-property": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||
@@ -3018,12 +3059,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/express-rate-limit": {
|
||||
"version": "8.4.1",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.4.1.tgz",
|
||||
"integrity": "sha512-NGVYwQSAyEQgzxX1iCM978PP9AdO/hW93gMcF6ZwQCm+rFvLsBH6w4xcXWTcliS8La5EPRN3p9wzItqBwJrfNw==",
|
||||
"version": "8.5.1",
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.1.tgz",
|
||||
"integrity": "sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ip-address": "10.1.0"
|
||||
"ip-address": "^10.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16"
|
||||
@@ -3050,16 +3091,28 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-copy": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.3.tgz",
|
||||
"integrity": "sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-safe-stringify": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
|
||||
"integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz",
|
||||
"integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==",
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz",
|
||||
"integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -3416,10 +3469,16 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/help-me": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
|
||||
"integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/hono": {
|
||||
"version": "4.12.16",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
|
||||
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
|
||||
"version": "4.12.18",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz",
|
||||
"integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=16.9.0"
|
||||
@@ -3486,9 +3545,9 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ip-address": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz",
|
||||
"integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==",
|
||||
"version": "10.2.0",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
|
||||
"integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
@@ -3575,6 +3634,15 @@
|
||||
"url": "https://github.com/sponsors/panva"
|
||||
}
|
||||
},
|
||||
"node_modules/joycon": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz",
|
||||
"integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz",
|
||||
@@ -4183,6 +4251,15 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/on-exit-leak-free": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
|
||||
"integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/on-finished": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
||||
@@ -4205,15 +4282,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/onnxruntime-common": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.25.1.tgz",
|
||||
"integrity": "sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg==",
|
||||
"version": "1.26.0",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.26.0.tgz",
|
||||
"integrity": "sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/onnxruntime-node": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.25.1.tgz",
|
||||
"integrity": "sha512-N0M58CGTiTsLkPpx9bxmRFi24GT6r67Qei/GrBEIiDyntcYdXU5vQZp112ypydG9vEKRFgbgUYQJnEi+jll8dg==",
|
||||
"version": "1.26.0",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.26.0.tgz",
|
||||
"integrity": "sha512-OHl6PiOEOqxaLHL0N9eFrbzS7IGmu3BtJNH3RTEnRAheCIkfc3gjcjl4sGcjp9C22ZC9YTquDOxSdT/stBQ6BQ==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"os": [
|
||||
@@ -4224,7 +4301,7 @@
|
||||
"dependencies": {
|
||||
"adm-zip": "^0.5.16",
|
||||
"global-agent": "^4.1.3",
|
||||
"onnxruntime-common": "1.25.1"
|
||||
"onnxruntime-common": "1.26.0"
|
||||
}
|
||||
},
|
||||
"node_modules/onnxruntime-web": {
|
||||
@@ -4332,6 +4409,79 @@
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/pino": {
|
||||
"version": "10.3.1",
|
||||
"resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",
|
||||
"integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pinojs/redact": "^0.4.0",
|
||||
"atomic-sleep": "^1.0.0",
|
||||
"on-exit-leak-free": "^2.1.0",
|
||||
"pino-abstract-transport": "^3.0.0",
|
||||
"pino-std-serializers": "^7.0.0",
|
||||
"process-warning": "^5.0.0",
|
||||
"quick-format-unescaped": "^4.0.3",
|
||||
"real-require": "^0.2.0",
|
||||
"safe-stable-stringify": "^2.3.1",
|
||||
"sonic-boom": "^4.0.1",
|
||||
"thread-stream": "^4.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"pino": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-abstract-transport": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",
|
||||
"integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-pretty": {
|
||||
"version": "13.1.3",
|
||||
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.1.3.tgz",
|
||||
"integrity": "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"colorette": "^2.0.7",
|
||||
"dateformat": "^4.6.3",
|
||||
"fast-copy": "^4.0.0",
|
||||
"fast-safe-stringify": "^2.1.1",
|
||||
"help-me": "^5.0.0",
|
||||
"joycon": "^3.1.1",
|
||||
"minimist": "^1.2.6",
|
||||
"on-exit-leak-free": "^2.1.0",
|
||||
"pino-abstract-transport": "^3.0.0",
|
||||
"pump": "^3.0.0",
|
||||
"secure-json-parse": "^4.0.0",
|
||||
"sonic-boom": "^4.0.1",
|
||||
"strip-json-comments": "^5.0.2"
|
||||
},
|
||||
"bin": {
|
||||
"pino-pretty": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-pretty/node_modules/strip-json-comments": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz",
|
||||
"integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-std-serializers": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz",
|
||||
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pkce-challenge": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz",
|
||||
@@ -4376,6 +4526,22 @@
|
||||
"node": "^10 || ^12 || >=14"
|
||||
}
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz",
|
||||
"integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/protobufjs": {
|
||||
"version": "7.5.5",
|
||||
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz",
|
||||
@@ -4413,6 +4579,16 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/pump": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
|
||||
"integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"end-of-stream": "^1.1.0",
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/qs": {
|
||||
"version": "6.14.2",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz",
|
||||
@@ -4428,6 +4604,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/quick-format-unescaped": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
|
||||
"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
@@ -4483,6 +4665,15 @@
|
||||
"rc": "cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/real-require": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
|
||||
"integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 12.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
@@ -4591,12 +4782,37 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-stable-stringify": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz",
|
||||
"integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/secure-json-parse": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
|
||||
"integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.7.4",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
|
||||
@@ -4828,6 +5044,15 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/sonic-boom": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
|
||||
"integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"atomic-sleep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
@@ -4838,6 +5063,15 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/stackback": {
|
||||
"version": "0.0.2",
|
||||
"resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz",
|
||||
@@ -4925,6 +5159,18 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/thread-stream": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz",
|
||||
"integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"real-require": "^0.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/tinybench": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz",
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
"dev": "tsx watch src/cli/index.ts",
|
||||
"test": "vitest run",
|
||||
"test:unit": "vitest run test/unit",
|
||||
"pretest:integration": "node scripts/build.js",
|
||||
"test:integration": "vitest run test/integration",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
@@ -72,6 +73,8 @@
|
||||
"mnemonist": "^0.40.3",
|
||||
"onnxruntime-node": "^1.24.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-c": "0.21.4",
|
||||
"tree-sitter-c-sharp": "0.23.1",
|
||||
|
||||
@@ -3,6 +3,17 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
// Opt-out: skip the native rebuild entirely. Dart parsing becomes
|
||||
// unavailable but `npm install gitnexus` finishes much faster on machines
|
||||
// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0'
|
||||
// (read as a string), and any other value all fall through to the rebuild.
|
||||
if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') {
|
||||
console.warn(
|
||||
'[tree-sitter-dart] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart parsing will be unavailable until reinstalled without the env var.',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart');
|
||||
const bindingGyp = path.join(dartDir, 'binding.gyp');
|
||||
const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node');
|
||||
|
||||
@@ -34,6 +34,17 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
// Opt-out: skip the native rebuild entirely. Proto parsing becomes
|
||||
// unavailable but `npm install gitnexus` finishes much faster on machines
|
||||
// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0'
|
||||
// (read as a string), and any other value all fall through to the rebuild.
|
||||
if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') {
|
||||
console.warn(
|
||||
'[tree-sitter-proto] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Proto parsing will be unavailable until reinstalled without the env var.',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto');
|
||||
const bindingGyp = path.join(protoDir, 'binding.gyp');
|
||||
const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node');
|
||||
|
||||
@@ -10,6 +10,7 @@ import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { type GeneratedSkillInfo } from './skill-gen.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
// ESM equivalent of __dirname
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
@@ -293,7 +294,7 @@ Use GitNexus tools to accomplish this task.
|
||||
installedSkills.push(skill.name);
|
||||
} catch (err) {
|
||||
// Skip on error, don't fail the whole process
|
||||
console.warn(`Warning: Could not install skill ${skill.name}:`, err);
|
||||
logger.warn({ err }, `Warning: Could not install skill ${skill.name}:`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+134
-33
@@ -23,7 +23,11 @@ import {
|
||||
import { getGitRoot, hasGitDir } from '../storage/git.js';
|
||||
import { runFullAnalysis } from '../core/run-analyze.js';
|
||||
import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
|
||||
import { warnMissingOptionalGrammars } from './optional-grammars.js';
|
||||
import { glob } from 'glob';
|
||||
import fs from 'fs/promises';
|
||||
import { cliError } from './cli-message.js';
|
||||
import { isHfDownloadFailure } from '../core/embeddings/hf-env.js';
|
||||
|
||||
// Capture stderr.write at module load BEFORE anything (LadybugDB native
|
||||
// init, progress bar, console redirection) can monkey-patch it. The
|
||||
@@ -95,7 +99,14 @@ function ensureHeap(): boolean {
|
||||
|
||||
export interface AnalyzeOptions {
|
||||
force?: boolean;
|
||||
embeddings?: boolean;
|
||||
/**
|
||||
* Embedding generation toggle. Commander parses `--embeddings [limit]` as:
|
||||
* - `undefined` when the flag is omitted
|
||||
* - `true` when passed without an argument (use default 50K node cap)
|
||||
* - a string when passed with an argument (`--embeddings 0` disables the
|
||||
* cap, `--embeddings <n>` uses `<n>` as the cap)
|
||||
*/
|
||||
embeddings?: boolean | string;
|
||||
/**
|
||||
* Explicitly drop existing embeddings on rebuild instead of preserving
|
||||
* them. Without this flag, a routine `analyze` keeps any embeddings
|
||||
@@ -158,7 +169,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
if (options?.workerTimeout) {
|
||||
const workerTimeoutSeconds = Number(options.workerTimeout);
|
||||
if (!Number.isFinite(workerTimeoutSeconds) || workerTimeoutSeconds < 1) {
|
||||
console.error(' --worker-timeout must be at least 1 second.\n');
|
||||
cliError(' --worker-timeout must be at least 1 second.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -167,6 +178,25 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
);
|
||||
}
|
||||
|
||||
// Parse `--embeddings [limit]`: `true` → default cap, string → numeric cap
|
||||
// (0 disables the cap entirely). Validated up here so failures match the
|
||||
// sibling-validation pattern (exit before bar.start() — otherwise
|
||||
// process.exit() leaves the progress bar's hidden cursor uncleared).
|
||||
let embeddingsNodeLimit: number | undefined;
|
||||
if (typeof options?.embeddings === 'string') {
|
||||
const parsed = Number(options.embeddings);
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
cliError(
|
||||
` --embeddings expects a non-negative integer (got "${options.embeddings}"). ` +
|
||||
`Pass 0 to disable the safety cap, or omit the value to keep the default.\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
embeddingsNodeLimit = parsed;
|
||||
}
|
||||
const embeddingsEnabled = !!options?.embeddings;
|
||||
|
||||
const setPositiveEnv = (
|
||||
optionName: string,
|
||||
envName: string,
|
||||
@@ -175,7 +205,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
if (value === undefined) return true;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
console.error(` ${optionName} must be a positive integer.\n`);
|
||||
cliError(` ${optionName} must be a positive integer.\n`);
|
||||
process.exitCode = 1;
|
||||
return false;
|
||||
}
|
||||
@@ -206,7 +236,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
if (options?.embeddingDevice) {
|
||||
const allowed = new Set(['auto', 'cpu', 'dml', 'cuda', 'wasm']);
|
||||
if (!allowed.has(options.embeddingDevice)) {
|
||||
console.error(' --embedding-device must be one of: auto, cpu, dml, cuda, wasm.\n');
|
||||
cliError(' --embedding-device must be one of: auto, cpu, dml, cuda, wasm.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -247,6 +277,30 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
);
|
||||
}
|
||||
|
||||
// If the target repo contains files an optional grammar would parse but
|
||||
// that grammar's native binding is absent, warn before analysis so users
|
||||
// learn why those files end up unparsed instead of silently getting a
|
||||
// degraded index.
|
||||
try {
|
||||
const matches = await glob(['**/*.dart', '**/*.proto'], {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'],
|
||||
dot: false,
|
||||
nodir: true,
|
||||
absolute: false,
|
||||
});
|
||||
if (matches.length > 0) {
|
||||
const present = new Set<string>();
|
||||
for (const m of matches) {
|
||||
const ext = path.extname(m).toLowerCase();
|
||||
if (ext) present.add(ext);
|
||||
}
|
||||
warnMissingOptionalGrammars({ context: 'analyze', relevantExtensions: present });
|
||||
}
|
||||
} catch {
|
||||
// Best-effort warning \u2014 never block analyze on the precheck.
|
||||
}
|
||||
|
||||
// KuzuDB migration cleanup is handled by runFullAnalysis internally.
|
||||
// Note: --skills is handled after runFullAnalysis using the returned pipelineResult.
|
||||
|
||||
@@ -278,7 +332,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
|
||||
bar.start(100, 0, { phase: 'Initializing...' });
|
||||
|
||||
// Graceful SIGINT handling
|
||||
// Graceful SIGINT handling. Pino's default destination is `sync: false`
|
||||
// (buffered) — flush before exit so in-flight records reach stderr.
|
||||
// See `gitnexus/src/core/logger.ts:flushLoggerSync`.
|
||||
let aborted = false;
|
||||
const sigintHandler = () => {
|
||||
if (aborted) process.exit(1);
|
||||
@@ -287,13 +343,23 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
console.log('\n Interrupted — cleaning up...');
|
||||
closeLbug()
|
||||
.catch(() => {})
|
||||
.finally(() => process.exit(130));
|
||||
.finally(async () => {
|
||||
const { flushLoggerSync } = await import('../core/logger.js');
|
||||
flushLoggerSync();
|
||||
process.exit(130);
|
||||
});
|
||||
};
|
||||
process.on('SIGINT', sigintHandler);
|
||||
|
||||
// Route console output through bar.log() to prevent progress bar corruption
|
||||
// Route console output through bar.log() to prevent progress bar corruption.
|
||||
// This is a deliberate UI pattern (not a logging concern): analyze runs a
|
||||
// long-lived progress bar on stdout; any concurrent console.* write would
|
||||
// overwrite the bar mid-render. We capture originals, swap to barLog for
|
||||
// the lifetime of the run, and restore on completion/error/SIGINT.
|
||||
const origLog = console.log.bind(console);
|
||||
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
|
||||
const origWarn = console.warn.bind(console);
|
||||
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
|
||||
const origError = console.error.bind(console);
|
||||
let barCurrentValue = 0;
|
||||
const barLog = (...args: any[]) => {
|
||||
@@ -302,7 +368,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
bar.update(barCurrentValue);
|
||||
};
|
||||
console.log = barLog;
|
||||
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
|
||||
console.warn = barLog;
|
||||
// eslint-disable-next-line no-console -- intentional console-routing for progress bar UX
|
||||
console.error = barLog;
|
||||
|
||||
// Track elapsed time per phase
|
||||
@@ -338,7 +406,8 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
// needs a fresh pipelineResult. Has no bearing on the registry
|
||||
// collision guard (see allowDuplicateName below).
|
||||
force: options?.force || options?.skills,
|
||||
embeddings: options?.embeddings,
|
||||
embeddings: embeddingsEnabled,
|
||||
embeddingsNodeLimit,
|
||||
dropEmbeddings: options?.dropEmbeddings,
|
||||
skipGit: options?.skipGit,
|
||||
skipAgentsMd: options?.skipAgentsMd,
|
||||
@@ -367,7 +436,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
clearInterval(elapsedTimer);
|
||||
process.removeListener('SIGINT', sigintHandler);
|
||||
console.log = origLog;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.warn = origWarn;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.error = origError;
|
||||
bar.stop();
|
||||
console.log(' Already up to date\n');
|
||||
@@ -440,7 +511,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
process.removeListener('SIGINT', sigintHandler);
|
||||
|
||||
console.log = origLog;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.warn = origWarn;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.error = origError;
|
||||
|
||||
bar.update(100, { phase: 'Done' });
|
||||
@@ -465,7 +538,9 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
clearInterval(elapsedTimer);
|
||||
process.removeListener('SIGINT', sigintHandler);
|
||||
console.log = origLog;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.warn = origWarn;
|
||||
// eslint-disable-next-line no-console -- restoring after intentional progress-bar routing
|
||||
console.error = origError;
|
||||
bar.stop();
|
||||
|
||||
@@ -474,14 +549,14 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
// Registry name-collision from --name (#829) — surface as an
|
||||
// actionable error rather than a generic stack-trace.
|
||||
if (err instanceof RegistryNameCollisionError) {
|
||||
console.error(`\n Registry name collision:\n`);
|
||||
console.error(` "${err.registryName}" is already used by "${err.existingPath}".\n`);
|
||||
console.error(` Options:`);
|
||||
console.error(` • Pick a different alias: gitnexus analyze --name <alias>`);
|
||||
console.error(
|
||||
` • Allow the duplicate: gitnexus analyze --allow-duplicate-name (leaves "-r ${err.registryName}" ambiguous)`,
|
||||
cliError(
|
||||
`\n Registry name collision:\n` +
|
||||
` "${err.registryName}" is already used by "${err.existingPath}".\n\n` +
|
||||
` Options:\n` +
|
||||
` • Pick a different alias: gitnexus analyze --name <alias>\n` +
|
||||
` • Allow the duplicate: gitnexus analyze --allow-duplicate-name (leaves "-r ${err.registryName}" ambiguous)\n`,
|
||||
{ registryName: err.registryName, existingPath: err.existingPath },
|
||||
);
|
||||
console.error('');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -502,6 +577,26 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
return;
|
||||
}
|
||||
|
||||
// HF download failure — show clean guidance without the raw stack trace.
|
||||
// Checked before writeFatalToStderr so the user sees one focused message
|
||||
// rather than a stack-trace dump followed by a second remediation block.
|
||||
if (isHfDownloadFailure(msg) || msg.includes('Failed to download embedding model')) {
|
||||
cliError(
|
||||
` The embedding model could not be downloaded.\n` +
|
||||
` huggingface.co may be unreachable from your network\n` +
|
||||
` (e.g. behind a corporate proxy or a regional firewall).\n` +
|
||||
` Suggestions:\n` +
|
||||
` 1. Set HF_ENDPOINT to a mirror and retry:\n` +
|
||||
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
|
||||
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)\n` +
|
||||
` 2. Check your proxy / VPN settings.\n` +
|
||||
` 3. Once downloaded the model is cached — future runs work offline.\n`,
|
||||
{ recoveryHint: 'hf-endpoint-unreachable' },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Bypass the redirected console.error and write the full stack to
|
||||
// the real stderr captured at module load. The redirected
|
||||
// console.error wraps every line with `\\x1b[2K\\r` (ANSI clear-line)
|
||||
@@ -521,34 +616,40 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
||||
msg.includes('heap out of memory') ||
|
||||
msg.includes('JavaScript heap')
|
||||
) {
|
||||
console.error(' This error typically occurs on very large repositories.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Add large vendored/generated directories to .gitnexusignore');
|
||||
console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"');
|
||||
console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"');
|
||||
console.error('');
|
||||
cliError(
|
||||
` This error typically occurs on very large repositories.\n` +
|
||||
` Suggestions:\n` +
|
||||
` 1. Add large vendored/generated directories to .gitnexusignore\n` +
|
||||
` 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"\n` +
|
||||
` 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"\n`,
|
||||
{ recoveryHint: 'large-repo' },
|
||||
);
|
||||
} else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) {
|
||||
// Note: the original arborist "Cannot destructure property 'package' of
|
||||
// 'node.target'" crash happens inside npm *before* gitnexus code runs,
|
||||
// so it can't be caught here. This branch handles dependency-resolution
|
||||
// errors that surface at runtime (e.g. dynamic require failures).
|
||||
console.error(' This looks like an npm dependency resolution issue.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Clear the npm cache: npm cache clean --force');
|
||||
console.error(' 2. Update npm: npm install -g npm@latest');
|
||||
console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest');
|
||||
console.error(' 4. Or try npx directly: npx gitnexus@latest analyze');
|
||||
console.error('');
|
||||
cliError(
|
||||
` This looks like an npm dependency resolution issue.\n` +
|
||||
` Suggestions:\n` +
|
||||
` 1. Clear the npm cache: npm cache clean --force\n` +
|
||||
` 2. Update npm: npm install -g npm@latest\n` +
|
||||
` 3. Reinstall gitnexus: npm install -g gitnexus@latest\n` +
|
||||
` 4. Or try npx directly: npx gitnexus@latest analyze\n`,
|
||||
{ recoveryHint: 'npm-resolution' },
|
||||
);
|
||||
} else if (
|
||||
msg.includes('MODULE_NOT_FOUND') ||
|
||||
msg.includes('Cannot find module') ||
|
||||
msg.includes('ERR_MODULE_NOT_FOUND')
|
||||
) {
|
||||
console.error(' A required module could not be loaded. The installation may be corrupt.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Reinstall: npm install -g gitnexus@latest');
|
||||
console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze');
|
||||
console.error('');
|
||||
cliError(
|
||||
` A required module could not be loaded. The installation may be corrupt.\n` +
|
||||
` Suggestions:\n` +
|
||||
` 1. Reinstall: npm install -g gitnexus@latest\n` +
|
||||
` 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze\n`,
|
||||
{ recoveryHint: 'module-not-found' },
|
||||
);
|
||||
}
|
||||
|
||||
process.exitCode = 1;
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import { logger } from '../core/logger.js';
|
||||
import {
|
||||
findRepo,
|
||||
unregisterRepo,
|
||||
@@ -45,7 +46,7 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
|
||||
assertSafeStoragePath(entry);
|
||||
} catch (err) {
|
||||
if (err instanceof UnsafeStoragePathError) {
|
||||
console.error(`Refusing to clean ${entry.name}: ${err.message}`);
|
||||
logger.error(`Refusing to clean ${entry.name}: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
@@ -56,7 +57,7 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
|
||||
await unregisterRepo(entry.path);
|
||||
console.log(`Deleted: ${entry.name} (${entry.storagePath})`);
|
||||
} catch (err) {
|
||||
console.error(`Failed to delete ${entry.name}:`, err);
|
||||
logger.error({ err }, `Failed to delete ${entry.name}:`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
@@ -85,6 +86,6 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean })
|
||||
await unregisterRepo(repo.repoPath);
|
||||
console.log(`Deleted: ${repo.storagePath}`);
|
||||
} catch (err) {
|
||||
console.error('Failed to delete:', err);
|
||||
logger.error({ err }, 'Failed to delete:');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* CLI message helpers — for user-facing banners, error guidance, and
|
||||
* recovery hints emitted by `gitnexus` subcommands.
|
||||
*
|
||||
* These functions write **plain text** directly to `process.stderr` AND
|
||||
* tee a structured pino record through the singleton `logger`. Plain text
|
||||
* preserves the human-readable contract for users running `gitnexus`
|
||||
* interactively, redirecting to a file, or piping to `cat`/`grep`. The
|
||||
* structured tee keeps log aggregators happy.
|
||||
*
|
||||
* **Use these for:**
|
||||
* - User-facing banners ("Server listening on http://...:N")
|
||||
* - Validation errors ("--worker-timeout must be at least 1 second")
|
||||
* - Recovery hints ("Suggestions: 1. Clear the npm cache, 2. ...")
|
||||
* - One-line user notices ("No indexed repositories found.")
|
||||
*
|
||||
* **Do NOT use these for:**
|
||||
* - Internal diagnostics (worker progress, retry counts, telemetry)
|
||||
* — use `logger.info`/`warn`/`error` directly. Internal logs only
|
||||
* need structured fields, not double-output to stderr.
|
||||
* - High-volume hot paths — every `cliMessage` call writes twice (raw
|
||||
* + structured). Acceptable for user-facing messages, wasteful for
|
||||
* ingestion pipeline events.
|
||||
*
|
||||
* Design note: stderr is the right channel even for non-error messages
|
||||
* because GitNexus CLI tools (`query`, `cypher`, `impact`) emit JSON
|
||||
* data on stdout for piping (`gitnexus query | jq`). User banners on
|
||||
* stdout would corrupt that pipeline.
|
||||
*/
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
function writeStderr(msg: string): void {
|
||||
// Direct write — bypassing `console.*` so it cannot be intercepted by
|
||||
// progress-bar redirection (see `cli/analyze.ts:barLog`) or other
|
||||
// routing. The structured tee below still goes through the logger so
|
||||
// log aggregation works either way.
|
||||
process.stderr.write(msg.endsWith('\n') ? msg : msg + '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing informational message. Use for banners, listening URLs,
|
||||
* and any message the user expects to read in plain text.
|
||||
*/
|
||||
export function cliInfo(msg: string, fields?: Record<string, unknown>): void {
|
||||
writeStderr(msg);
|
||||
logger.info(fields ?? {}, msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing warning. Operator-actionable but non-fatal — `cliWarn`
|
||||
* indicates the command can still proceed in some form.
|
||||
*/
|
||||
export function cliWarn(msg: string, fields?: Record<string, unknown>): void {
|
||||
writeStderr(msg);
|
||||
logger.warn(fields ?? {}, msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing error. Indicates the command cannot proceed; usually
|
||||
* paired with a non-zero exit code at the call site.
|
||||
*/
|
||||
export function cliError(msg: string, fields?: Record<string, unknown>): void {
|
||||
writeStderr(msg);
|
||||
logger.error(fields ?? {}, msg);
|
||||
}
|
||||
@@ -27,6 +27,8 @@
|
||||
import http from 'http';
|
||||
import { writeSync } from 'node:fs';
|
||||
import { LocalBackend } from '../mcp/local/local-backend.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
import { cliInfo, cliWarn } from './cli-message.js';
|
||||
|
||||
export interface EvalServerOptions {
|
||||
port?: string;
|
||||
@@ -332,13 +334,19 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
|
||||
const ok = await backend.init();
|
||||
|
||||
if (!ok) {
|
||||
console.error('GitNexus eval-server: No indexed repositories found. Run: gitnexus analyze');
|
||||
// Operator-actionable but the server cannot start; warn-level so log
|
||||
// aggregators don't trip error alerts on a configuration miss. Use
|
||||
// cliWarn so the diagnostic reaches stderr synchronously before
|
||||
// process.exit() — direct logger.warn would be lost to the buffered
|
||||
// pino destination on hard exit (skips beforeExit flush).
|
||||
cliWarn('GitNexus eval-server: No indexed repositories found. Run: gitnexus analyze');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const repos = await backend.listRepos();
|
||||
console.error(
|
||||
`GitNexus eval-server: ${repos.length} repo(s) loaded: ${repos.map((r) => r.name).join(', ')}`,
|
||||
logger.info(
|
||||
{ repoCount: repos.length, repos: repos.map((r) => r.name) },
|
||||
'GitNexus eval-server: repos loaded',
|
||||
);
|
||||
|
||||
let idleTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
@@ -347,7 +355,7 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
|
||||
if (idleTimeoutSec <= 0) return;
|
||||
if (idleTimer) clearTimeout(idleTimer);
|
||||
idleTimer = setTimeout(async () => {
|
||||
console.error('GitNexus eval-server: Idle timeout reached, shutting down');
|
||||
logger.info({ idleTimeoutSec }, 'GitNexus eval-server: idle timeout reached, shutting down');
|
||||
await backend.disconnect();
|
||||
process.exit(0);
|
||||
}, idleTimeoutSec * 1000);
|
||||
@@ -419,16 +427,34 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
|
||||
});
|
||||
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
console.error(`GitNexus eval-server: listening on http://127.0.0.1:${port}`);
|
||||
console.error(` POST /tool/query — search execution flows`);
|
||||
console.error(` POST /tool/context — 360-degree symbol view`);
|
||||
console.error(` POST /tool/impact — blast radius analysis`);
|
||||
console.error(` POST /tool/cypher — raw Cypher query`);
|
||||
console.error(` GET /health — health check`);
|
||||
console.error(` POST /shutdown — graceful shutdown`);
|
||||
// Plain-text banner for the human watching stderr; structured record
|
||||
// for log aggregation (split into two so the user sees a real banner
|
||||
// not `{"level":30,"msg":"...","port":4747,"endpoints":[...]}`).
|
||||
const bannerLines = [
|
||||
`GitNexus eval-server: listening on http://127.0.0.1:${port}`,
|
||||
` POST /tool/query — search execution flows`,
|
||||
` POST /tool/context — 360-degree symbol view`,
|
||||
` POST /tool/impact — blast radius analysis`,
|
||||
` POST /tool/cypher — raw Cypher query`,
|
||||
` GET /health — health check`,
|
||||
` POST /shutdown — graceful shutdown`,
|
||||
];
|
||||
if (idleTimeoutSec > 0) {
|
||||
console.error(` Auto-shutdown after ${idleTimeoutSec}s idle`);
|
||||
bannerLines.push(` Auto-shutdown after ${idleTimeoutSec}s idle`);
|
||||
}
|
||||
cliInfo(bannerLines.join('\n'), {
|
||||
port,
|
||||
host: '127.0.0.1',
|
||||
idleTimeoutSec: idleTimeoutSec > 0 ? idleTimeoutSec : undefined,
|
||||
endpoints: [
|
||||
'POST /tool/query',
|
||||
'POST /tool/context',
|
||||
'POST /tool/impact',
|
||||
'POST /tool/cypher',
|
||||
'GET /health',
|
||||
'POST /shutdown',
|
||||
],
|
||||
});
|
||||
try {
|
||||
// Use fd 1 directly — LadybugDB captures process.stdout (#324)
|
||||
writeSync(1, `GITNEXUS_EVAL_SERVER_READY:${port}\n`);
|
||||
@@ -440,7 +466,7 @@ export async function evalServerCommand(options?: EvalServerOptions): Promise<vo
|
||||
resetIdleTimer();
|
||||
|
||||
const shutdown = async () => {
|
||||
console.error('GitNexus eval-server: shutting down...');
|
||||
logger.info('GitNexus eval-server: shutting down...');
|
||||
await backend.disconnect();
|
||||
server.close();
|
||||
process.exit(0);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// gitnexus/src/cli/group.ts
|
||||
import { createRequire } from 'node:module';
|
||||
import type { Command } from 'commander';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
const yaml = _require('js-yaml') as typeof import('js-yaml');
|
||||
@@ -51,7 +52,7 @@ export function registerGroupCommands(program: Command): void {
|
||||
const groupDir = getGroupDir(getDefaultGitnexusDir(), groupName);
|
||||
const config = await loadGroupConfig(groupDir);
|
||||
if (!(repoPath in config.repos)) {
|
||||
console.error(`Repo path "${repoPath}" not found in group "${groupName}"`);
|
||||
logger.error(`Repo path "${repoPath}" not found in group "${groupName}"`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -239,7 +240,7 @@ export function registerGroupCommands(program: Command): void {
|
||||
|
||||
const raw = await backend.getGroupService().groupImpact(payload);
|
||||
if (raw && typeof raw === 'object' && 'error' in raw) {
|
||||
console.error(String((raw as { error: string }).error));
|
||||
logger.error(String((raw as { error: string }).error));
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -333,7 +334,7 @@ export function registerGroupCommands(program: Command): void {
|
||||
});
|
||||
|
||||
if (raw && typeof raw === 'object' && 'error' in raw) {
|
||||
console.error(String((raw as { error: string }).error));
|
||||
logger.error(String((raw as { error: string }).error));
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,11 @@ program
|
||||
.command('analyze [path]')
|
||||
.description('Index a repository (full analysis)')
|
||||
.option('-f, --force', 'Force full re-index even if up to date')
|
||||
.option('--embeddings', 'Enable embedding generation for semantic search (off by default)')
|
||||
.option(
|
||||
'--embeddings [limit]',
|
||||
'Enable embedding generation for semantic search (off by default). ' +
|
||||
'Optional [limit] overrides the 50,000-node safety cap; pass 0 to disable the cap entirely.',
|
||||
)
|
||||
.option(
|
||||
'--drop-embeddings',
|
||||
'Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` ' +
|
||||
@@ -156,6 +160,18 @@ program
|
||||
.description('Augment a search pattern with knowledge graph context (used by hooks)')
|
||||
.action(createLazyAction(() => import('./augment.js'), 'augmentCommand'));
|
||||
|
||||
program
|
||||
.command('publish [path]')
|
||||
.description(
|
||||
'Notify the understand-quickly registry that this repo has a fresh GitNexus index. ' +
|
||||
'Opt-in: requires UNDERSTAND_QUICKLY_TOKEN (fine-grained PAT with ' +
|
||||
'`Repository dispatches: write` on looptech-ai/understand-quickly). ' +
|
||||
'No-op without the token. See https://github.com/looptech-ai/understand-quickly.',
|
||||
)
|
||||
.option('--id <owner/repo>', 'Override the registry id (defaults to the origin remote)')
|
||||
.option('--skip-git', 'Treat cwd as the repo root and skip parent git-root discovery')
|
||||
.action(createLazyAction(() => import('./publish.js'), 'publishCommand'));
|
||||
|
||||
// ─── Direct Tool Commands (no MCP overhead) ────────────────────────
|
||||
// These invoke LocalBackend directly for use in eval, scripts, and CI.
|
||||
|
||||
|
||||
+57
-16
@@ -4,23 +4,61 @@
|
||||
* Starts the MCP server in standalone mode.
|
||||
* Loads all indexed repos from the global registry.
|
||||
* No longer depends on cwd — works from any directory.
|
||||
*
|
||||
* IMPORTANT: this module's static-import closure is intentionally tiny
|
||||
* (one chain: `mcp/stdio-context.js` → `mcp/stdio-capture.js`, which is a
|
||||
* leaf with zero non-`node:` imports). All heavy backend modules
|
||||
* (`startMCPServer`, `LocalBackend`, `warnMissingOptionalGrammars`) load
|
||||
* via `await import(...)` AFTER `installGlobalStdoutSentinel()` runs.
|
||||
*
|
||||
* This closes the ESM-evaluation-order window where native init banners
|
||||
* from `@ladybugdb/core` (or any future heavy import) could reach raw
|
||||
* stdout before the sentinel exists. Codex's adversarial review on
|
||||
* PR #1383 found that even with the sentinel-install call as the first
|
||||
* statement of `mcpCommand`, ESM evaluates static imports of THIS module
|
||||
* before the function body runs — so any native side effects during
|
||||
* those imports happen before the sentinel can intercept them.
|
||||
*
|
||||
* If you find yourself adding a static `import` to this file, ask
|
||||
* whether the imported module (or anything it transitively imports)
|
||||
* touches `process.stdout` or loads a native binding at module init. If
|
||||
* either is true, switch it to a dynamic `await import(...)` inside
|
||||
* `mcpCommand` after the sentinel install. The regression test at
|
||||
* `gitnexus/test/integration/mcp/import-closure.test.ts` enforces this.
|
||||
*/
|
||||
|
||||
import { startMCPServer } from '../mcp/server.js';
|
||||
import { LocalBackend } from '../mcp/local/local-backend.js';
|
||||
import { installGlobalStdoutSentinel } from '../mcp/stdio-context.js';
|
||||
|
||||
export const mcpCommand = async () => {
|
||||
// Prevent unhandled errors from crashing the MCP server process.
|
||||
// LadybugDB lock conflicts and transient errors should degrade gracefully.
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error(`GitNexus MCP: uncaught exception — ${err.message}`);
|
||||
// Process is in an undefined state after uncaughtException — exit after flushing
|
||||
setTimeout(() => process.exit(1), 100);
|
||||
});
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
const msg = reason instanceof Error ? reason.message : String(reason);
|
||||
console.error(`GitNexus MCP: unhandled rejection — ${msg}`);
|
||||
});
|
||||
// Install the global stdout sentinel as the very first thing — before
|
||||
// ANY other module loads. The static-import closure above is leaf-only
|
||||
// (stdio-context → stdio-capture, zero non-`node:` deps), so this is
|
||||
// also the first chance any code in this process has to write to stdout.
|
||||
installGlobalStdoutSentinel();
|
||||
|
||||
// uncaughtException/unhandledRejection handlers are owned by
|
||||
// startMCPServer (gitnexus/src/mcp/server.ts) so the server's shutdown
|
||||
// path runs cleanly with full stack traces. Registering duplicates here
|
||||
// would only produce noisy double-logging on the same exception.
|
||||
|
||||
// Dynamically import heavy backend modules AND the pino logger AFTER
|
||||
// the sentinel installs. The logger is dynamic-imported (rather than
|
||||
// static) to preserve the leaf-only static-import closure documented at
|
||||
// the top of this file — `core/logger.js` itself doesn't write to
|
||||
// stdout at module init, but transitive deps (pino, pino-pretty, the
|
||||
// worker-thread transport) could in theory, and the import-closure
|
||||
// regression test enforces the leaf invariant.
|
||||
const [{ startMCPServer }, { LocalBackend }, { logger }] = await Promise.all([
|
||||
import('../mcp/server.js'),
|
||||
import('../mcp/local/local-backend.js'),
|
||||
import('../core/logger.js'),
|
||||
]);
|
||||
|
||||
// Missing-optional-grammar warnings are intentionally NOT emitted here.
|
||||
// `gitnexus analyze` already warns at index time, filtered by the repo's
|
||||
// actual extensions, and a repo can only be served by MCP after analyze
|
||||
// has run. Repeating an unconditional warning at every MCP startup is
|
||||
// pure noise for users whose indexed repos don't use Dart/Proto.
|
||||
|
||||
// Initialize multi-repo backend from registry.
|
||||
// The server starts even with 0 repos — tools call refreshRepos() lazily,
|
||||
@@ -30,12 +68,15 @@ export const mcpCommand = async () => {
|
||||
|
||||
const repos = await backend.listRepos();
|
||||
if (repos.length === 0) {
|
||||
console.error(
|
||||
// Operator-actionable but the server still starts and serves; warn-level,
|
||||
// not error. Tools will discover newly-analyzed repos via lazy refresh.
|
||||
logger.warn(
|
||||
'GitNexus: No indexed repos yet. Run `gitnexus analyze` in a git repo — the server will pick it up automatically.',
|
||||
);
|
||||
} else {
|
||||
console.error(
|
||||
`GitNexus: MCP server starting with ${repos.length} repo(s): ${repos.map((r) => r.name).join(', ')}`,
|
||||
logger.info(
|
||||
{ repoCount: repos.length, repos: repos.map((r) => r.name) },
|
||||
'GitNexus: MCP server starting',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Optional grammar availability check.
|
||||
*
|
||||
* tree-sitter-dart and tree-sitter-proto are optionalDependencies that
|
||||
* require a `node-gyp rebuild` at install time. The build can be skipped
|
||||
* via GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or it can
|
||||
* silently soft-fail when the C++ toolchain is missing.
|
||||
*
|
||||
* Either path produces the same observable: the .node binding is absent
|
||||
* at runtime. This helper detects that condition and surfaces a single
|
||||
* stderr line per missing grammar so users learn why .dart/.proto support
|
||||
* is unavailable instead of silently getting a degraded index.
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { cliWarn } from './cli-message.js';
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
|
||||
interface OptionalGrammar {
|
||||
/** Display name in warnings */
|
||||
name: string;
|
||||
/** Module name to require.resolve */
|
||||
pkg: string;
|
||||
/** File extensions this grammar parses */
|
||||
extensions: string[];
|
||||
}
|
||||
|
||||
const OPTIONAL_GRAMMARS: OptionalGrammar[] = [
|
||||
{ name: 'tree-sitter-dart', pkg: 'tree-sitter-dart', extensions: ['.dart'] },
|
||||
{ name: 'tree-sitter-proto', pkg: 'tree-sitter-proto', extensions: ['.proto'] },
|
||||
];
|
||||
|
||||
export interface MissingGrammar {
|
||||
name: string;
|
||||
extensions: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of optional grammars whose native binding cannot be
|
||||
* loaded. Actually `require()`s the package — `require.resolve` would
|
||||
* locate the entry path even when the `.node` binding is absent (the
|
||||
* `file:` package directory is installed regardless of postinstall
|
||||
* outcome), giving false negatives for the exact users we want to warn:
|
||||
* those who installed with `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` or whose
|
||||
* native rebuild soft-failed for missing toolchain.
|
||||
*
|
||||
* Node's module cache memoizes `require()` for us — calling this multiple
|
||||
* times is cheap. The catch distinguishes "missing" (MODULE_NOT_FOUND or
|
||||
* the typical node-gyp-build "could not find any binding" pattern) from
|
||||
* "broken" (SyntaxError, EACCES, native crash). Broken bindings surface a
|
||||
* separate stderr line so users get an actionable message instead of a
|
||||
* misleading "reinstall" hint.
|
||||
*/
|
||||
export function detectMissingOptionalGrammars(): MissingGrammar[] {
|
||||
const missing: MissingGrammar[] = [];
|
||||
for (const g of OPTIONAL_GRAMMARS) {
|
||||
try {
|
||||
_require(g.pkg);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const looksMissing =
|
||||
code === 'MODULE_NOT_FOUND' ||
|
||||
code === 'ERR_MODULE_NOT_FOUND' ||
|
||||
/could not find|no native build|prebuilds/i.test(msg);
|
||||
if (!looksMissing) {
|
||||
// Present but broken — surface so the user doesn't get a misleading
|
||||
// "reinstall" recovery message that wouldn't actually help. cliWarn
|
||||
// writes plain text to stderr AND tees a structured logger.warn
|
||||
// record; the merged repo-wide ESLint pino-migration rule forbids
|
||||
// direct `console.error` in CLI code (only `console.log` is allowed
|
||||
// there for tool-data stdout output).
|
||||
cliWarn(
|
||||
`GitNexus: optional grammar "${g.name}" is installed but failed to load (${msg.slice(0, 200)}). ${g.extensions.join('/')} files will not be parsed.`,
|
||||
{ grammar: g.name, extensions: g.extensions, error: msg },
|
||||
);
|
||||
}
|
||||
missing.push({ name: g.name, extensions: g.extensions });
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a one-line stderr warning for each missing grammar. Safe to call
|
||||
* unconditionally — silent if all grammars are present.
|
||||
*
|
||||
* `relevantExtensions`, if provided, filters the warning to grammars whose
|
||||
* extensions appear in the set (e.g. an analyze run can pass the set of
|
||||
* extensions actually present in the target repo so users without any
|
||||
* .dart/.proto files don't see noise).
|
||||
*/
|
||||
export function warnMissingOptionalGrammars(opts?: {
|
||||
context?: string;
|
||||
relevantExtensions?: ReadonlySet<string>;
|
||||
}): void {
|
||||
const missing = detectMissingOptionalGrammars();
|
||||
if (missing.length === 0) return;
|
||||
const ctx = opts?.context ? ` [${opts.context}]` : '';
|
||||
// Hoist the optional set into a local so the closure below can narrow
|
||||
// its type; references to `opts?.relevantExtensions` inside `.some()`
|
||||
// lose the outer null-check narrowing and require a non-null assertion.
|
||||
const relevantExtensions = opts?.relevantExtensions;
|
||||
for (const g of missing) {
|
||||
if (relevantExtensions && !g.extensions.some((e) => relevantExtensions.has(e))) {
|
||||
continue;
|
||||
}
|
||||
cliWarn(
|
||||
`GitNexus${ctx}: optional grammar "${g.name}" is unavailable — ${g.extensions.join('/')} files will not be parsed. Reinstall without GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (and ensure python3, make, g++) to enable.`,
|
||||
{ grammar: g.name, extensions: g.extensions, context: opts?.context },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* `gitnexus publish` — opt-in ping to the understand-quickly registry.
|
||||
*
|
||||
* Fires a single `repository_dispatch` event at
|
||||
* `looptech-ai/understand-quickly` so the registry knows to refresh its
|
||||
* entry for the current repo. Does NOT upload anything: per the
|
||||
* understand-quickly protocol, the registry pulls the graph from a
|
||||
* raw-GitHub URL the user controls.
|
||||
*
|
||||
* https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md
|
||||
*
|
||||
* Defaults:
|
||||
* - Without `UNDERSTAND_QUICKLY_TOKEN` in the env, this is a no-op
|
||||
* (prints one informational line, exit 0). Same shape as the
|
||||
* `--publish` patterns in sibling tools.
|
||||
* - With the token, fires the dispatch and reports the response code.
|
||||
*
|
||||
* The `id` is derived from the repo's `origin` remote unless the caller
|
||||
* passes `--id <owner/repo>` explicitly. We deliberately do NOT auto-add
|
||||
* the repo to the registry — registration is one-time and uses the
|
||||
* `npx @understand-quickly/cli add` path documented in the protocol.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import {
|
||||
UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
UNDERSTAND_QUICKLY_TOKEN_ENV,
|
||||
buildUqDispatchPayload,
|
||||
isValidOwnerRepo,
|
||||
parseOwnerRepoFromRemote,
|
||||
} from 'gitnexus-shared';
|
||||
import { getGitRoot, getRemoteOriginUrl, getCurrentCommit } from '../storage/git.js';
|
||||
import { hasIndex } from '../storage/repo-manager.js';
|
||||
import { cliInfo, cliError } from './cli-message.js';
|
||||
|
||||
export interface PublishOptions {
|
||||
/** Override the auto-derived `owner/repo` id. */
|
||||
id?: string;
|
||||
/** Treat the cwd as the repo root (skip git-root walk). */
|
||||
skipGit?: boolean;
|
||||
}
|
||||
|
||||
const REGISTER_HINT =
|
||||
'Register your repo once with: npx @understand-quickly/cli add\n' +
|
||||
'Or use the wizard: https://looptech-ai.github.io/understand-quickly/add.html';
|
||||
|
||||
/**
|
||||
* Hard cap on the dispatch fetch to keep CI publish steps from stalling
|
||||
* for the OS TCP timeout (~2 min) when api.github.com is unreachable.
|
||||
* Matches the pattern used in `src/core/embeddings/http-client.ts`.
|
||||
*/
|
||||
const DISPATCH_TIMEOUT_MS = 15_000;
|
||||
|
||||
export const publishCommand = async (
|
||||
inputPath?: string,
|
||||
options: PublishOptions = {},
|
||||
): Promise<void> => {
|
||||
// ── 0. Token gate FIRST — guarantees true no-op without the token. ──
|
||||
// The README, CLI --help, and PR body all promise "exit 0 without
|
||||
// UNDERSTAND_QUICKLY_TOKEN". Doing the index/repo-root checks before
|
||||
// the token gate would make those promises false for users who haven't
|
||||
// run `gitnexus analyze` yet but want to verify the command is wired.
|
||||
const token = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV];
|
||||
if (!token) {
|
||||
cliInfo(
|
||||
`[understand-quickly] ${UNDERSTAND_QUICKLY_TOKEN_ENV} is not set — skipping dispatch.\n` +
|
||||
`Set it to a fine-grained PAT with "Repository dispatches: write" on ` +
|
||||
`looptech-ai/understand-quickly to enable instant resync.\n` +
|
||||
`(Without the token, the registry's nightly sync still picks up your entry.)`,
|
||||
{ skipped: 'no-token' },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 1. Resolve the repo root (same precedence as `analyze`) ──────────
|
||||
let repoPath: string;
|
||||
if (inputPath) {
|
||||
repoPath = path.resolve(inputPath);
|
||||
} else if (options.skipGit) {
|
||||
repoPath = path.resolve(process.cwd());
|
||||
} else {
|
||||
const gitRoot = getGitRoot(process.cwd());
|
||||
if (!gitRoot) {
|
||||
cliError(
|
||||
'[understand-quickly] not inside a git repository.\n' +
|
||||
'Run from a repo, or pass --skip-git to publish from the current directory.',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
repoPath = gitRoot;
|
||||
}
|
||||
|
||||
// ── 2. Confirm a GitNexus index exists ───────────────────────────────
|
||||
// Publishing without an index is almost always a mistake — the
|
||||
// registry's nightly sync would fetch a stale or missing graph file
|
||||
// and mark the entry `missing`. Refuse loudly with a fix-it hint.
|
||||
if (!(await hasIndex(repoPath))) {
|
||||
cliError(
|
||||
`[understand-quickly] no GitNexus index found at ${repoPath}/.gitnexus.\n` +
|
||||
'Run `gitnexus analyze` first, then re-run `gitnexus publish`.',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 3. Derive the registry id ─────────────────────────────────────────
|
||||
const id =
|
||||
options.id ?? parseOwnerRepoFromRemote(getRemoteOriginUrl(repoPath) ?? undefined) ?? null;
|
||||
if (!id || !isValidOwnerRepo(id)) {
|
||||
cliError(
|
||||
`[understand-quickly] could not derive a registry id from this repo.\n` +
|
||||
`Pass --id <owner/repo> explicitly (e.g. --id looptech-ai/${path.basename(repoPath)}).\n` +
|
||||
REGISTER_HINT,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 4. Fire the dispatch ─────────────────────────────────────────────
|
||||
const payload = buildUqDispatchPayload(id);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(UNDERSTAND_QUICKLY_DISPATCH_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'gitnexus-cli',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(DISPATCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (err) {
|
||||
// `AbortSignal.timeout()` throws a `DOMException` with `name ===
|
||||
// 'TimeoutError'` on Node 18.14+ (and on browsers/Bun). It is NOT
|
||||
// a plain `AbortError`. Match the pattern used in
|
||||
// gitnexus/src/core/embeddings/http-client.ts so the user sees the
|
||||
// targeted "timed out" message instead of a generic "operation
|
||||
// was aborted".
|
||||
const isTimeout = err instanceof DOMException && err.name === 'TimeoutError';
|
||||
if (isTimeout) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch timed out after ${DISPATCH_TIMEOUT_MS}ms. ` +
|
||||
`Check network access to api.github.com and retry.`,
|
||||
{ id },
|
||||
);
|
||||
} else {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
cliError(`[understand-quickly] dispatch network error: ${msg}`, { id });
|
||||
}
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// GitHub returns 204 on success. Distinct branches for 401/403/404/422
|
||||
// so users debug without checking the docs.
|
||||
if (response.status === 204) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
// `getCurrentCommit` is only meaningful in the success path — moving
|
||||
// it inside this branch removes a wasted child-process spawn on every
|
||||
// error response (LOW 7).
|
||||
const commit = getCurrentCommit(repoPath);
|
||||
cliInfo(
|
||||
`[understand-quickly] dispatched sync-entry for ${id}` +
|
||||
(commit ? ` @ ${commit.slice(0, 7)}` : '') +
|
||||
'.\n' +
|
||||
`Note: a 204 only confirms GitHub accepted the dispatch. Whether the ` +
|
||||
`registry workflow finds an entry for "${id}" is logged at ` +
|
||||
`https://github.com/looptech-ai/understand-quickly/actions/workflows/sync.yml`,
|
||||
{ id, commit, status: response.status },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 401) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 401 — the ${UNDERSTAND_QUICKLY_TOKEN_ENV} value is invalid or expired.\n` +
|
||||
`Regenerate a fine-grained PAT at https://github.com/settings/personal-access-tokens ` +
|
||||
`with Repository access scoped to looptech-ai/understand-quickly and the ` +
|
||||
`"Repository dispatches: write" permission, then retry.`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 403) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 403 — the token authenticated but ` +
|
||||
`lacks the "Repository dispatches: write" permission on ` +
|
||||
`looptech-ai/understand-quickly. Edit the PAT scopes and retry.`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 404 — the token cannot reach ` +
|
||||
`looptech-ai/understand-quickly. Verify the PAT has Repository access to ` +
|
||||
`that exact repo (not just your own org).`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 422) {
|
||||
// Malformed event_type / client_payload — a code bug in this CLI,
|
||||
// not a user mistake. Surface so we get bug reports.
|
||||
const body422 = await response.text().catch(() => '');
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 422 (this is a CLI bug; please report).\n` +
|
||||
`Body: ${body422 || '(empty)'}`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// 5xx and anything else → bubble the body so the user has something to act on.
|
||||
const body = await response.text().catch(() => '');
|
||||
cliError(
|
||||
`[understand-quickly] dispatch failed with HTTP ${response.status}: ${body || '(empty body)'}`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
};
|
||||
@@ -27,6 +27,8 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import { logger } from '../core/logger.js';
|
||||
import { cliError } from './cli-message.js';
|
||||
import {
|
||||
readRegistry,
|
||||
resolveRegistryEntry,
|
||||
@@ -51,14 +53,14 @@ export const removeCommand = async (target: string, options?: { force?: boolean
|
||||
// Idempotent: missing target is a no-op warning, not an error.
|
||||
// The `availableNames` hint comes from the error itself so users
|
||||
// can see what they might have meant.
|
||||
console.warn(`Nothing to remove: ${err.message}`);
|
||||
logger.warn(`Nothing to remove: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
if (err instanceof RegistryAmbiguousTargetError) {
|
||||
// Duplicate aliases are allowed via --allow-duplicate-name (#829);
|
||||
// refuse to guess which one the user meant — surface the full list
|
||||
// and exit non-zero so scripts don't silently pick the wrong repo.
|
||||
console.error(`Error: ${err.message}`);
|
||||
cliError(`Error: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
throw err;
|
||||
@@ -86,7 +88,7 @@ export const removeCommand = async (target: string, options?: { force?: boolean
|
||||
assertSafeStoragePath(entry);
|
||||
} catch (err) {
|
||||
if (err instanceof UnsafeStoragePathError) {
|
||||
console.error(`Error: ${err.message}`);
|
||||
cliError(`Error: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
throw err;
|
||||
@@ -104,7 +106,8 @@ export const removeCommand = async (target: string, options?: { force?: boolean
|
||||
console.log(` Path: ${entry.path}`);
|
||||
console.log(` Storage: ${entry.storagePath}`);
|
||||
} catch (err) {
|
||||
console.error(`Failed to remove ${entry.name}:`, err);
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
cliError(`Failed to remove ${entry.name}: ${msg}`, { err });
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
+34
-12
@@ -1,14 +1,26 @@
|
||||
import { createServer } from '../server/api.js';
|
||||
import { logger, flushLoggerSync } from '../core/logger.js';
|
||||
import { cliError } from './cli-message.js';
|
||||
|
||||
// Catch anything that would cause a silent exit
|
||||
// Catch anything that would cause a silent exit. Pino v10's default
|
||||
// destination is `sync: false` (SonicBoom buffered) — call
|
||||
// `flushLoggerSync()` between the log and `process.exit(1)` so the crash
|
||||
// record is not lost to the unflushed buffer. Worker-thread transports
|
||||
// (pino-pretty under TTY) handle their own flush on process exit in v10,
|
||||
// so no separate `pino.final` integration is needed (the API was removed
|
||||
// in v10 because the transport architecture made it unnecessary).
|
||||
//
|
||||
// We pass the Error itself in `{ err }` so pino's built-in err serializer
|
||||
// captures `type`, `message`, and `stack` as structured fields.
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error('\n[gitnexus serve] Uncaught exception:', err.message);
|
||||
if (process.env.DEBUG) console.error(err.stack);
|
||||
logger.error({ err }, '[gitnexus serve] Uncaught exception');
|
||||
flushLoggerSync();
|
||||
process.exit(1);
|
||||
});
|
||||
process.on('unhandledRejection', (reason: any) => {
|
||||
console.error('\n[gitnexus serve] Unhandled rejection:', reason?.message || reason);
|
||||
if (process.env.DEBUG) console.error(reason?.stack);
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
const err = reason instanceof Error ? reason : new Error(String(reason));
|
||||
logger.error({ err }, '[gitnexus serve] Unhandled rejection');
|
||||
flushLoggerSync();
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
@@ -22,16 +34,26 @@ export const serveCommand = async (options?: { port?: string; host?: string }) =
|
||||
try {
|
||||
await createServer(port, host);
|
||||
} catch (err: any) {
|
||||
console.error(`\nFailed to start GitNexus server:\n`);
|
||||
console.error(` ${err.message || err}\n`);
|
||||
if (err.code === 'EADDRINUSE') {
|
||||
console.error(` Port ${port} is already in use. Either:`);
|
||||
console.error(` 1. Stop the other process using port ${port}`);
|
||||
console.error(` 2. Use a different port: gitnexus serve --port 4748\n`);
|
||||
cliError(
|
||||
`\nFailed to start GitNexus server:\n` +
|
||||
` ${err.message || err}\n\n` +
|
||||
` Port ${port} is already in use. Either:\n` +
|
||||
` 1. Stop the other process using port ${port}\n` +
|
||||
` 2. Use a different port: gitnexus serve --port 4748\n`,
|
||||
{ code: err.code, port, host },
|
||||
);
|
||||
} else {
|
||||
cliError(`\nFailed to start GitNexus server:\n ${err.message || err}\n`, {
|
||||
code: err.code,
|
||||
port,
|
||||
host,
|
||||
});
|
||||
}
|
||||
if (err.stack && process.env.DEBUG) {
|
||||
console.error(err.stack);
|
||||
logger.debug({ stack: err.stack }, 'serve start error stack');
|
||||
}
|
||||
flushLoggerSync();
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -10,6 +10,7 @@ import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { execFile, execFileSync } from 'child_process';
|
||||
import { createRequire } from 'module';
|
||||
import { promisify } from 'util';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { glob } from 'glob';
|
||||
@@ -20,6 +21,21 @@ const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
// Pin the npx fallback to the installed version. Reason: setup.ts writes
|
||||
// a config that persists in the user's editor and is invoked on every MCP
|
||||
// connect. Pinning to the installed version means subsequent invocations
|
||||
// skip the npm-registry metadata roundtrip (and stay reproducible until
|
||||
// the user upgrades). Static configs and READMEs intentionally use
|
||||
// `gitnexus@latest` since they're quickstart docs, not persisted state.
|
||||
const _require = createRequire(import.meta.url);
|
||||
const _pkg = _require('../../package.json') as { version?: unknown };
|
||||
if (typeof _pkg.version !== 'string' || !_pkg.version) {
|
||||
throw new Error(
|
||||
'gitnexus/package.json#version is missing or not a string — cannot generate MCP fallback config.',
|
||||
);
|
||||
}
|
||||
const NPX_REF = `gitnexus@${_pkg.version}`;
|
||||
|
||||
interface SetupResult {
|
||||
configured: string[];
|
||||
skipped: string[];
|
||||
@@ -62,8 +78,10 @@ function resolveGitnexusBin(): string | null {
|
||||
* The MCP server entry for all editors.
|
||||
*
|
||||
* Prefers the globally-installed `gitnexus` binary (starts in ~1 s) over
|
||||
* `npx -y gitnexus@latest` (cold-cache install of native deps can take
|
||||
* >60 s, exceeding Claude Code's 30 s MCP connection timeout).
|
||||
* `npx -y gitnexus@<version>` (cold-cache install of native deps can take
|
||||
* >60 s, exceeding Claude Code's 30 s MCP connection timeout). The fallback
|
||||
* version is read from gitnexus/package.json#version at module load so the
|
||||
* persisted user config matches the installed package.
|
||||
*
|
||||
* Falls back to npx when the binary isn't on PATH — e.g. first-time
|
||||
* users who ran `npx gitnexus analyze` but haven't done `npm i -g`.
|
||||
@@ -79,12 +97,12 @@ function getMcpEntry() {
|
||||
if (process.platform === 'win32') {
|
||||
return {
|
||||
command: 'cmd',
|
||||
args: ['/c', 'npx', '-y', 'gitnexus@latest', 'mcp'],
|
||||
args: ['/c', 'npx', '-y', NPX_REF, 'mcp'],
|
||||
};
|
||||
}
|
||||
return {
|
||||
command: 'npx',
|
||||
args: ['-y', 'gitnexus@latest', 'mcp'],
|
||||
args: ['-y', NPX_REF, 'mcp'],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -100,9 +118,9 @@ function getOpenCodeMcpEntry() {
|
||||
}
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
return { type: 'local', command: ['cmd', '/c', 'npx', '-y', 'gitnexus@latest', 'mcp'] };
|
||||
return { type: 'local', command: ['cmd', '/c', 'npx', '-y', NPX_REF, 'mcp'] };
|
||||
}
|
||||
return { type: 'local', command: ['npx', '-y', 'gitnexus@latest', 'mcp'] };
|
||||
return { type: 'local', command: ['npx', '-y', NPX_REF, 'mcp'] };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -347,7 +365,12 @@ async function installClaudeCodeHooks(result: SetupResult): Promise<void> {
|
||||
}
|
||||
|
||||
const hookPath = path.join(destHooksDir, 'gitnexus-hook.cjs').replace(/\\/g, '/');
|
||||
const hookCmd = `node "${hookPath.replace(/"/g, '\\"')}"`;
|
||||
// Escape backslashes FIRST, then quotes (CodeQL js/incomplete-sanitization).
|
||||
// The previous shape `replace(/"/g, '\\"')` alone would let `path\with"quote`
|
||||
// become `path\with\"quote`, where the trailing `\` before `"` could
|
||||
// unescape the quote inside the surrounding double-quoted shell context.
|
||||
const escapedHookPath = hookPath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
||||
const hookCmd = `node "${escapedHookPath}"`;
|
||||
|
||||
// Check which hook events need entries (idempotent: skip if already registered)
|
||||
const parsed = await (async () => {
|
||||
@@ -604,7 +627,7 @@ async function installOpenCodeSkills(result: SetupResult): Promise<void> {
|
||||
const installed = await installSkillsTo(skillsDir);
|
||||
if (installed.length > 0) {
|
||||
result.configured.push(
|
||||
`OpenCode skills (${installed.length} skills → ~/.config/opencode/skill/)`,
|
||||
`OpenCode skills (${installed.length} skills → ~/.config/opencode/skills/)`,
|
||||
);
|
||||
}
|
||||
} catch (err: any) {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
import { writeSync } from 'node:fs';
|
||||
import { LocalBackend } from '../mcp/local/local-backend.js';
|
||||
import { cliError } from './cli-message.js';
|
||||
|
||||
let _backend: LocalBackend | null = null;
|
||||
|
||||
@@ -25,7 +26,7 @@ async function getBackend(): Promise<LocalBackend> {
|
||||
_backend = new LocalBackend();
|
||||
const ok = await _backend.init();
|
||||
if (!ok) {
|
||||
console.error('GitNexus: No indexed repositories found. Run: gitnexus analyze');
|
||||
cliError('GitNexus: No indexed repositories found. Run: gitnexus analyze');
|
||||
process.exit(1);
|
||||
}
|
||||
return _backend;
|
||||
@@ -67,7 +68,7 @@ export async function queryCommand(
|
||||
},
|
||||
): Promise<void> {
|
||||
if (!queryText?.trim()) {
|
||||
console.error('Usage: gitnexus query <search_query>');
|
||||
cliError('Usage: gitnexus query <search_query>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -93,7 +94,7 @@ export async function contextCommand(
|
||||
},
|
||||
): Promise<void> {
|
||||
if (!name?.trim() && !options?.uid) {
|
||||
console.error('Usage: gitnexus context <symbol_name> [--uid <uid>] [--file <path>]');
|
||||
cliError('Usage: gitnexus context <symbol_name> [--uid <uid>] [--file <path>]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -118,7 +119,7 @@ export async function impactCommand(
|
||||
},
|
||||
): Promise<void> {
|
||||
if (!target?.trim()) {
|
||||
console.error('Usage: gitnexus impact <symbol_name> [--direction upstream|downstream]');
|
||||
cliError('Usage: gitnexus impact <symbol_name> [--direction upstream|downstream]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -153,7 +154,7 @@ export async function cypherCommand(
|
||||
},
|
||||
): Promise<void> {
|
||||
if (!query?.trim()) {
|
||||
console.error('Usage: gitnexus cypher <cypher_query>');
|
||||
cliError('Usage: gitnexus cypher <cypher_query>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
import { WikiGenerator, type WikiOptions } from '../core/wiki/generator.js';
|
||||
import { resolveLLMConfig, type LLMProvider } from '../core/wiki/llm-client.js';
|
||||
import { detectCursorCLI } from '../core/wiki/cursor-client.js';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
export interface WikiCommandOptions {
|
||||
force?: boolean;
|
||||
@@ -583,7 +584,7 @@ export const wikiCommand = async (inputPath?: string, options?: WikiCommandOptio
|
||||
} else {
|
||||
console.log(`\n Error: ${err.message}\n`);
|
||||
if (process.env.GITNEXUS_VERBOSE) {
|
||||
console.error(err);
|
||||
logger.error({ err }, 'wiki command failed');
|
||||
}
|
||||
}
|
||||
process.exitCode = 1;
|
||||
@@ -601,6 +602,38 @@ function hasGhCLI(): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict Gist URL predicate. Rejects:
|
||||
* - any URL that does not parse (URL constructor throws)
|
||||
* - schemes other than https (drops `http:`, `file:`, `gist:`-style spoofs)
|
||||
* - hostnames that are not exactly `gist.github.com` (drops substring spoofs
|
||||
* like `https://evil.com/?u=gist.github.com` and userinfo-prefixed shapes
|
||||
* like `https://[email protected]/...` — note that URL.hostname
|
||||
* strips userinfo, so the equality check rejects the userinfo-prefixed
|
||||
* spoof if the actual host differs from gist.github.com)
|
||||
* - any URL containing userinfo (`username[:password]@`), which the URL
|
||||
* parser exposes via `.username` / `.password`. Defense-in-depth: even
|
||||
* when hostname matches, a credential-bearing URL is suspect and not
|
||||
* produced by `gh gist create`.
|
||||
*
|
||||
* Closes the substring-bypass class CodeQL `js/incomplete-url-substring-
|
||||
* sanitization` flags.
|
||||
*/
|
||||
function isGistUrl(line: string): boolean {
|
||||
const trimmed = line.trim();
|
||||
try {
|
||||
const u = new URL(trimmed);
|
||||
return (
|
||||
u.protocol === 'https:' &&
|
||||
u.hostname === 'gist.github.com' &&
|
||||
u.username === '' &&
|
||||
u.password === ''
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
|
||||
try {
|
||||
const output = execFileSync(
|
||||
@@ -609,13 +642,14 @@ function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
|
||||
{ encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
).trim();
|
||||
|
||||
// gh gist create prints the gist URL as the last line
|
||||
const lines = output.split('\n');
|
||||
const gistUrl = lines.find((l) => l.includes('gist.github.com')) || lines[lines.length - 1];
|
||||
// `gh gist create` prints the gist URL as a line in the output. Find the
|
||||
// first parseable Gist URL — if no line is a valid Gist URL, fail closed
|
||||
// (do NOT fall back to lines[last]: a non-Gist last line would propagate
|
||||
// through the regex below and produce a malformed `rawUrl`).
|
||||
const gistUrl = output.split('\n').find(isGistUrl);
|
||||
if (!gistUrl) return null;
|
||||
|
||||
if (!gistUrl || !gistUrl.includes('gist.github.com')) return null;
|
||||
|
||||
// Build a raw viewer URL via gist.githack.com
|
||||
// Build a raw viewer URL via gist.githack.com.
|
||||
// gist URL format: https://gist.github.com/{user}/{id}
|
||||
const match = gistUrl.match(/gist\.github\.com\/([^/]+)\/([a-f0-9]+)/);
|
||||
let rawUrl = gistUrl;
|
||||
|
||||
@@ -2,6 +2,7 @@ import ignore, { type Ignore } from 'ignore';
|
||||
import fs from 'fs/promises';
|
||||
import nodePath from 'path';
|
||||
import type { Path } from 'path-scurry';
|
||||
import { logger } from '../core/logger.js';
|
||||
|
||||
const DEFAULT_IGNORE_LIST = new Set([
|
||||
// Version Control
|
||||
@@ -24,6 +25,8 @@ const DEFAULT_IGNORE_LIST = new Set([
|
||||
'bower_components',
|
||||
'jspm_packages',
|
||||
'vendor', // PHP/Go
|
||||
'third_party', // C/C++ (Google-style vendored dependencies)
|
||||
'3rdparty', // C/C++ (alternate spelling, also Qt convention)
|
||||
// 'packages' removed - commonly used for monorepo source code (lerna, pnpm, yarn workspaces)
|
||||
'venv',
|
||||
'.venv',
|
||||
@@ -365,7 +368,7 @@ export const loadIgnoreRules = async (
|
||||
} catch (err: unknown) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code !== 'ENOENT') {
|
||||
console.warn(` Warning: could not read ${filename}: ${(err as Error).message}`);
|
||||
logger.warn(` Warning: could not read ${filename}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,7 +104,7 @@ export async function augment(pattern: string, cwd?: string): Promise<string> {
|
||||
}
|
||||
|
||||
// Step 1: BM25 search (fast, no embeddings)
|
||||
const bm25Results = await searchFTSFromLbug(pattern, 10, repoId);
|
||||
const { results: bm25Results } = await searchFTSFromLbug(pattern, 10, repoId);
|
||||
|
||||
if (bm25Results.length === 0) return '';
|
||||
|
||||
|
||||
@@ -30,6 +30,38 @@ export interface EmbeddingMode {
|
||||
shouldLoadCache: boolean;
|
||||
}
|
||||
|
||||
/** Default safety cap on graph node count for embedding generation. */
|
||||
export const DEFAULT_EMBEDDING_NODE_LIMIT = 50_000;
|
||||
|
||||
export interface EmbeddingCapDecision {
|
||||
/** True when the node-count cap blocks generation for this graph. */
|
||||
skipForCap: boolean;
|
||||
/** True when the user explicitly disabled the cap (`--embeddings 0`). */
|
||||
capDisabled: boolean;
|
||||
/** Effective node limit applied (`0` means disabled). */
|
||||
nodeLimit: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether the node-count safety cap blocks embedding generation.
|
||||
*
|
||||
* - `embeddingsNodeLimit === undefined` → use {@link DEFAULT_EMBEDDING_NODE_LIMIT}
|
||||
* - `embeddingsNodeLimit === 0` → cap disabled, generation always proceeds
|
||||
* - any positive integer → custom cap (skip if `nodeCount > limit`)
|
||||
*
|
||||
* Lives in `embedding-mode.ts` (not `run-analyze.ts`) so the branching
|
||||
* contract is unit-testable without spinning up LadybugDB or the pipeline.
|
||||
*/
|
||||
export function deriveEmbeddingCap(
|
||||
nodeCount: number,
|
||||
embeddingsNodeLimit: number | undefined,
|
||||
): EmbeddingCapDecision {
|
||||
const nodeLimit = embeddingsNodeLimit ?? DEFAULT_EMBEDDING_NODE_LIMIT;
|
||||
const capDisabled = nodeLimit === 0;
|
||||
const skipForCap = !capDisabled && nodeCount > nodeLimit;
|
||||
return { skipForCap, capDisabled, nodeLimit };
|
||||
}
|
||||
|
||||
export function deriveEmbeddingMode(
|
||||
options: EmbeddingModeInput,
|
||||
existingEmbeddingCount: number,
|
||||
|
||||
@@ -14,7 +14,12 @@ if (!process.env.ORT_LOG_LEVEL) {
|
||||
process.env.ORT_LOG_LEVEL = '3';
|
||||
}
|
||||
|
||||
import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
||||
import {
|
||||
pipeline,
|
||||
env,
|
||||
type FeatureExtractionPipeline,
|
||||
type ProgressInfo,
|
||||
} from '@huggingface/transformers';
|
||||
import { existsSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { join, dirname } from 'path';
|
||||
@@ -22,7 +27,8 @@ import { createRequire } from 'module';
|
||||
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
|
||||
import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
|
||||
import { resolveEmbeddingConfig } from './config.js';
|
||||
import { applyHfEnvOverrides } from './hf-env.js';
|
||||
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
/**
|
||||
* Check whether the onnxruntime-node package that @huggingface/transformers
|
||||
@@ -166,17 +172,22 @@ export const initEmbedder = async (
|
||||
|
||||
const isDev = process.env.NODE_ENV === 'development';
|
||||
if (isDev) {
|
||||
console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);
|
||||
logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`);
|
||||
}
|
||||
|
||||
const progressCallback = onProgress
|
||||
? (data: any) => {
|
||||
? (data: ProgressInfo) => {
|
||||
const progress: ModelProgress = {
|
||||
status: data.status || 'progress',
|
||||
file: data.file,
|
||||
progress: data.progress,
|
||||
loaded: data.loaded,
|
||||
total: data.total,
|
||||
// Map the `progress_total` aggregate event (not in ModelProgress.status)
|
||||
// back to 'progress' so callers don't need to handle it separately.
|
||||
status:
|
||||
data.status === 'progress_total'
|
||||
? 'progress'
|
||||
: ((data.status as ModelProgress['status']) ?? 'progress'),
|
||||
file: 'file' in data ? data.file : undefined,
|
||||
progress: 'progress' in data ? data.progress : undefined,
|
||||
loaded: 'loaded' in data ? data.loaded : undefined,
|
||||
total: 'total' in data ? data.total : undefined,
|
||||
};
|
||||
onProgress(progress);
|
||||
}
|
||||
@@ -192,26 +203,38 @@ export const initEmbedder = async (
|
||||
for (const device of devicesToTry) {
|
||||
try {
|
||||
if (isDev && device === 'dml') {
|
||||
console.log('🔧 Trying DirectML (DirectX12) GPU backend...');
|
||||
logger.info('🔧 Trying DirectML (DirectX12) GPU backend...');
|
||||
} else if (isDev && device === 'cuda') {
|
||||
console.log('🔧 Trying CUDA GPU backend...');
|
||||
logger.info('🔧 Trying CUDA GPU backend...');
|
||||
} else if (isDev && device === 'cpu') {
|
||||
console.log('🔧 Using CPU backend...');
|
||||
logger.info('🔧 Using CPU backend...');
|
||||
} else if (isDev && device === 'wasm') {
|
||||
console.log('🔧 Using WASM backend (slower)...');
|
||||
logger.info('🔧 Using WASM backend (slower)...');
|
||||
}
|
||||
|
||||
embedderInstance = await (pipeline as any)('feature-extraction', finalConfig.modelId, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
progress_callback: progressCallback,
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: finalConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
embedderInstance = await withHfDownloadRetry(
|
||||
() =>
|
||||
pipeline('feature-extraction', finalConfig.modelId, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
progress_callback: progressCallback,
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: finalConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
},
|
||||
}),
|
||||
{
|
||||
onRetry: isDev
|
||||
? (attempt, max, err) =>
|
||||
logger.warn(
|
||||
{ attempt, max, err: err.message },
|
||||
`⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`,
|
||||
)
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
);
|
||||
currentDevice = device;
|
||||
|
||||
if (isDev) {
|
||||
@@ -221,15 +244,29 @@ export const initEmbedder = async (
|
||||
: device === 'cuda'
|
||||
? 'GPU (CUDA)'
|
||||
: device.toUpperCase();
|
||||
console.log(`✅ Using ${label} backend`);
|
||||
console.log('✅ Embedding model loaded successfully');
|
||||
logger.info(`✅ Using ${label} backend`);
|
||||
logger.info('✅ Embedding model loaded successfully');
|
||||
}
|
||||
|
||||
return embedderInstance!;
|
||||
} catch (deviceError) {
|
||||
// Network errors and circuit-open errors are not device-specific —
|
||||
// they will fail the same way on every device. Rethrow immediately
|
||||
// with actionable HF_ENDPOINT guidance rather than silently falling
|
||||
// back to the next device.
|
||||
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
|
||||
if (isHfDownloadFailure(errMsg)) {
|
||||
const endpointHint = process.env.HF_ENDPOINT
|
||||
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
|
||||
: `huggingface.co may be unreachable from your network.\n` +
|
||||
` Set HF_ENDPOINT to a mirror and retry:\n` +
|
||||
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
|
||||
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
|
||||
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
|
||||
}
|
||||
if (isDev && (device === 'cuda' || device === 'dml')) {
|
||||
const gpuType = device === 'dml' ? 'DirectML' : 'CUDA';
|
||||
console.log(`⚠️ ${gpuType} not available, falling back to CPU...`);
|
||||
logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`);
|
||||
}
|
||||
// Continue to next device in list
|
||||
if (device === devicesToTry[devicesToTry.length - 1]) {
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
} from '../lbug/schema.js';
|
||||
import { loadVectorExtension } from '../lbug/lbug-adapter.js';
|
||||
import { getExactScanLimit } from '../platform/capabilities.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
const isDev = process.env.NODE_ENV === 'development';
|
||||
|
||||
@@ -157,7 +158,7 @@ const queryEmbeddableNodes = async (
|
||||
}
|
||||
} catch (error) {
|
||||
if (isDev) {
|
||||
console.warn(`Query for ${label} nodes failed:`, error);
|
||||
logger.warn({ error }, `Query for ${label} nodes failed:`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,7 +213,7 @@ const createVectorIndex = async (
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isDev) {
|
||||
console.warn('Vector index creation warning:', error);
|
||||
logger.warn({ error }, 'Vector index creation warning:');
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -256,7 +257,9 @@ export const runEmbeddingPipeline = async (
|
||||
|
||||
try {
|
||||
const vectorAvailable = await ensureVectorExtensionAvailable();
|
||||
if (!vectorAvailable && isDev) console.warn(vectorUnavailableMessage);
|
||||
if (!vectorAvailable && isDev) {
|
||||
logger.warn(vectorUnavailableMessage);
|
||||
}
|
||||
|
||||
// Phase 1: Load embedding model
|
||||
onProgress({
|
||||
@@ -283,7 +286,7 @@ export const runEmbeddingPipeline = async (
|
||||
});
|
||||
|
||||
if (isDev) {
|
||||
console.log('🔍 Querying embeddable nodes...');
|
||||
logger.info('🔍 Querying embeddable nodes...');
|
||||
}
|
||||
|
||||
// Phase 2: Query embeddable nodes
|
||||
@@ -325,7 +328,7 @@ export const runEmbeddingPipeline = async (
|
||||
// (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
|
||||
if (staleNodeIds.length > 0) {
|
||||
if (isDev) {
|
||||
console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
|
||||
logger.info(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
|
||||
}
|
||||
try {
|
||||
await executeWithReusedStatement(
|
||||
@@ -346,7 +349,7 @@ export const runEmbeddingPipeline = async (
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`,
|
||||
);
|
||||
}
|
||||
@@ -355,7 +358,7 @@ export const runEmbeddingPipeline = async (
|
||||
const totalNodes = nodes.length;
|
||||
|
||||
if (isDev) {
|
||||
console.log(`📊 Found ${totalNodes} embeddable nodes`);
|
||||
logger.info(`📊 Found ${totalNodes} embeddable nodes`);
|
||||
}
|
||||
|
||||
if (totalNodes === 0) {
|
||||
@@ -442,9 +445,9 @@ export const runEmbeddingPipeline = async (
|
||||
);
|
||||
} catch (chunkErr) {
|
||||
if (isDev) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
{ chunkErr },
|
||||
`⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`,
|
||||
chunkErr,
|
||||
);
|
||||
}
|
||||
chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap);
|
||||
@@ -482,9 +485,9 @@ export const runEmbeddingPipeline = async (
|
||||
try {
|
||||
embeddings = await embedBatch(subTexts);
|
||||
} catch (embedErr) {
|
||||
console.error(
|
||||
logger.error(
|
||||
{ embedErr },
|
||||
`❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`,
|
||||
embedErr,
|
||||
);
|
||||
throw embedErr;
|
||||
}
|
||||
@@ -520,7 +523,7 @@ export const runEmbeddingPipeline = async (
|
||||
});
|
||||
|
||||
if (isDev) {
|
||||
console.log('📇 Creating vector index...');
|
||||
logger.info('📇 Creating vector index...');
|
||||
}
|
||||
|
||||
const vectorIndexReady = await createVectorIndex(executeQuery);
|
||||
@@ -533,7 +536,7 @@ export const runEmbeddingPipeline = async (
|
||||
});
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`,
|
||||
);
|
||||
}
|
||||
@@ -547,7 +550,7 @@ export const runEmbeddingPipeline = async (
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
|
||||
if (isDev) {
|
||||
console.error('❌ Embedding pipeline error:', error);
|
||||
logger.error({ error }, '❌ Embedding pipeline error:');
|
||||
}
|
||||
|
||||
onProgress({
|
||||
|
||||
@@ -1,6 +1,27 @@
|
||||
import os from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { CircuitBreaker, withRetry } from 'gitnexus-shared';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Download resilience defaults
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Per-attempt timeout for the full model download (5 minutes). */
|
||||
export const HF_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
/** Maximum total download attempts (1 initial + N-1 retries). */
|
||||
export const HF_MAX_ATTEMPTS = 3;
|
||||
/** Initial delay between retry attempts; doubles on each subsequent retry. */
|
||||
export const HF_BASE_DELAY_MS = 2_000;
|
||||
/** Number of consecutive failures required to open the circuit. */
|
||||
export const CB_FAILURE_THRESHOLD = 3;
|
||||
/** How long the circuit stays open before transitioning to half-open. */
|
||||
export const CB_RESET_TIMEOUT_MS = 60_000;
|
||||
/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */
|
||||
export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000;
|
||||
/** Upper bound clamped on the env-override attempt count. */
|
||||
export const HF_MAX_ATTEMPTS_CAP = 10;
|
||||
|
||||
/**
|
||||
* @internal Exported only for unit tests and the two embedder entry points
|
||||
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
|
||||
@@ -60,3 +81,234 @@ export function applyHfEnvOverrides(env: HfEnvSubset): void {
|
||||
env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal Exported for unit tests and the two embedder entry points.
|
||||
*
|
||||
* Returns true when an error message indicates a network-level fetch failure
|
||||
* during HuggingFace model download (e.g. `TypeError: fetch failed`,
|
||||
* `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`).
|
||||
*
|
||||
* These errors are not device-specific and cannot be fixed by falling back to
|
||||
* a different ONNX device — the caller should rethrow immediately with
|
||||
* guidance about `HF_ENDPOINT`.
|
||||
*/
|
||||
export function isNetworkFetchError(message: string): boolean {
|
||||
return (
|
||||
message.includes('fetch failed') ||
|
||||
message.includes('ECONNREFUSED') ||
|
||||
message.includes('ENOTFOUND') ||
|
||||
message.includes('ETIMEDOUT') ||
|
||||
message.includes('ECONNRESET')
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Circuit breaker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */
|
||||
export const CIRCUIT_OPEN_TAG = 'hf-circuit-open';
|
||||
|
||||
/**
|
||||
* Module-level singleton shared by both embedder entry points
|
||||
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process
|
||||
* only — not persisted across restarts. Backed by the shared
|
||||
* `CircuitBreaker` from `gitnexus-shared` (same state machine, same
|
||||
* semantics, plus the single-permit half-open gate that prevents
|
||||
* recovery-time stampedes).
|
||||
*/
|
||||
export const hfDownloadCircuit = new CircuitBreaker({
|
||||
failureThreshold: CB_FAILURE_THRESHOLD,
|
||||
cooldownMs: CB_RESET_TIMEOUT_MS,
|
||||
key: 'hf-download',
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Retry + timeout wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** @internal Returns true for errors that should abort without retry (circuit-open). */
|
||||
export function isHfCircuitOpenError(message: string): boolean {
|
||||
return message.includes(CIRCUIT_OPEN_TAG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true for any HuggingFace download failure that warrants showing the
|
||||
* `HF_ENDPOINT` remediation hint: either a raw network error or a
|
||||
* circuit-open rejection (which itself was caused by repeated network errors).
|
||||
*/
|
||||
export function isHfDownloadFailure(message: string): boolean {
|
||||
return isNetworkFetchError(message) || isHfCircuitOpenError(message);
|
||||
}
|
||||
|
||||
/** @internal Wraps `fn` in a hard time-limit. The timeout error contains
|
||||
* `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly.
|
||||
*/
|
||||
export function withDownloadTimeout<T>(fn: () => Promise<T>, timeoutMs: number): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
`ETIMEDOUT: model download timed out after ${Math.round(timeoutMs / 1000)}s — ` +
|
||||
`check your network speed or set HF_ENDPOINT to a faster mirror`,
|
||||
),
|
||||
),
|
||||
timeoutMs,
|
||||
);
|
||||
fn().then(
|
||||
(v) => {
|
||||
clearTimeout(timer);
|
||||
resolve(v);
|
||||
},
|
||||
(e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export interface HfRetryOptions {
|
||||
/** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */
|
||||
maxAttempts?: number;
|
||||
/** Delay before the first retry; doubles on each subsequent attempt (default: `HF_BASE_DELAY_MS`). */
|
||||
baseDelayMs?: number;
|
||||
/** Per-attempt wall-clock timeout in ms (default: `HF_DOWNLOAD_TIMEOUT_MS`). */
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* Circuit-breaker instance to use. Defaults to the module-level
|
||||
* `hfDownloadCircuit` singleton. Pass a fresh instance in tests.
|
||||
*/
|
||||
circuit?: CircuitBreaker;
|
||||
/**
|
||||
* Optional callback invoked before each retry (not the initial attempt).
|
||||
* @param attempt - 1-based retry number
|
||||
* @param max - total allowed attempts
|
||||
* @param error - the error that triggered the retry
|
||||
*/
|
||||
onRetry?: (attempt: number, max: number, error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry wrapper for HuggingFace model downloads with per-attempt timeout and
|
||||
* circuit-breaker protection.
|
||||
*
|
||||
* Behaviour:
|
||||
* - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG`
|
||||
* message (so `isHfDownloadFailure` still returns true and the caller can
|
||||
* show `HF_ENDPOINT` guidance).
|
||||
* - Each attempt is wrapped in `withDownloadTimeout`.
|
||||
* - On a network-level error (`isNetworkFetchError`) the attempt is retried
|
||||
* with exponential back-off; non-network errors (e.g. ONNX device failure)
|
||||
* are rethrown immediately without retry.
|
||||
* - Every network failure is recorded on the circuit breaker; a success resets
|
||||
* it.
|
||||
* - After all attempts are exhausted, the last network error is rethrown
|
||||
* so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in
|
||||
* the calling code still fire.
|
||||
*/
|
||||
export async function withHfDownloadRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: HfRetryOptions = {},
|
||||
): Promise<T> {
|
||||
// Resolve effective values — explicit options take precedence over env vars,
|
||||
// which take precedence over built-in defaults. This lets users lower the
|
||||
// per-attempt timeout without rebuilding (e.g.
|
||||
// HF_DOWNLOAD_TIMEOUT_MS=60000 npx gitnexus analyze --embeddings
|
||||
// reduces the worst-case wait from 15 minutes to ~3 minutes).
|
||||
//
|
||||
// Upper bounds are clamped to prevent accidental runaway configuration:
|
||||
// - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min)
|
||||
// - maxAttempts is floored (fractional values → integer) and capped at
|
||||
// HF_MAX_ATTEMPTS_CAP (10). Values ≤ 0, NaN, or Infinity fall back to
|
||||
// the built-in defaults.
|
||||
const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
|
||||
const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
|
||||
const resolvedTimeout =
|
||||
Number.isFinite(envTimeout) && envTimeout > 0
|
||||
? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
|
||||
: HF_DOWNLOAD_TIMEOUT_MS;
|
||||
const resolvedMaxAttempts =
|
||||
Number.isFinite(envMaxAttempts) && envMaxAttempts > 0
|
||||
? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP)
|
||||
: HF_MAX_ATTEMPTS;
|
||||
const {
|
||||
maxAttempts = resolvedMaxAttempts,
|
||||
baseDelayMs = HF_BASE_DELAY_MS,
|
||||
timeoutMs = resolvedTimeout,
|
||||
circuit = hfDownloadCircuit,
|
||||
onRetry,
|
||||
} = options;
|
||||
if (circuit.getState() === 'open') {
|
||||
// Compute remaining cooldown without consuming a probe permit.
|
||||
const openedAt = circuit.getOpenedAt();
|
||||
const secsUntilReset =
|
||||
openedAt !== null ? Math.ceil((circuit.getCooldownMs() - (Date.now() - openedAt)) / 1000) : 0;
|
||||
throw new Error(
|
||||
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` +
|
||||
(secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''),
|
||||
);
|
||||
}
|
||||
|
||||
// Retry budget delegated to `withRetry` from gitnexus-shared. The
|
||||
// HF-specific bits — per-attempt timeout, network-vs-non-network
|
||||
// classification, circuit-breaker recording, onRetry callback — wire
|
||||
// through the `isRetryable` callback. `circuitTripped` is the
|
||||
// sentinel that lets us replace the final thrown error with a
|
||||
// CIRCUIT_OPEN_TAG message when the breaker tripped mid-loop.
|
||||
let circuitTripped = false;
|
||||
|
||||
try {
|
||||
return await withRetry(
|
||||
async () => {
|
||||
const result = await withDownloadTimeout(fn, timeoutMs);
|
||||
circuit.recordSuccess();
|
||||
return result;
|
||||
},
|
||||
{
|
||||
maxAttempts,
|
||||
baseDelayMs,
|
||||
// Disable the cap to match the bespoke pure-exponential
|
||||
// progression. With the default `HF_MAX_ATTEMPTS_CAP = 10` and
|
||||
// `baseDelayMs = 2000`, the largest possible delay is
|
||||
// `2000 * 2^9 = ~17 minutes` — bounded enough not to need a cap.
|
||||
capDelayMs: Number.MAX_SAFE_INTEGER,
|
||||
isRetryable: (err, attempt) => {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (!isNetworkFetchError(error.message)) {
|
||||
// Non-network error (e.g. CUDA unavailable) — propagate
|
||||
// without retry. Use recordNeutral so the breaker's existing
|
||||
// failure-count progress isn't reset by a non-network failure
|
||||
// that says nothing about the CDN's health.
|
||||
circuit.recordNeutral();
|
||||
return { retry: false };
|
||||
}
|
||||
circuit.recordFailure();
|
||||
if (circuit.getState() === 'open') {
|
||||
// Circuit just tripped — fail fast, no more retries.
|
||||
circuitTripped = true;
|
||||
return { retry: false };
|
||||
}
|
||||
// Mirror the bespoke onRetry contract: fire only when there's
|
||||
// actually a next attempt.
|
||||
if (attempt + 1 < maxAttempts) {
|
||||
onRetry?.(attempt + 1, maxAttempts, error);
|
||||
}
|
||||
return { retry: true };
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
if (circuitTripped) {
|
||||
throw new Error(
|
||||
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${CB_FAILURE_THRESHOLD} consecutive failures`,
|
||||
);
|
||||
}
|
||||
// All retries exhausted — rethrow the last network error so
|
||||
// isNetworkFetchError patterns in the calling code still match and
|
||||
// surface HF_ENDPOINT guidance.
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,13 +3,22 @@
|
||||
*
|
||||
* Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints.
|
||||
* Imported by both the core embedder (batch) and MCP embedder (query).
|
||||
*
|
||||
* Network resilience is delegated to `resilientFetch` from
|
||||
* `gitnexus-shared` — bounded retries with exponential-backoff jitter,
|
||||
* `Retry-After` honored on 429, and an in-process circuit breaker that
|
||||
* fails fast on a flapping endpoint. Per-attempt timeout is enforced
|
||||
* via `AbortSignal.timeout` on the underlying fetch.
|
||||
*/
|
||||
|
||||
import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
const HTTP_TIMEOUT_MS = 30_000;
|
||||
const HTTP_MAX_RETRIES = 2;
|
||||
const HTTP_RETRY_BACKOFF_MS = 1_000;
|
||||
const HTTP_BATCH_SIZE = 64;
|
||||
const DEFAULT_DIMS = 384;
|
||||
const HTTP_BREAKER_KEY = 'embeddings-http';
|
||||
|
||||
interface HttpConfig {
|
||||
baseUrl: string;
|
||||
@@ -90,46 +99,51 @@ const httpEmbedBatch = async (
|
||||
model: string,
|
||||
apiKey: string,
|
||||
batchIndex = 0,
|
||||
attempt = 0,
|
||||
): Promise<EmbeddingItem[]> => {
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
resp = await resilientFetch(
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({ input: batch, model }),
|
||||
},
|
||||
body: JSON.stringify({ input: batch, model }),
|
||||
});
|
||||
{
|
||||
breakerKey: HTTP_BREAKER_KEY,
|
||||
retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS },
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
// Timeouts should not be retried — the server is unresponsive.
|
||||
// AbortSignal.timeout() throws DOMException with name 'TimeoutError'.
|
||||
const isTimeout = err instanceof DOMException && err.name === 'TimeoutError';
|
||||
if (isTimeout) {
|
||||
if (err instanceof CircuitOpenError) {
|
||||
throw new Error(
|
||||
`Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`,
|
||||
);
|
||||
}
|
||||
if (err instanceof DOMException && err.name === 'TimeoutError') {
|
||||
throw new Error(
|
||||
`Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
// DNS, connection errors — retry with backoff
|
||||
if (attempt < HTTP_MAX_RETRIES) {
|
||||
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
||||
if (err instanceof ResilientFetchExhaustedError) {
|
||||
throw new Error(
|
||||
`Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`);
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
const status = resp.status;
|
||||
if ((status === 429 || status >= 500) && attempt < HTTP_MAX_RETRIES) {
|
||||
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
||||
}
|
||||
throw new Error(`Embedding endpoint returned ${status} (${safeUrl(url)}, batch ${batchIndex})`);
|
||||
// resilientFetch already retried 5xx/429; any non-OK response here is
|
||||
// a terminal client error (4xx other than 429).
|
||||
throw new Error(
|
||||
`Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await resp.json()) as { data: EmbeddingItem[] };
|
||||
|
||||
@@ -3,11 +3,14 @@
|
||||
* Lives in core/ so application code does not depend on the MCP package layer.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { execFile, execFileSync } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import path from 'path';
|
||||
import { readRegistry, type RegistryEntry, type CwdMatch } from '../storage/repo-manager.js';
|
||||
import { findGitRootByDotGit, getCurrentCommit, getRemoteUrl } from '../storage/git.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface StalenessInfo {
|
||||
isStale: boolean;
|
||||
commitsBehind: number;
|
||||
@@ -41,6 +44,39 @@ export function checkStaleness(repoPath: string, lastCommit: string): StalenessI
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Async variant of {@link checkStaleness} — spawns git as a child process
|
||||
* instead of blocking the event loop. Used by `listRepos()` to check many
|
||||
* repos in parallel (issue #1363: 200 repos × sync spawn ≈ 50 s).
|
||||
*/
|
||||
export async function checkStalenessAsync(
|
||||
repoPath: string,
|
||||
lastCommit: string,
|
||||
): Promise<StalenessInfo> {
|
||||
try {
|
||||
// Note: promisified execFile captures stdout/stderr by default (no stdio option needed,
|
||||
// unlike the sync variant which requires explicit stdio: ['pipe','pipe','pipe']).
|
||||
const { stdout } = await execFileAsync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], {
|
||||
cwd: repoPath,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
|
||||
const commitsBehind = parseInt(stdout.trim(), 10) || 0;
|
||||
|
||||
if (commitsBehind > 0) {
|
||||
return {
|
||||
isStale: true,
|
||||
commitsBehind,
|
||||
hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`,
|
||||
};
|
||||
}
|
||||
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
} catch {
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a sibling-clone HEAD against an indexed `lastCommit`. Returns
|
||||
* `undefined` when the indexed commit is not reachable from the sibling
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import type { LbugValue } from '@ladybugdb/core';
|
||||
import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js';
|
||||
@@ -11,6 +11,9 @@ import {
|
||||
type LbugConnectionHandle,
|
||||
} from '../lbug/lbug-config.js';
|
||||
import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
|
||||
import { createLogger } from '../logger.js';
|
||||
|
||||
const bridgeLogger = createLogger('bridge-db', { debugEnvVar: 'GITNEXUS_DEBUG_BRIDGE' });
|
||||
|
||||
/**
|
||||
* Sidecar files that LadybugDB creates next to a `bridge.lbug` file.
|
||||
@@ -24,7 +27,7 @@ import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
|
||||
* - `.shadow` — non-blocking concurrent checkpoint sidecar (added in
|
||||
* LadybugDB 0.15.4); same pairing constraint as `.wal`.
|
||||
*
|
||||
* `bridge-db` writes to a `bridge.lbug.tmp` file and then atomically renames
|
||||
* `bridge-db` writes to a `bridge.lbug.tmp.<random>` file and then atomically renames
|
||||
* it into place. The rename only moves the main file; sidecars must be
|
||||
* cleaned up explicitly or the next writer trips the database-id check.
|
||||
*/
|
||||
@@ -276,8 +279,24 @@ export async function retryRename(src: string, dst: string, attempts = 3): Promi
|
||||
|
||||
export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise<void> {
|
||||
const target = path.join(groupDir, 'meta.json');
|
||||
const tmp = `${target}.tmp.${Date.now()}`;
|
||||
await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8');
|
||||
// Unpredictable suffix + O_EXCL via `'wx'` flag closes the symlink/
|
||||
// pre-create attack window. The third argument `0o600` is the
|
||||
// user-only mode mask — CodeQL's `js/insecure-temporary-file` query
|
||||
// sources its verdict from the `mode` argument, NOT from `flags`:
|
||||
// its `isSecureMode(mode)` predicate requires the low 6 bits to be
|
||||
// zero (no group/world bits). Without an explicit mode the file is
|
||||
// created with the process umask (typically 0o644 = group/world
|
||||
// readable), which the query treats as the actual vulnerability.
|
||||
// Both `'wx'` (runtime O_EXCL) AND `0o600` (CodeQL-credited mode)
|
||||
// are needed: one closes the symlink race, the other closes the
|
||||
// permissions exposure.
|
||||
const tmp = `${target}.tmp.${randomBytes(8).toString('hex')}`;
|
||||
const handle = await fsp.open(tmp, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(JSON.stringify(meta, null, 2), 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
// Use retryRename for consistency with writeBridge's atomic swap — on
|
||||
// Windows a concurrent reader can cause EBUSY/EPERM even on a tiny
|
||||
// meta.json, and we don't want meta write to be less robust than the
|
||||
@@ -346,7 +365,19 @@ export async function writeBridge(
|
||||
const crossLinks = dedupeCrossLinks(input.crossLinks);
|
||||
|
||||
const finalPath = path.join(groupDir, 'bridge.lbug');
|
||||
const tmpPath = path.join(groupDir, 'bridge.lbug.tmp');
|
||||
// Stage the temp database inside a unique mkdtemp directory rather than
|
||||
// a fixed `bridge.lbug.tmp` name. The previous shape was flagged by
|
||||
// CodeQL js/insecure-temporary-file as a predictable path: a co-located
|
||||
// attacker (or a parallel writeBridge call into the same group) could
|
||||
// pre-create or symlink that path before this writer opens it. mkdtemp
|
||||
// returns a directory whose suffix is filled with cryptographically
|
||||
// random bytes, so the staging path is unguessable AND collision-free
|
||||
// across parallel callers. We anchor the staging directory inside
|
||||
// `groupDir` so the subsequent rename of `bridge.lbug` (and its
|
||||
// `.wal` / `.shadow` sidecars) into place stays on the same filesystem
|
||||
// and remains atomic — moving across `os.tmpdir()` could trip EXDEV.
|
||||
const stagingDir = await fsp.mkdtemp(path.join(groupDir, 'bridge-tmp-'));
|
||||
const tmpPath = path.join(stagingDir, 'bridge.lbug');
|
||||
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
|
||||
|
||||
const report: WriteBridgeReport = {
|
||||
@@ -366,42 +397,42 @@ export async function writeBridge(
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up any leftover tmp main file AND its `.wal` / `.shadow` sidecars.
|
||||
// LadybugDB 0.16.0 rejects opening a database whose sidecars belong to a
|
||||
// different database instance (database-id check), so any stale sidecar
|
||||
// from a crashed previous run will fail the next writeBridge.
|
||||
await removeLbugFile(tmpPath);
|
||||
// The mkdtemp staging directory above is freshly created with a unique
|
||||
// random suffix, so there are no leftover `bridge.lbug.tmp` / `.wal` /
|
||||
// `.shadow` sidecars from a previous crashed run to clean up here — the
|
||||
// directory is empty by construction.
|
||||
|
||||
// 1. Create temp DB, insert all data.
|
||||
//
|
||||
// Everything after `openBridgeDb` must run inside a try/finally so that
|
||||
// if ANY step before the explicit `closeBridgeDb` throws — schema
|
||||
// creation, a contract insert loop that rethrows, a snapshot write, the
|
||||
// cross-link loop, or anything else — the handle is still released. A
|
||||
// leaked handle holds the native LadybugDB file lock on tmpPath, which
|
||||
// (a) leaks a FD and (b) prevents the next writeBridge call from
|
||||
// reusing the same tmp slot.
|
||||
const handle = await openBridgeDb(tmpPath);
|
||||
let handleClosed = false;
|
||||
try {
|
||||
await ensureBridgeSchema(handle);
|
||||
// 1. Create temp DB, insert all data.
|
||||
//
|
||||
// Everything after `openBridgeDb` must run inside a try/finally so that
|
||||
// if ANY step before the explicit `closeBridgeDb` throws — schema
|
||||
// creation, a contract insert loop that rethrows, a snapshot write, the
|
||||
// cross-link loop, or anything else — the handle is still released. A
|
||||
// leaked handle holds the native LadybugDB file lock on tmpPath, which
|
||||
// (a) leaks a FD and (b) prevents the next writeBridge call from
|
||||
// reusing the same tmp slot.
|
||||
const handle = await openBridgeDb(tmpPath);
|
||||
let handleClosed = false;
|
||||
try {
|
||||
await ensureBridgeSchema(handle);
|
||||
|
||||
// Build the lookup index incrementally as contracts are inserted, so
|
||||
// failed inserts are never in the index (and therefore never resolved
|
||||
// by the cross-link loop below). This replaces a previous N+1 query
|
||||
// pattern where each link made up to 6 DB round-trips to find its
|
||||
// endpoints — see ContractLookupIndex.
|
||||
const lookupIndex = createContractLookupIndex();
|
||||
// Build the lookup index incrementally as contracts are inserted, so
|
||||
// failed inserts are never in the index (and therefore never resolved
|
||||
// by the cross-link loop below). This replaces a previous N+1 query
|
||||
// pattern where each link made up to 6 DB round-trips to find its
|
||||
// endpoints — see ContractLookupIndex.
|
||||
const lookupIndex = createContractLookupIndex();
|
||||
|
||||
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
|
||||
// that can't be serialized). The whole sync must not fail because one
|
||||
// contract is broken.
|
||||
for (const c of contracts) {
|
||||
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (n:Contract {
|
||||
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
|
||||
// that can't be serialized). The whole sync must not fail because one
|
||||
// contract is broken.
|
||||
for (const c of contracts) {
|
||||
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (n:Contract {
|
||||
id: $id,
|
||||
contractId: $contractId,
|
||||
type: $type,
|
||||
@@ -414,91 +445,91 @@ export async function writeBridge(
|
||||
confidence: $confidence,
|
||||
meta: $meta
|
||||
})`,
|
||||
{
|
||||
id,
|
||||
contractId: c.contractId,
|
||||
type: c.type,
|
||||
role: c.role,
|
||||
repo: c.repo,
|
||||
service: c.service ?? '',
|
||||
symbolUid: c.symbolUid,
|
||||
filePath: c.symbolRef.filePath,
|
||||
symbolName: c.symbolName,
|
||||
confidence: c.confidence,
|
||||
meta: JSON.stringify(c.meta),
|
||||
},
|
||||
);
|
||||
report.contractsInserted++;
|
||||
// Only index on successful insert — the cross-link loop must never
|
||||
// resolve to a row that isn't actually in the DB.
|
||||
indexContract(lookupIndex, c, id);
|
||||
} catch (err) {
|
||||
report.contractsFailed++;
|
||||
recordError('contract', id, err);
|
||||
{
|
||||
id,
|
||||
contractId: c.contractId,
|
||||
type: c.type,
|
||||
role: c.role,
|
||||
repo: c.repo,
|
||||
service: c.service ?? '',
|
||||
symbolUid: c.symbolUid,
|
||||
filePath: c.symbolRef.filePath,
|
||||
symbolName: c.symbolName,
|
||||
confidence: c.confidence,
|
||||
meta: JSON.stringify(c.meta),
|
||||
},
|
||||
);
|
||||
report.contractsInserted++;
|
||||
// Only index on successful insert — the cross-link loop must never
|
||||
// resolve to a row that isn't actually in the DB.
|
||||
indexContract(lookupIndex, c, id);
|
||||
} catch (err) {
|
||||
report.contractsFailed++;
|
||||
recordError('contract', id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Insert repo snapshots
|
||||
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (s:RepoSnapshot {
|
||||
// Insert repo snapshots
|
||||
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (s:RepoSnapshot {
|
||||
id: $id,
|
||||
indexedAt: $indexedAt,
|
||||
lastCommit: $lastCommit
|
||||
})`,
|
||||
{
|
||||
id: repoId,
|
||||
indexedAt: snap.indexedAt,
|
||||
lastCommit: snap.lastCommit,
|
||||
},
|
||||
);
|
||||
report.snapshotsInserted++;
|
||||
} catch (err) {
|
||||
report.snapshotsFailed++;
|
||||
recordError('snapshot', repoId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert cross-links (tolerating missing nodes).
|
||||
//
|
||||
// `findContractNode` consults the in-memory lookup index built above,
|
||||
// not the DB — that's an O(1) pure-function lookup per endpoint instead
|
||||
// of the previous 2-3 DB queries. For M cross-links, the previous code
|
||||
// issued up to 6M round-trips; this version issues zero.
|
||||
//
|
||||
// `link.contractId` may differ between the consumer and provider sides
|
||||
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
|
||||
// `grpc::Service/Method`) — that's why we resolve each endpoint
|
||||
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
|
||||
// tuple rather than matching on contractId.
|
||||
for (const link of crossLinks) {
|
||||
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
|
||||
try {
|
||||
const fromId = findContractNode(
|
||||
lookupIndex,
|
||||
link.from.repo,
|
||||
'consumer',
|
||||
link.from.symbolUid,
|
||||
link.from.symbolRef.filePath,
|
||||
link.from.symbolRef.name,
|
||||
);
|
||||
const toId = findContractNode(
|
||||
lookupIndex,
|
||||
link.to.repo,
|
||||
'provider',
|
||||
link.to.symbolUid,
|
||||
link.to.symbolRef.filePath,
|
||||
link.to.symbolRef.name,
|
||||
);
|
||||
if (!fromId || !toId) {
|
||||
report.linksDroppedMissingNode++;
|
||||
continue;
|
||||
{
|
||||
id: repoId,
|
||||
indexedAt: snap.indexedAt,
|
||||
lastCommit: snap.lastCommit,
|
||||
},
|
||||
);
|
||||
report.snapshotsInserted++;
|
||||
} catch (err) {
|
||||
report.snapshotsFailed++;
|
||||
recordError('snapshot', repoId, err);
|
||||
}
|
||||
await queryBridge(
|
||||
handle,
|
||||
`
|
||||
}
|
||||
|
||||
// Insert cross-links (tolerating missing nodes).
|
||||
//
|
||||
// `findContractNode` consults the in-memory lookup index built above,
|
||||
// not the DB — that's an O(1) pure-function lookup per endpoint instead
|
||||
// of the previous 2-3 DB queries. For M cross-links, the previous code
|
||||
// issued up to 6M round-trips; this version issues zero.
|
||||
//
|
||||
// `link.contractId` may differ between the consumer and provider sides
|
||||
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
|
||||
// `grpc::Service/Method`) — that's why we resolve each endpoint
|
||||
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
|
||||
// tuple rather than matching on contractId.
|
||||
for (const link of crossLinks) {
|
||||
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
|
||||
try {
|
||||
const fromId = findContractNode(
|
||||
lookupIndex,
|
||||
link.from.repo,
|
||||
'consumer',
|
||||
link.from.symbolUid,
|
||||
link.from.symbolRef.filePath,
|
||||
link.from.symbolRef.name,
|
||||
);
|
||||
const toId = findContractNode(
|
||||
lookupIndex,
|
||||
link.to.repo,
|
||||
'provider',
|
||||
link.to.symbolUid,
|
||||
link.to.symbolRef.filePath,
|
||||
link.to.symbolRef.name,
|
||||
);
|
||||
if (!fromId || !toId) {
|
||||
report.linksDroppedMissingNode++;
|
||||
continue;
|
||||
}
|
||||
await queryBridge(
|
||||
handle,
|
||||
`
|
||||
MATCH (a:Contract), (b:Contract)
|
||||
WHERE a.id = $fromId AND b.id = $toId
|
||||
CREATE (a)-[:ContractLink {
|
||||
@@ -509,83 +540,93 @@ export async function writeBridge(
|
||||
toRepo: $toRepo
|
||||
}]->(b)
|
||||
`,
|
||||
{
|
||||
fromId,
|
||||
toId,
|
||||
matchType: link.matchType,
|
||||
confidence: link.confidence,
|
||||
contractId: link.contractId,
|
||||
fromRepo: link.from.repo,
|
||||
toRepo: link.to.repo,
|
||||
},
|
||||
);
|
||||
report.linksInserted++;
|
||||
} catch (err) {
|
||||
report.linksFailed++;
|
||||
recordError('link', linkId, err);
|
||||
{
|
||||
fromId,
|
||||
toId,
|
||||
matchType: link.matchType,
|
||||
confidence: link.confidence,
|
||||
contractId: link.contractId,
|
||||
fromRepo: link.from.repo,
|
||||
toRepo: link.to.repo,
|
||||
},
|
||||
);
|
||||
report.linksInserted++;
|
||||
} catch (err) {
|
||||
report.linksFailed++;
|
||||
recordError('link', linkId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Close temp DB (happy path). The finally block also calls
|
||||
// closeBridgeDb if we threw above; `handleClosed` prevents a
|
||||
// double-close on the native handle.
|
||||
await closeBridgeDb(handle);
|
||||
handleClosed = true;
|
||||
} finally {
|
||||
if (!handleClosed) {
|
||||
await closeBridgeDb(handle).catch(() => {
|
||||
/* ignore: cleanup path, best effort */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Close temp DB (happy path). The finally block also calls
|
||||
// closeBridgeDb if we threw above; `handleClosed` prevents a
|
||||
// double-close on the native handle.
|
||||
await closeBridgeDb(handle);
|
||||
handleClosed = true;
|
||||
} finally {
|
||||
if (!handleClosed) {
|
||||
await closeBridgeDb(handle).catch(() => {
|
||||
/* ignore: cleanup path, best effort */
|
||||
});
|
||||
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
|
||||
//
|
||||
// The current database file (with its `.wal` / `.shadow` sidecars) is
|
||||
// moved aside, then the freshly built tmp database takes its place.
|
||||
// We move the sidecars together with the main file so the open below
|
||||
// and any external readers see a consistent set; orphan sidecars from
|
||||
// the tmp namespace are then removed because LadybugDB looks for them
|
||||
// under the renamed-to base name and would reject mismatching IDs.
|
||||
try {
|
||||
await fsp.access(finalPath);
|
||||
await retryRename(finalPath, bakPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
try {
|
||||
await fsp.access(`${finalPath}${suffix}`);
|
||||
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing db */
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
|
||||
//
|
||||
// The current database file (with its `.wal` / `.shadow` sidecars) is
|
||||
// moved aside, then the freshly built tmp database takes its place.
|
||||
// We move the sidecars together with the main file so the open below
|
||||
// and any external readers see a consistent set; orphan sidecars from
|
||||
// the tmp namespace are then removed because LadybugDB looks for them
|
||||
// under the renamed-to base name and would reject mismatching IDs.
|
||||
try {
|
||||
await fsp.access(finalPath);
|
||||
await retryRename(finalPath, bakPath);
|
||||
await retryRename(tmpPath, finalPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
// Rename — not delete — so the WAL (which may carry uncommitted-at-
|
||||
// close-time pages on a graceful close, depending on
|
||||
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
|
||||
// checkpoint snapshot stay paired with the database file under its
|
||||
// final name. LadybugDB 0.16.0's database-id check rejects an open
|
||||
// when the sidecars belong to a different base name.
|
||||
try {
|
||||
await fsp.access(`${finalPath}${suffix}`);
|
||||
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
|
||||
await fsp.access(`${tmpPath}${suffix}`);
|
||||
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing db */
|
||||
}
|
||||
await retryRename(tmpPath, finalPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
// Rename — not delete — so the WAL (which may carry uncommitted-at-
|
||||
// close-time pages on a graceful close, depending on
|
||||
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
|
||||
// checkpoint snapshot stay paired with the database file under its
|
||||
// final name. LadybugDB 0.16.0's database-id check rejects an open
|
||||
// when the sidecars belong to a different base name.
|
||||
try {
|
||||
await fsp.access(`${tmpPath}${suffix}`);
|
||||
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
await removeLbugFile(bakPath);
|
||||
await removeLbugFile(bakPath);
|
||||
|
||||
// 4. Write meta.json
|
||||
await writeBridgeMeta(groupDir, {
|
||||
version: BRIDGE_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
missingRepos: input.missingRepos,
|
||||
});
|
||||
// 4. Write meta.json
|
||||
await writeBridgeMeta(groupDir, {
|
||||
version: BRIDGE_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
missingRepos: input.missingRepos,
|
||||
});
|
||||
|
||||
return report;
|
||||
return report;
|
||||
} finally {
|
||||
// Always remove the mkdtemp staging directory. On the happy path the
|
||||
// main file and sidecars have been renamed out of it, so it's empty;
|
||||
// on any error path it may still contain a partial database — either
|
||||
// way `recursive: true, force: true` removes it without surfacing
|
||||
// "directory not empty" or ENOENT.
|
||||
await fsp.rm(stagingDir, { recursive: true, force: true }).catch(() => {
|
||||
/* best-effort cleanup */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
@@ -681,14 +722,15 @@ export async function openBridgeDbReadOnly(groupDir: string): Promise<BridgeHand
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
if (process.env.GITNEXUS_DEBUG_BRIDGE) {
|
||||
console.warn(
|
||||
`[bridge-db] openBridgeDbReadOnly(${groupDir}) gave up after ` +
|
||||
`${LBUG_OPEN_RETRY_ATTEMPTS} attempts: ${
|
||||
lastErr instanceof Error ? lastErr.message : String(lastErr)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
// Pino's NDJSON serialization is structurally injection-resistant
|
||||
// (CodeQL js/log-injection): groupDir and err.message are JSON-escaped
|
||||
// by the serializer, so no manual CRLF / U+2028 / ANSI sanitization is
|
||||
// needed. Demoted to debug — only fires when the bridge truly gave up
|
||||
// after retries, and operators only need it at debug verbosity.
|
||||
bridgeLogger.debug(
|
||||
{ groupDir, err: lastErr, attempts: LBUG_OPEN_RETRY_ATTEMPTS },
|
||||
'openBridgeDbReadOnly gave up',
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,15 +4,34 @@ import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from
|
||||
const _require = createRequire(import.meta.url);
|
||||
const yaml = _require('js-yaml') as typeof import('js-yaml');
|
||||
|
||||
const VALID_CONTRACT_TYPES: ContractType[] = ['http', 'grpc', 'topic', 'lib', 'custom'];
|
||||
const VALID_CONTRACT_TYPES: ContractType[] = [
|
||||
'http',
|
||||
'grpc',
|
||||
'thrift',
|
||||
'topic',
|
||||
'lib',
|
||||
'custom',
|
||||
'include',
|
||||
];
|
||||
const VALID_ROLES: ContractRole[] = ['provider', 'consumer'];
|
||||
|
||||
// Defaults matter for backward compatibility: any group.yaml that omits a
|
||||
// `detect.<type>` key inherits its value from this constant. Adding a new
|
||||
// extractor that defaults to `true` silently changes the behavior of every
|
||||
// existing group on the next sync. New extractors must default to `false`
|
||||
// (opt-in) so operators consciously enable them via group.yaml.
|
||||
//
|
||||
// `includes`: opt-in. The C/C++ IncludeExtractor (PR #1156) ships disabled by
|
||||
// default; enable with `detect.includes: true` for groups containing C/C++
|
||||
// repos that need cross-repo header tracking.
|
||||
const DEFAULT_DETECT = {
|
||||
http: true,
|
||||
grpc: true,
|
||||
thrift: true,
|
||||
topics: true,
|
||||
shared_libs: true,
|
||||
embedding_fallback: true,
|
||||
includes: false,
|
||||
workspace_deps: false,
|
||||
};
|
||||
|
||||
|
||||
@@ -91,6 +91,25 @@ function clampCrossDepth(raw: unknown): { depth: number; warning?: string } {
|
||||
return { depth: d };
|
||||
}
|
||||
|
||||
/**
|
||||
* Clamp the impact timeout to a sane bounded range. Callers can feed this
|
||||
* via tool params, so an unclamped value lets a single request hold a
|
||||
* timer slot for an arbitrarily long duration (CodeQL js/resource-
|
||||
* exhaustion). 100ms lower bound preserves test-suite scenarios that
|
||||
* exercise tight timeouts; 5min upper bound is well above any legitimate
|
||||
* single-impact compute. Applied at the validate boundary so the
|
||||
* downstream `deadline` (Date.now() + timeoutMs) and the local-leg
|
||||
* `setTimeout` see the same clamped value — earlier shapes had a 1hr
|
||||
* outer cap and a 5min inner clamp that disagreed.
|
||||
*/
|
||||
export const IMPACT_TIMEOUT_MIN_MS = 100;
|
||||
export const IMPACT_TIMEOUT_MAX_MS = 5 * 60 * 1_000;
|
||||
|
||||
export function clampTimeout(timeoutMs: number): number {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return IMPACT_TIMEOUT_MIN_MS;
|
||||
return Math.min(IMPACT_TIMEOUT_MAX_MS, Math.max(IMPACT_TIMEOUT_MIN_MS, Math.trunc(timeoutMs)));
|
||||
}
|
||||
|
||||
export function validateGroupImpactParams(params: Record<string, unknown>):
|
||||
| {
|
||||
ok: true;
|
||||
@@ -143,13 +162,19 @@ export function validateGroupImpactParams(params: Record<string, unknown>):
|
||||
const service = normalizeServicePrefix(params.service);
|
||||
const subgroup = typeof params.subgroup === 'string' ? params.subgroup : undefined;
|
||||
|
||||
let timeoutMs =
|
||||
// Clamp at the validate boundary so the downstream `deadline` (line
|
||||
// ~366) and `safeLocalImpact`'s `setTimeout` both see a single
|
||||
// bounded value. Without this, the outer deadline budgeted Phase-2
|
||||
// cross-repo fanout up to 1hr while only the inner setTimeout was
|
||||
// capped to 5min — the two halves of CodeQL #184's mitigation
|
||||
// disagreed.
|
||||
const rawTimeoutMs =
|
||||
typeof params.timeoutMs === 'number' && params.timeoutMs > 0
|
||||
? params.timeoutMs
|
||||
: typeof params.timeout === 'number' && params.timeout > 0
|
||||
? params.timeout
|
||||
: DEFAULT_LOCAL_IMPACT_TIMEOUT_MS;
|
||||
if (timeoutMs > 3_600_000) timeoutMs = 3_600_000;
|
||||
const timeoutMs = clampTimeout(rawTimeoutMs);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
@@ -191,12 +216,13 @@ async function safeLocalImpact(
|
||||
impactParams: Parameters<GroupToolPort['impact']>[1],
|
||||
timeoutMs: number,
|
||||
): Promise<{ value: unknown; timedOut: boolean }> {
|
||||
const safeTimeoutMs = clampTimeout(timeoutMs);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const impactP = port.impact(repo, impactParams).catch((err) => ({
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}));
|
||||
const timeoutP = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), timeoutMs);
|
||||
timer = setTimeout(() => resolve('timeout'), safeTimeoutMs);
|
||||
});
|
||||
const won = await Promise.race([
|
||||
impactP.then((v) => ({ tag: 'impact' as const, v })),
|
||||
@@ -212,6 +238,65 @@ async function safeLocalImpact(
|
||||
return { value: won.v, timedOut: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Race a single Phase-2 `impactByUid` call against a remaining-budget
|
||||
* timer. The Codex adversarial review on PR #1331 surfaced that the
|
||||
* fanout loop only checked `Date.now() > deadline` *between* neighbor
|
||||
* calls — once `await port.impactByUid(...)` was reached, a hung
|
||||
* neighbor could pin the request indefinitely, and slow neighbors
|
||||
* could compound past the 5-min `IMPACT_TIMEOUT_MAX_MS` cap.
|
||||
*
|
||||
* This helper wraps each call: a `setTimeout(remainingMs)` aborts an
|
||||
* `AbortController` whose signal is forwarded to `impactByUid`, and a
|
||||
* `Promise.race` resolves to `{ timedOut: true }` when the timer
|
||||
* fires before the call completes. Implementors that ignore the
|
||||
* signal (current local backend) still see their await resolved by
|
||||
* the race; full cooperative cancellation inside the BFS is a future
|
||||
* follow-up. On rejection, the value is `null` (matching the
|
||||
* fanout's existing `if (fan == null)` truncation contract).
|
||||
*
|
||||
* Exported for direct unit testing — the helper IS the load-bearing
|
||||
* mitigation surface, so the U3 regression test pins it directly
|
||||
* rather than driving the full `runGroupImpact` path.
|
||||
*/
|
||||
export async function safeNeighborImpact(
|
||||
port: GroupToolPort,
|
||||
repoId: string,
|
||||
uid: string,
|
||||
direction: string,
|
||||
opts: {
|
||||
maxDepth: number;
|
||||
relationTypes: string[];
|
||||
minConfidence: number;
|
||||
includeTests: boolean;
|
||||
},
|
||||
remainingMs: number,
|
||||
): Promise<{ value: unknown; timedOut: boolean }> {
|
||||
const controller = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const callP = port
|
||||
.impactByUid(repoId, uid, direction, { ...opts, signal: controller.signal })
|
||||
.catch(() => null);
|
||||
const timeoutP = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(
|
||||
() => {
|
||||
controller.abort();
|
||||
resolve('timeout');
|
||||
},
|
||||
Math.max(0, remainingMs),
|
||||
);
|
||||
});
|
||||
const won = await Promise.race([
|
||||
callP.then((v) => ({ tag: 'impact' as const, v })),
|
||||
timeoutP.then(() => ({ tag: 'timeout' as const })),
|
||||
]);
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
if (won.tag === 'timeout') {
|
||||
return { value: null, timedOut: true };
|
||||
}
|
||||
return { value: won.v, timedOut: false };
|
||||
}
|
||||
|
||||
export function collectImpactSymbolUids(
|
||||
local: unknown,
|
||||
servicePrefix: string | undefined,
|
||||
@@ -476,7 +561,8 @@ export async function runGroupImpact(
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
if (Date.now() > deadline) {
|
||||
const remainingMs = deadline - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
truncatedRepos.push(n.neighborRepo);
|
||||
continue;
|
||||
}
|
||||
@@ -492,13 +578,25 @@ export async function runGroupImpact(
|
||||
continue;
|
||||
}
|
||||
|
||||
const fan = await deps.port.impactByUid(neighborHandle.id, n.neighborUid, direction, {
|
||||
maxDepth,
|
||||
relationTypes: relationTypes ?? [],
|
||||
minConfidence,
|
||||
includeTests,
|
||||
});
|
||||
if (fan == null) {
|
||||
// Phase-2 hardening: race each impactByUid against a per-call
|
||||
// timeout derived from the remaining budget. Without this wrap a
|
||||
// single hung neighbor would pin the request past the clamped
|
||||
// timeout, which Codex's adversarial review on PR #1331 flagged
|
||||
// as the still-open half of CodeQL #184 / js/resource-exhaustion.
|
||||
const { value: fan, timedOut: neighborTimedOut } = await safeNeighborImpact(
|
||||
deps.port,
|
||||
neighborHandle.id,
|
||||
n.neighborUid,
|
||||
direction,
|
||||
{
|
||||
maxDepth,
|
||||
relationTypes: relationTypes ?? [],
|
||||
minConfidence,
|
||||
includeTests,
|
||||
},
|
||||
remainingMs,
|
||||
);
|
||||
if (neighborTimedOut || fan == null) {
|
||||
truncatedRepos.push(n.neighborRepo);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
interface ElixirAppMeta {
|
||||
appName: string;
|
||||
modulePrefix: string;
|
||||
@@ -202,7 +203,7 @@ export async function extractElixirWorkspaceLinks(
|
||||
};
|
||||
const existing = appsByName.get(manifest.appName);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[elixir-workspace-extractor] duplicate app "${manifest.appName}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
interface GoModuleMeta {
|
||||
modulePath: string;
|
||||
groupPath: string;
|
||||
@@ -211,7 +212,7 @@ export async function extractGoWorkspaceLinks(
|
||||
};
|
||||
const existing = modulesByPath.get(manifest.modulePath);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[go-workspace-extractor] duplicate module "${manifest.modulePath}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -5,6 +5,7 @@ import { createIgnoreFilter } from '../../../config/ignore-service.js';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import { logger } from '../../logger.js';
|
||||
import {
|
||||
GRPC_SCAN_GLOB,
|
||||
getPluginForFile,
|
||||
@@ -344,7 +345,7 @@ export function resolveProtoConflict(
|
||||
// services under a fabricated package-qualified contract id.
|
||||
if (winners.length !== 1) {
|
||||
const paths = candidates.map((c) => c.protoPath).join(', ');
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`,
|
||||
);
|
||||
return null;
|
||||
|
||||
@@ -0,0 +1,610 @@
|
||||
import * as path from 'node:path';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import C from 'tree-sitter-c';
|
||||
import Cpp from 'tree-sitter-cpp';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import { buildSuffixIndex, type SuffixIndex } from '../../ingestion/import-resolvers/utils.js';
|
||||
import { createIgnoreFilter } from '../../../config/ignore-service.js';
|
||||
import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
/**
|
||||
* Cross-repo C/C++ `#include` dependency extractor.
|
||||
*
|
||||
* **Provider side:** registers every `.h/.hpp/.hxx/.hh` file in the repo
|
||||
* as a provider contract with `include::<relative-path>`.
|
||||
*
|
||||
* **Consumer side:** parses all C/C++ source/header files for `#include "…"`
|
||||
* directives, attempts suffix-based resolution against the repo's own file
|
||||
* list (reusing the same algorithm as the single-repo ingestion pipeline),
|
||||
* and emits unresolved include paths as consumer contracts.
|
||||
*
|
||||
* Matching: a consumer's `include::map/base/dice_map_view.h` in repo A
|
||||
* matches a provider's `include::map/base/dice_map_view.h` in repo B via
|
||||
* exact contract-id equality in `runExactMatch`.
|
||||
*/
|
||||
|
||||
// ---------- constants ----------
|
||||
|
||||
const HEADER_EXTENSIONS = new Set(['.h', '.hpp', '.hxx', '.hh']);
|
||||
|
||||
// Source = headers (provider-eligible) ∪ implementation files (.c/.cpp/.cc/.cxx).
|
||||
// Spread keeps the subset relationship explicit so a future contributor adding
|
||||
// a new header extension to HEADER_EXTENSIONS does not have to remember to
|
||||
// also add it here.
|
||||
const SOURCE_EXTENSIONS = new Set<string>([...HEADER_EXTENSIONS, '.c', '.cpp', '.cc', '.cxx']);
|
||||
|
||||
const INCLUDE_QUERY_SRC = '(preproc_include path: (_) @import.source) @import';
|
||||
|
||||
/**
|
||||
* Well-known C/C++ standard library headers that can appear in `#include "…"`
|
||||
* form (some projects use quotes for system headers).
|
||||
*/
|
||||
const SYSTEM_HEADERS = new Set([
|
||||
// C standard
|
||||
'assert.h',
|
||||
'complex.h',
|
||||
'ctype.h',
|
||||
'errno.h',
|
||||
'fenv.h',
|
||||
'float.h',
|
||||
'inttypes.h',
|
||||
'iso646.h',
|
||||
'limits.h',
|
||||
'locale.h',
|
||||
'math.h',
|
||||
'setjmp.h',
|
||||
'signal.h',
|
||||
'stdalign.h',
|
||||
'stdarg.h',
|
||||
'stdatomic.h',
|
||||
'stdbool.h',
|
||||
'stddef.h',
|
||||
'stdint.h',
|
||||
'stdio.h',
|
||||
'stdlib.h',
|
||||
'stdnoreturn.h',
|
||||
'string.h',
|
||||
'tgmath.h',
|
||||
'threads.h',
|
||||
'time.h',
|
||||
'uchar.h',
|
||||
'wchar.h',
|
||||
'wctype.h',
|
||||
// C++ standard (extensionless)
|
||||
'algorithm',
|
||||
'any',
|
||||
'array',
|
||||
'atomic',
|
||||
'barrier',
|
||||
'bit',
|
||||
'bitset',
|
||||
'cassert',
|
||||
'cctype',
|
||||
'cerrno',
|
||||
'cfenv',
|
||||
'cfloat',
|
||||
'charconv',
|
||||
'chrono',
|
||||
'cinttypes',
|
||||
'climits',
|
||||
'clocale',
|
||||
'cmath',
|
||||
'codecvt',
|
||||
'compare',
|
||||
'complex',
|
||||
'concepts',
|
||||
'condition_variable',
|
||||
'coroutine',
|
||||
'csetjmp',
|
||||
'csignal',
|
||||
'cstdarg',
|
||||
'cstddef',
|
||||
'cstdint',
|
||||
'cstdio',
|
||||
'cstdlib',
|
||||
'cstring',
|
||||
'ctime',
|
||||
'cuchar',
|
||||
'cwchar',
|
||||
'cwctype',
|
||||
'deque',
|
||||
'exception',
|
||||
'execution',
|
||||
'expected',
|
||||
'filesystem',
|
||||
'format',
|
||||
'forward_list',
|
||||
'fstream',
|
||||
'functional',
|
||||
'future',
|
||||
'generator',
|
||||
'initializer_list',
|
||||
'iomanip',
|
||||
'ios',
|
||||
'iosfwd',
|
||||
'iostream',
|
||||
'istream',
|
||||
'iterator',
|
||||
'latch',
|
||||
'limits',
|
||||
'list',
|
||||
'locale',
|
||||
'map',
|
||||
'mdspan',
|
||||
'memory',
|
||||
'memory_resource',
|
||||
'mutex',
|
||||
'new',
|
||||
'numbers',
|
||||
'numeric',
|
||||
'optional',
|
||||
'ostream',
|
||||
'print',
|
||||
'queue',
|
||||
'random',
|
||||
'ranges',
|
||||
'ratio',
|
||||
'regex',
|
||||
'scoped_allocator',
|
||||
'semaphore',
|
||||
'set',
|
||||
'shared_mutex',
|
||||
'source_location',
|
||||
'span',
|
||||
'spanstream',
|
||||
'sstream',
|
||||
'stack',
|
||||
'stacktrace',
|
||||
'stdexcept',
|
||||
'stdfloat',
|
||||
'stop_token',
|
||||
'streambuf',
|
||||
'string',
|
||||
'string_view',
|
||||
'strstream',
|
||||
'syncstream',
|
||||
'system_error',
|
||||
'thread',
|
||||
'tuple',
|
||||
'type_traits',
|
||||
'typeindex',
|
||||
'typeinfo',
|
||||
'unordered_map',
|
||||
'unordered_set',
|
||||
'utility',
|
||||
'valarray',
|
||||
'variant',
|
||||
'vector',
|
||||
'version',
|
||||
]);
|
||||
|
||||
/** Path prefixes that indicate system/kernel headers. */
|
||||
const SYSTEM_PATH_PREFIXES = [
|
||||
'sys/',
|
||||
'net/',
|
||||
'netinet/',
|
||||
'arpa/',
|
||||
'linux/',
|
||||
'asm/',
|
||||
'bits/',
|
||||
'gnu/',
|
||||
'mach/',
|
||||
'machine/',
|
||||
'xlocale/',
|
||||
];
|
||||
|
||||
/** Regex fallback for files that exceed tree-sitter's 32 KB parse limit. */
|
||||
const INCLUDE_REGEX = /^[ \t]*#\s*include\s*"([^"]+)"/gm;
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
/**
|
||||
* Normalize an include path to a canonical lowercase forward-slash form.
|
||||
*
|
||||
* IMPORTANT — case-folding caveat (PR #1156 review finding #3):
|
||||
* Header paths are lowercased so consumer `#include "Foo/Bar.h"` and
|
||||
* provider file `Foo/Bar.h` normalize to the same contract-id. This is
|
||||
* the right trade-off on case-insensitive filesystems (macOS, Windows)
|
||||
* but on case-sensitive Linux filesystems two distinct headers `Foo.h`
|
||||
* and `foo.h` in the same repo will collide onto the same provider
|
||||
* contract-id; only one survives `dedupe()`. The gain (reliable
|
||||
* cross-platform matching) outweighs the cost (extremely rare header
|
||||
* casing collisions inside a single repo).
|
||||
*/
|
||||
function normalizeIncludePath(raw: string): string {
|
||||
return raw.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip C/C++ block comments from a source blob. Used only by the
|
||||
* regex-fallback path to avoid emitting consumer contracts for
|
||||
* commented-out #include directives. Line comments (`// …`) cannot hide
|
||||
* #include directives because the regex anchors on start-of-line.
|
||||
* See PR #1156 review finding #5.
|
||||
*/
|
||||
function stripBlockComments(src: string): string {
|
||||
return src.replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
}
|
||||
|
||||
function isAngleBracketInclude(rawNodeText: string): boolean {
|
||||
const trimmed = rawNodeText.trim();
|
||||
return trimmed.startsWith('<') && trimmed.endsWith('>');
|
||||
}
|
||||
|
||||
function isSystemHeader(cleanedPath: string): boolean {
|
||||
// Check well-known standard headers
|
||||
if (SYSTEM_HEADERS.has(cleanedPath)) return true;
|
||||
// Check system path prefixes
|
||||
const lower = cleanedPath.toLowerCase();
|
||||
return SYSTEM_PATH_PREFIXES.some((prefix) => lower.startsWith(prefix));
|
||||
}
|
||||
|
||||
function isHeaderFile(filePath: string): boolean {
|
||||
return HEADER_EXTENSIONS.has(path.extname(filePath).toLowerCase());
|
||||
}
|
||||
|
||||
function getLanguageForFile(filePath: string): unknown | null {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.c':
|
||||
case '.h':
|
||||
return C;
|
||||
case '.cpp':
|
||||
case '.cc':
|
||||
case '.cxx':
|
||||
case '.hpp':
|
||||
case '.hxx':
|
||||
case '.hh':
|
||||
return Cpp;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether an include path resolves to a file inside the local repo.
|
||||
*
|
||||
* Uses *exact full-path* matching on the suffix index — we never accept a
|
||||
* truncated suffix match. For `#include "foo/bar.h"` this checks:
|
||||
* (a) a file whose path ends with the full `foo/bar.h`
|
||||
* (b) if the include omitted the extension, a file whose path ends with
|
||||
* the include + one of the C/C++ header extensions
|
||||
*
|
||||
* Returns `true` when a local file matches — caller should suppress the
|
||||
* cross-repo consumer contract.
|
||||
*
|
||||
* See PR #1156 review finding #4 (suffixResolve ambiguity).
|
||||
*/
|
||||
function isLocalInclude(cleaned: string, suffixIndex: SuffixIndex): boolean {
|
||||
const candidates = [cleaned];
|
||||
if (!/\.[a-zA-Z0-9]+$/.test(cleaned)) {
|
||||
for (const ext of ['.h', '.hpp', '.hxx', '.hh']) candidates.push(cleaned + ext);
|
||||
}
|
||||
for (const c of candidates) {
|
||||
if (suffixIndex.get(c) || suffixIndex.getInsensitive(c)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---------- main class ----------
|
||||
|
||||
export class IncludeExtractor implements ContractExtractor {
|
||||
type = 'include' as const;
|
||||
|
||||
/**
|
||||
* Always returns `true`. NOT called by `sync.ts`, which gates extraction via
|
||||
* `config.detect.includes` instead (see `sync.ts:174`). Kept solely to satisfy
|
||||
* the `ContractExtractor` interface so the type stays uniform across extractors.
|
||||
*/
|
||||
async canExtract(_repo: RepoHandle): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
|
||||
async extract(
|
||||
dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
// 1. Build the local file list using the same discovery as ingestion
|
||||
// (createIgnoreFilter + getMaxFileSizeBytes). This guarantees the
|
||||
// universe of provider/consumer paths matches the universe of File
|
||||
// nodes in the LadybugDB graph — so no cross-link points at a UID
|
||||
// that group impact cannot fan out to.
|
||||
// (PR #1156 Codex follow-up: discovery aligned with ingestion.)
|
||||
const allFiles = await this.discoverIndexableFiles(repoPath);
|
||||
const normalizedFiles = allFiles.map((f) => f.replace(/\\/g, '/'));
|
||||
const suffixIndex = buildSuffixIndex(normalizedFiles, allFiles);
|
||||
|
||||
// 2. Provider: register all header files
|
||||
const providers = await this.extractProviders(dbExecutor, repoPath, allFiles);
|
||||
|
||||
// 3. Consumer: filter the shared discovery list for source extensions
|
||||
// and parse #include directives in those files.
|
||||
const sourceFiles = allFiles.filter((f) =>
|
||||
SOURCE_EXTENSIONS.has(path.extname(f).toLowerCase()),
|
||||
);
|
||||
const consumers = await this.extractConsumers(repoPath, sourceFiles, suffixIndex);
|
||||
|
||||
return this.dedupe([...providers, ...consumers]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover repo-relative file paths using exactly the same rules the
|
||||
* ingestion pipeline uses (`walkRepositoryPaths` in
|
||||
* `gitnexus/src/core/ingestion/filesystem-walker.ts`):
|
||||
* - `createIgnoreFilter` honors `.gitignore`, `.gitnexusignore`, the
|
||||
* hardcoded ignore list, and `.gitnexusignore` last-match-wins
|
||||
* negation.
|
||||
* - `getMaxFileSizeBytes()` drops files larger than the cap so we
|
||||
* never emit `File:<rel>` UIDs for files ingestion would skip.
|
||||
*
|
||||
* Uses sequential stat — there is no `READ_CONCURRENCY` batching here
|
||||
* because group sync runs at startup-time, not the ingestion hot path,
|
||||
* and parallelism gains are not worth the import-graph weight.
|
||||
*
|
||||
* MAINTENANCE: if `walkRepositoryPaths` changes its glob options, ignore
|
||||
* filter shape, or size-cap logic, mirror those changes here. The two
|
||||
* implementations exist because the consumers need different return
|
||||
* shapes (string[] vs ScannedFile[]) and different concurrency, but
|
||||
* they MUST agree on which files are reachable — that is what makes
|
||||
* `File:<rel>` UIDs in cross-links correspond to graph File nodes.
|
||||
*/
|
||||
private async discoverIndexableFiles(repoPath: string): Promise<string[]> {
|
||||
const ignoreFilter = await createIgnoreFilter(repoPath);
|
||||
const maxFileSizeBytes = getMaxFileSizeBytes();
|
||||
|
||||
const candidates = await glob('**/*', {
|
||||
cwd: repoPath,
|
||||
nodir: true,
|
||||
dot: false,
|
||||
ignore: ignoreFilter,
|
||||
});
|
||||
|
||||
const survivors: string[] = [];
|
||||
for (const rel of candidates) {
|
||||
try {
|
||||
const stat = await fs.stat(path.join(repoPath, rel));
|
||||
if (stat.size > maxFileSizeBytes) continue;
|
||||
survivors.push(rel);
|
||||
} catch (err) {
|
||||
// ENOENT is the documented benign race (glob enumerated a file
|
||||
// that was deleted before we stat'd it — same race
|
||||
// walkRepositoryPaths absorbs via Promise.allSettled). Anything
|
||||
// else (EACCES, EMFILE, EIO) deserves a warning so an operator
|
||||
// can spot a permission/resource problem instead of silently
|
||||
// shipping fewer contracts than expected.
|
||||
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
||||
if (code !== 'ENOENT') {
|
||||
logger.warn(
|
||||
{ err: (err as Error).message, file: rel, repoPath },
|
||||
'⚠️ IncludeExtractor: stat failed during discovery; skipping file',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return survivors;
|
||||
}
|
||||
|
||||
// ---------- provider extraction ----------
|
||||
|
||||
private async extractProviders(
|
||||
dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
allFiles: string[],
|
||||
): Promise<ExtractedContract[]> {
|
||||
// Strategy A: graph-assisted
|
||||
if (dbExecutor) {
|
||||
const graphProviders = await this.extractProvidersGraph(dbExecutor, repoPath);
|
||||
if (graphProviders.length > 0) return graphProviders;
|
||||
}
|
||||
// Strategy B: filesystem fallback
|
||||
return this.extractProvidersFallback(repoPath, allFiles);
|
||||
}
|
||||
|
||||
private async extractProvidersGraph(
|
||||
db: CypherExecutor,
|
||||
repoPath: string,
|
||||
): Promise<ExtractedContract[]> {
|
||||
try {
|
||||
const rows = await db(
|
||||
`MATCH (f:File)
|
||||
WHERE f.filePath =~ '.*\\\\.(h|hpp|hxx|hh)$'
|
||||
RETURN f.filePath AS filePath, f.id AS fileId`,
|
||||
);
|
||||
// gitnexus analyze stores absolute paths in the File.filePath column.
|
||||
// Provider contract IDs MUST be repo-relative — otherwise the consumer
|
||||
// emits `include::map/base/view.h` and the provider emits
|
||||
// `include::/abs/path/to/repo/map/base/view.h`, which never match
|
||||
// through runExactMatch and the cross-link silently disappears.
|
||||
// (PR #1156 follow-up review: graph provider absolute-path bug.)
|
||||
const normalizedRepoPath = path.resolve(repoPath);
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const r of rows) {
|
||||
if (typeof r.filePath !== 'string' || !r.filePath) continue;
|
||||
const absolute = r.filePath as string;
|
||||
const rel = path.relative(normalizedRepoPath, absolute);
|
||||
// Skip rows that resolve outside the repo (e.g., system headers
|
||||
// somehow indexed, or stale absolute paths from a different machine).
|
||||
// path.relative returns a `..`-prefixed path or an absolute path
|
||||
// when the target is outside the base — both are wrong for our IDs.
|
||||
if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) continue;
|
||||
const normalizedRel = rel.replace(/\\/g, '/');
|
||||
out.push({
|
||||
contractId: `include::${normalizeIncludePath(normalizedRel)}`,
|
||||
type: 'include' as const,
|
||||
role: 'provider' as const,
|
||||
symbolUid: String(r.fileId ?? ''),
|
||||
symbolRef: { filePath: normalizedRel, name: path.basename(normalizedRel) },
|
||||
symbolName: path.basename(normalizedRel),
|
||||
confidence: 1.0,
|
||||
meta: { source: 'graph' },
|
||||
});
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private extractProvidersFallback(_repoPath: string, allFiles: string[]): ExtractedContract[] {
|
||||
return allFiles
|
||||
.filter((f) => isHeaderFile(f))
|
||||
.map((f) => {
|
||||
const filePath = f.replace(/\\/g, '/');
|
||||
return {
|
||||
contractId: `include::${normalizeIncludePath(filePath)}`,
|
||||
type: 'include' as const,
|
||||
role: 'provider' as const,
|
||||
symbolUid: `File:${filePath}`,
|
||||
symbolRef: { filePath, name: path.basename(filePath) },
|
||||
symbolName: path.basename(filePath),
|
||||
confidence: 0.95,
|
||||
meta: { source: 'filesystem' },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- consumer extraction ----------
|
||||
|
||||
private async extractConsumers(
|
||||
repoPath: string,
|
||||
sourceFiles: string[],
|
||||
suffixIndex: SuffixIndex,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const parser = new Parser();
|
||||
const out: ExtractedContract[] = [];
|
||||
// Compile the include query once per grammar to avoid re-compilation per file
|
||||
const queryCache = new Map<unknown, Parser.Query>();
|
||||
|
||||
for (const rel of sourceFiles) {
|
||||
const lang = getLanguageForFile(rel);
|
||||
if (!lang) continue;
|
||||
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
|
||||
let query = queryCache.get(lang);
|
||||
if (!query) {
|
||||
try {
|
||||
query = new Parser.Query(lang, INCLUDE_QUERY_SRC);
|
||||
queryCache.set(lang, query);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Collect raw include paths: tree-sitter first, regex fallback for large files.
|
||||
// `extractionSource` is stamped on each emitted consumer contract so
|
||||
// regex-fallback contracts stay auditable post-hoc (PR #1156 review finding #6).
|
||||
let rawIncludes: string[];
|
||||
let extractionSource: 'tree_sitter' | 'regex_fallback';
|
||||
try {
|
||||
parser.setLanguage(lang);
|
||||
const tree = parser.parse(content);
|
||||
let matches: Parser.QueryMatch[];
|
||||
try {
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch {
|
||||
matches = [];
|
||||
}
|
||||
rawIncludes = [];
|
||||
extractionSource = 'tree_sitter';
|
||||
for (const match of matches) {
|
||||
const sourceNode = match.captures.find((c) => c.name === 'import.source');
|
||||
if (!sourceNode) continue;
|
||||
const rawText = sourceNode.node.text;
|
||||
if (isAngleBracketInclude(rawText)) continue;
|
||||
const cleaned = rawText.replace(/['"<>]/g, '');
|
||||
if (cleaned && cleaned.length <= 2048) rawIncludes.push(cleaned);
|
||||
}
|
||||
} catch {
|
||||
// tree-sitter failed (e.g. file > 32 KB) — fall back to regex.
|
||||
// Strip block comments first so we don't emit a consumer contract
|
||||
// for a commented-out #include (PR #1156 review finding #5).
|
||||
rawIncludes = [];
|
||||
extractionSource = 'regex_fallback';
|
||||
const scanTarget = stripBlockComments(content);
|
||||
INCLUDE_REGEX.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = INCLUDE_REGEX.exec(scanTarget)) !== null) {
|
||||
if (m[1] && m[1].length <= 2048) rawIncludes.push(m[1]);
|
||||
}
|
||||
}
|
||||
|
||||
for (const cleaned of rawIncludes) {
|
||||
// Filter: skip known system headers and system path prefixes
|
||||
if (isSystemHeader(cleaned)) continue;
|
||||
|
||||
// Skip relative-up includes: `#include "../include/foo.h"` is
|
||||
// almost always an intra-repo reference. The suffix index is built
|
||||
// from repo-relative paths, so isLocalInclude can never match
|
||||
// `../foo.h`, and emitting it as a consumer contract just pollutes
|
||||
// the registry with an entry no provider can ever satisfy.
|
||||
// (PR #1156 follow-up review: `../` relative includes produce
|
||||
// spurious consumer contracts.)
|
||||
if (cleaned.startsWith('../') || cleaned.startsWith('..\\')) continue;
|
||||
|
||||
// Skip macro-style includes: `#include PLATFORM_HEADER` parses as an
|
||||
// identifier under tree-sitter's `(_) @import.source` wildcard. The
|
||||
// identifier text passes the strip/clean step unchanged, so without
|
||||
// this guard we would emit `include::platform_header` as a consumer
|
||||
// contract — and no provider in any repo will ever expose a contract
|
||||
// for a macro identifier (no file is named `PLATFORM_HEADER`). The
|
||||
// contract would sit permanently orphaned in the registry. Real
|
||||
// header references always contain a path separator (`/`, `\`) or an
|
||||
// extension dot (`foo.h`), so an absent both is a reliable signal we
|
||||
// are looking at a macro identifier. (PR #1156 follow-up review:
|
||||
// macro includes emit orphaned consumer contracts.)
|
||||
if (!/[./\\]/.test(cleaned)) continue;
|
||||
|
||||
// Local resolution (PR #1156 review finding #4): only accept an
|
||||
// exact-suffix match on the *full* include path. The generic
|
||||
// suffixResolve() iterates all truncated suffixes, which would
|
||||
// silently suppress a cross-repo `#include "map/base/view.h"`
|
||||
// when the local repo has any `internal/view.h` — a realistic
|
||||
// false-negative in large C++ codebases. Here we only resolve
|
||||
// locally if a file path ends with the complete include string
|
||||
// (optionally re-appending one of the C/C++ header extensions
|
||||
// when the include already omits it).
|
||||
if (isLocalInclude(cleaned, suffixIndex)) continue;
|
||||
|
||||
// Unresolved: emit as consumer contract
|
||||
const normalizedRel = rel.replace(/\\/g, '/');
|
||||
out.push({
|
||||
contractId: `include::${normalizeIncludePath(cleaned)}`,
|
||||
type: 'include' as const,
|
||||
role: 'consumer' as const,
|
||||
symbolUid: `File:${normalizedRel}`,
|
||||
symbolRef: { filePath: normalizedRel, name: cleaned },
|
||||
symbolName: cleaned,
|
||||
confidence: 0.85,
|
||||
meta: {
|
||||
source: extractionSource,
|
||||
includePath: cleaned,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- deduplication ----------
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const c of items) {
|
||||
const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
interface JavaProjectMeta {
|
||||
groupId: string;
|
||||
artifactId: string;
|
||||
@@ -213,7 +214,7 @@ export async function extractJavaWorkspaceLinks(
|
||||
};
|
||||
const existing = projectsByKey.get(key);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[java-workspace-extractor] duplicate artifact "${key}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js';
|
||||
import type { CypherExecutor } from '../contract-extractor.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
export interface ManifestExtractResult {
|
||||
contracts: StoredContract[];
|
||||
crossLinks: CrossLink[];
|
||||
@@ -177,7 +178,7 @@ export class ManifestExtractor {
|
||||
|
||||
// NOTE: All lookups use EXACT equality on the relevant name field and
|
||||
// deterministic ORDER BY before LIMIT 1. Previous versions used CONTAINS
|
||||
// for fuzzy matching (plus an unconditional ".proto" fallback for gRPC)
|
||||
// for fuzzy matching (plus an unconditional IDL file fallback for gRPC)
|
||||
// which produced silent false positives: e.g. manifest "/orders" would
|
||||
// match "/suborders", and a gRPC manifest entry in a repo with any
|
||||
// .proto file would attach to a random proto symbol.
|
||||
@@ -225,16 +226,21 @@ export class ManifestExtractor {
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'grpc') {
|
||||
} else if (link.type === 'grpc' || link.type === 'thrift') {
|
||||
// Contract is "Service/Method" or just "Service" (or package.Service
|
||||
// variants). Prefer matching by method name when present, otherwise
|
||||
// by service name. NO .proto path fallback — that's guaranteed to
|
||||
// return a wrong symbol in any repo with more than one proto file.
|
||||
// by service name. Thrift generated Java classes often use
|
||||
// package.Service in manifests while graph Class/Interface names are
|
||||
// stored as bare Service, so strip the package prefix for thrift
|
||||
// service-name lookups. NO IDL path fallback — that's guaranteed to
|
||||
// return a wrong symbol in any repo with more than one IDL file.
|
||||
// Label filters scope lookups: methods → Function|Method, services
|
||||
// → Class|Interface (no label match = no silent wrong hits on
|
||||
// File/Variable nodes that happen to share the name).
|
||||
const parts = link.contract.split('/');
|
||||
const serviceName = parts[0]?.trim() ?? '';
|
||||
const rawServiceName = parts[0]?.trim() ?? '';
|
||||
const serviceName =
|
||||
link.type === 'thrift' ? (rawServiceName.split('.').pop() ?? '') : rawServiceName;
|
||||
const methodName = parts[1]?.trim() ?? '';
|
||||
if (methodName) {
|
||||
rows = await executor(
|
||||
@@ -268,6 +274,14 @@ export class ManifestExtractor {
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'include') {
|
||||
rows = await executor(
|
||||
`MATCH (f:File) WHERE f.filePath = $contract
|
||||
RETURN f.id AS uid, f.name AS name, f.filePath AS filePath
|
||||
ORDER BY f.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'custom') {
|
||||
// Workspace extractors produce qualified contracts like "mathlex::Expression".
|
||||
// Graph nodes store the unqualified symbol name ("Expression"), so strip
|
||||
@@ -298,7 +312,7 @@ export class ManifestExtractor {
|
||||
// fail the whole manifest extraction. Unresolved contracts still
|
||||
// get a synthetic symbolUid below, so cross-impact can proceed.
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` +
|
||||
`in ${repoPathKey}: ${message}`,
|
||||
);
|
||||
@@ -344,12 +358,16 @@ export class ManifestExtractor {
|
||||
}
|
||||
case 'grpc':
|
||||
return `grpc::${contract}`;
|
||||
case 'thrift':
|
||||
return `thrift::${contract}`;
|
||||
case 'topic':
|
||||
return `topic::${contract}`;
|
||||
case 'lib':
|
||||
return `lib::${contract}`;
|
||||
case 'custom':
|
||||
return `custom::${contract}`;
|
||||
case 'include':
|
||||
return `include::${contract}`;
|
||||
default: {
|
||||
const _exhaustive: never = type;
|
||||
throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`);
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
interface PackageMeta {
|
||||
name: string;
|
||||
groupPath: string;
|
||||
@@ -205,7 +206,7 @@ export async function extractNodeWorkspaceLinks(
|
||||
};
|
||||
const existing = packagesByName.get(manifest.name);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[node-workspace-extractor] duplicate package name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
interface PythonPackageMeta {
|
||||
name: string;
|
||||
importName: string;
|
||||
@@ -204,7 +205,7 @@ export async function extractPythonWorkspaceLinks(
|
||||
};
|
||||
const existing = packagesByImportName.get(manifest.importName);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[python-workspace-extractor] duplicate package "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
import { shouldIgnorePath } from '../../../config/ignore-service.js';
|
||||
import { loadIgnoreRules } from '../../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
/**
|
||||
* Discover cross-crate contracts in a Rust workspace by reading each
|
||||
* member's `Cargo.toml` dependencies and scanning source files for
|
||||
@@ -30,6 +31,32 @@ interface ImportedSymbol {
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Linear-time `[package].name = "..."` lookup. The previous regex
|
||||
* `^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"` had a nested
|
||||
* lazy quantifier on `\n` that CodeQL js/redos flagged as exponential
|
||||
* on inputs like `[package]\n` + many bare `\n`. We walk lines
|
||||
* explicitly: scan from the first `[package]` header until we hit the
|
||||
* next `[...]` section header, looking for the `name = "..."` line.
|
||||
* O(n) with the line count.
|
||||
*
|
||||
* Exported so the U8 ReDoS regression test can drive the production
|
||||
* line-walk directly with adversarial fixtures (multi-line strings,
|
||||
* trailing sections, etc.) instead of duplicating it inline.
|
||||
*/
|
||||
export function parseCargoPackageName(content: string): string | null {
|
||||
const lines = content.split('\n');
|
||||
const packageStart = lines.findIndex((l) => l.trim() === '[package]');
|
||||
if (packageStart < 0) return null;
|
||||
for (let i = packageStart + 1; i < lines.length; i++) {
|
||||
const line = lines[i].trimStart();
|
||||
if (line.startsWith('[')) break; // hit the next section header
|
||||
const m = /^name\s*=\s*"([^"]+)"/.exec(line);
|
||||
if (m) return m[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Cargo.toml to extract the crate name and workspace dependency
|
||||
* names. Uses simple line-based parsing — no TOML library needed for
|
||||
@@ -46,12 +73,9 @@ async function parseCrateManifest(
|
||||
return null;
|
||||
}
|
||||
|
||||
let name = '';
|
||||
const name = parseCargoPackageName(content) ?? '';
|
||||
const workspaceDeps: string[] = [];
|
||||
|
||||
const nameMatch = content.match(/^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m);
|
||||
if (nameMatch) name = nameMatch[1];
|
||||
|
||||
// Match dependencies that use workspace = true, which indicates they
|
||||
// are workspace-internal deps:
|
||||
// dep_name = { workspace = true }
|
||||
@@ -224,7 +248,7 @@ export async function extractRustWorkspaceLinks(
|
||||
};
|
||||
const existing = cratesByName.get(manifest.name);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[rust-workspace-extractor] duplicate crate name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`,
|
||||
);
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import {
|
||||
getPluginForFile,
|
||||
THRIFT_SCAN_GLOB,
|
||||
type ThriftDetection,
|
||||
} from './thrift-patterns/index.js';
|
||||
|
||||
export interface ThriftServiceInfo {
|
||||
namespace: string;
|
||||
serviceName: string;
|
||||
methods: string[];
|
||||
thriftPath: string;
|
||||
}
|
||||
|
||||
export interface ThriftContext {
|
||||
namespacesByThrift: Map<string, string>;
|
||||
servicesByName: Map<string, ThriftServiceInfo[]>;
|
||||
}
|
||||
|
||||
function normalizeThriftPath(rel: string): string {
|
||||
return rel.replace(/\\/g, '/');
|
||||
}
|
||||
|
||||
export function thriftMethodContractId(
|
||||
namespace: string,
|
||||
serviceName: string,
|
||||
methodName: string,
|
||||
): string {
|
||||
const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
|
||||
return `thrift::${prefix}/${methodName}`;
|
||||
}
|
||||
|
||||
export function thriftServiceContractId(namespace: string, serviceName: string): string {
|
||||
const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
|
||||
return `thrift::${prefix}/*`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace Thrift comments and string literals with spaces while preserving
|
||||
* newlines and character offsets. Service block scanning can then count braces
|
||||
* without being confused by examples or comments inside the IDL.
|
||||
*/
|
||||
function stripThriftCommentsAndStrings(content: string): string {
|
||||
const out = new Array<string>(content.length);
|
||||
let i = 0;
|
||||
|
||||
while (i < content.length) {
|
||||
const ch = content[i];
|
||||
const next = content[i + 1];
|
||||
|
||||
if (ch === '/' && next === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length && content[i] !== '\n') {
|
||||
out[i] = content[i] === '\r' ? '\r' : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '#') {
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
while (i < content.length && content[i] !== '\n') {
|
||||
out[i] = content[i] === '\r' ? '\r' : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '/' && next === '*') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length) {
|
||||
if (content[i] === '*' && content[i + 1] === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '"' || ch === "'") {
|
||||
const quote = ch;
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
while (i < content.length) {
|
||||
const c = content[i];
|
||||
if (c === '\\' && i + 1 < content.length) {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === quote) {
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
out[i] = c === '\n' || c === '\r' ? c : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out[i] = ch;
|
||||
i++;
|
||||
}
|
||||
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
function extractNamespace(sanitizedContent: string): string {
|
||||
const namespaces: Array<{ language: string; namespace: string }> = [];
|
||||
const namespaceRe = /^\s*namespace\s+([A-Za-z_*][\w.*-]*)\s+([A-Za-z_][\w.]*)\s*$/gm;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = namespaceRe.exec(sanitizedContent)) !== null) {
|
||||
namespaces.push({ language: match[1], namespace: match[2] });
|
||||
}
|
||||
|
||||
return (
|
||||
namespaces.find((entry) => entry.language === 'java')?.namespace ??
|
||||
namespaces[0]?.namespace ??
|
||||
''
|
||||
);
|
||||
}
|
||||
|
||||
function extractServiceBlocks(sanitizedContent: string): Array<{ name: string; body: string }> {
|
||||
const results: Array<{ name: string; body: string }> = [];
|
||||
const headerRe = /service\s+([A-Za-z_]\w*)\s*(?:extends\s+[A-Za-z_][\w.]*)?\s*\{/g;
|
||||
let headerMatch: RegExpExecArray | null;
|
||||
|
||||
while ((headerMatch = headerRe.exec(sanitizedContent)) !== null) {
|
||||
const serviceName = headerMatch[1];
|
||||
const bodyStart = headerMatch.index + headerMatch[0].length;
|
||||
let depth = 1;
|
||||
let pos = bodyStart;
|
||||
|
||||
while (pos < sanitizedContent.length && depth > 0) {
|
||||
const ch = sanitizedContent[pos];
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
pos++;
|
||||
}
|
||||
|
||||
if (depth !== 0) continue;
|
||||
|
||||
results.push({
|
||||
name: serviceName,
|
||||
body: sanitizedContent.slice(bodyStart, pos - 1),
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function extractMethods(sanitizedServiceBody: string): string[] {
|
||||
const methods: string[] = [];
|
||||
const methodRe =
|
||||
/(?:^|[;,\n\r])\s*(?:oneway\s+)?[A-Za-z_][\w.]*(?:\s*<[^(){};]*>)?\s+([A-Za-z_]\w*)\s*\(/g;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = methodRe.exec(sanitizedServiceBody)) !== null) {
|
||||
methods.push(match[1]);
|
||||
}
|
||||
|
||||
return methods;
|
||||
}
|
||||
|
||||
function thriftSourceScanSymbolUid(
|
||||
contractId: string,
|
||||
role: 'provider' | 'consumer',
|
||||
filePath: string,
|
||||
symbolName: string,
|
||||
): string {
|
||||
const contractKey = contractId.startsWith('thrift::')
|
||||
? contractId.slice('thrift::'.length)
|
||||
: contractId;
|
||||
return ['source-scan::thrift', role, contractKey, normalizeThriftPath(filePath), symbolName].join(
|
||||
'::',
|
||||
);
|
||||
}
|
||||
|
||||
function makeContract(
|
||||
cid: string,
|
||||
role: 'provider' | 'consumer',
|
||||
filePath: string,
|
||||
symbolName: string,
|
||||
confidence: number,
|
||||
meta: Record<string, unknown>,
|
||||
): ExtractedContract {
|
||||
return {
|
||||
contractId: cid,
|
||||
type: 'thrift',
|
||||
role,
|
||||
symbolUid: thriftSourceScanSymbolUid(cid, role, filePath, symbolName),
|
||||
symbolRef: { filePath: normalizeThriftPath(filePath), name: symbolName },
|
||||
symbolName,
|
||||
confidence,
|
||||
meta: { ...meta, extractionStrategy: 'source_scan' },
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildThriftContext(repoPath: string): Promise<ThriftContext> {
|
||||
const thriftFiles = await glob('**/*.thrift', {
|
||||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
// TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
|
||||
// (the canonical ingestion ignore filter, like include-extractor.ts now uses).
|
||||
// New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
|
||||
// third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
});
|
||||
const namespacesByThrift = new Map<string, string>();
|
||||
const servicesByName = new Map<string, ThriftServiceInfo[]>();
|
||||
|
||||
for (const rel of thriftFiles) {
|
||||
const thriftPath = normalizeThriftPath(rel);
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
|
||||
const sanitized = stripThriftCommentsAndStrings(content);
|
||||
const namespace = extractNamespace(sanitized);
|
||||
namespacesByThrift.set(thriftPath, namespace);
|
||||
|
||||
for (const block of extractServiceBlocks(sanitized)) {
|
||||
const methods = extractMethods(block.body);
|
||||
const info: ThriftServiceInfo = {
|
||||
namespace,
|
||||
serviceName: block.name,
|
||||
methods,
|
||||
thriftPath,
|
||||
};
|
||||
const existing = servicesByName.get(block.name) ?? [];
|
||||
existing.push(info);
|
||||
servicesByName.set(block.name, existing);
|
||||
}
|
||||
}
|
||||
|
||||
return { namespacesByThrift, servicesByName };
|
||||
}
|
||||
|
||||
export class ThriftExtractor implements ContractExtractor {
|
||||
type = 'thrift' as const;
|
||||
|
||||
async canExtract(_repo: RepoHandle): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
|
||||
async extract(
|
||||
_dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const out: ExtractedContract[] = [];
|
||||
const context = await buildThriftContext(repoPath);
|
||||
|
||||
for (const infos of context.servicesByName.values()) {
|
||||
for (const info of infos) {
|
||||
for (const methodName of info.methods) {
|
||||
const symbolName = `${info.serviceName}.${methodName}`;
|
||||
out.push(
|
||||
makeContract(
|
||||
thriftMethodContractId(info.namespace, info.serviceName, methodName),
|
||||
'provider',
|
||||
info.thriftPath,
|
||||
symbolName,
|
||||
0.85,
|
||||
{
|
||||
namespace: info.namespace,
|
||||
service: info.serviceName,
|
||||
method: methodName,
|
||||
source: 'thrift_idl',
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sourceFiles = await glob(THRIFT_SCAN_GLOB, {
|
||||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
// TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
|
||||
// (the canonical ingestion ignore filter, like include-extractor.ts now uses).
|
||||
// New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
|
||||
// third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
});
|
||||
|
||||
const parser = new Parser();
|
||||
for (const rel of sourceFiles) {
|
||||
const plugin = getPluginForFile(rel);
|
||||
if (!plugin) continue;
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
|
||||
let detections: ThriftDetection[] = [];
|
||||
try {
|
||||
parser.setLanguage(plugin.language);
|
||||
const tree = parser.parse(content);
|
||||
detections = plugin.scan(tree);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const detection of detections) {
|
||||
const contract = this.detectionToContract(detection, rel, context);
|
||||
if (contract) out.push(contract);
|
||||
}
|
||||
}
|
||||
|
||||
return this.dedupe(out);
|
||||
}
|
||||
|
||||
private detectionToContract(
|
||||
detection: ThriftDetection,
|
||||
filePath: string,
|
||||
context: ThriftContext,
|
||||
): ExtractedContract | null {
|
||||
const candidates = context.servicesByName.get(detection.serviceName) ?? [];
|
||||
if (candidates.length > 1) return null;
|
||||
|
||||
const info = candidates[0];
|
||||
if (info) {
|
||||
if (!info.methods.includes(detection.methodName)) return null;
|
||||
return makeContract(
|
||||
thriftMethodContractId(info.namespace, info.serviceName, detection.methodName),
|
||||
detection.role,
|
||||
filePath,
|
||||
detection.symbolName,
|
||||
detection.confidenceWithIdl,
|
||||
{
|
||||
namespace: info.namespace,
|
||||
service: info.serviceName,
|
||||
method: detection.methodName,
|
||||
source: detection.source,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
detection.role !== 'consumer' ||
|
||||
!detection.methodName ||
|
||||
!detection.usesGeneratedServiceMember
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return makeContract(
|
||||
thriftMethodContractId('', detection.serviceName, detection.methodName),
|
||||
detection.role,
|
||||
filePath,
|
||||
detection.symbolName,
|
||||
detection.confidenceWithoutIdl,
|
||||
{
|
||||
service: detection.serviceName,
|
||||
method: detection.methodName,
|
||||
source: 'java_thrift_consumer_weak',
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const byKey = new Map<string, ExtractedContract>();
|
||||
for (const c of items) {
|
||||
const key = `${c.contractId}|${c.role}|${c.symbolRef.filePath}|${c.symbolName}`;
|
||||
const existing = byKey.get(key);
|
||||
if (!existing || c.confidence > existing.confidence) {
|
||||
byKey.set(key, c);
|
||||
}
|
||||
}
|
||||
return Array.from(byKey.values());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import * as path from 'node:path';
|
||||
import type { ThriftLanguagePlugin } from './types.js';
|
||||
import { JAVA_THRIFT_PLUGIN } from './java.js';
|
||||
|
||||
export type { ThriftDetection, ThriftLanguagePlugin, ThriftRole } from './types.js';
|
||||
|
||||
const REGISTRY: Record<string, ThriftLanguagePlugin> = {
|
||||
'.java': JAVA_THRIFT_PLUGIN,
|
||||
};
|
||||
|
||||
export const THRIFT_SCAN_GLOB = '**/*.java';
|
||||
|
||||
export function getPluginForFile(rel: string): ThriftLanguagePlugin | undefined {
|
||||
const ext = path.extname(rel).toLowerCase();
|
||||
return REGISTRY[ext];
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import Parser from 'tree-sitter';
|
||||
import Java from 'tree-sitter-java';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { ThriftDetection, ThriftLanguagePlugin } from './types.js';
|
||||
|
||||
const GENERATED_MEMBER_TYPES = new Set(['Iface', 'Client']);
|
||||
const SERVICE_TYPE_RE = /^[A-Z][A-Za-z0-9]*(?:Service|Management)$/;
|
||||
|
||||
interface VariableBinding {
|
||||
name: string;
|
||||
serviceName: string;
|
||||
usesGeneratedServiceMember: boolean;
|
||||
scopeStart: number;
|
||||
scopeEnd: number;
|
||||
declarationEnd: number;
|
||||
scopeSize: number;
|
||||
}
|
||||
|
||||
interface ServiceTypeMatch {
|
||||
serviceName: string;
|
||||
usesGeneratedServiceMember: boolean;
|
||||
}
|
||||
|
||||
const VARIABLE_PATTERNS = compilePatterns({
|
||||
name: 'java-thrift-variables',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(field_declaration
|
||||
type: (_) @type
|
||||
declarator: (variable_declarator
|
||||
name: (identifier) @var))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(local_variable_declaration
|
||||
type: (_) @type
|
||||
declarator: (variable_declarator
|
||||
name: (identifier) @var))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(formal_parameter
|
||||
type: (_) @type
|
||||
name: (identifier) @var)
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
const CALL_PATTERNS = compilePatterns({
|
||||
name: 'java-thrift-method-calls',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @receiver
|
||||
name: (identifier) @method)
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (field_access
|
||||
object: (this)
|
||||
field: (identifier) @receiver)
|
||||
name: (identifier) @method)
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
const PROVIDER_PATTERNS = compilePatterns({
|
||||
name: 'java-thrift-providers',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(class_declaration
|
||||
name: (identifier) @class_name
|
||||
(super_interfaces
|
||||
(type_list
|
||||
(_) @type))
|
||||
body: (class_body) @body) @class
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
function serviceFromType(typeText: string): ServiceTypeMatch | null {
|
||||
const segments = typeText.split('.').filter((segment) => segment.length > 0);
|
||||
const last = segments.at(-1);
|
||||
const service = segments.at(-2);
|
||||
if (last && service && GENERATED_MEMBER_TYPES.has(last)) {
|
||||
return { serviceName: service, usesGeneratedServiceMember: true };
|
||||
}
|
||||
return last && SERVICE_TYPE_RE.test(last)
|
||||
? { serviceName: last, usesGeneratedServiceMember: false }
|
||||
: null;
|
||||
}
|
||||
|
||||
function methodNamesInClassBody(body: Parser.SyntaxNode): string[] {
|
||||
const names: string[] = [];
|
||||
for (let i = 0; i < body.namedChildCount; i++) {
|
||||
const child = body.namedChild(i);
|
||||
if (!child || child.type !== 'method_declaration') continue;
|
||||
const name = child.childForFieldName('name');
|
||||
if (name?.text) names.push(name.text);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function nearestAncestor(node: Parser.SyntaxNode, types: Set<string>): Parser.SyntaxNode | null {
|
||||
let current: Parser.SyntaxNode | null = node;
|
||||
while (current) {
|
||||
if (types.has(current.type)) return current;
|
||||
current = current.parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function bindingScope(varNode: Parser.SyntaxNode): {
|
||||
scope: Parser.SyntaxNode;
|
||||
declarationEnd: number;
|
||||
} | null {
|
||||
const declaration = nearestAncestor(
|
||||
varNode,
|
||||
new Set(['field_declaration', 'local_variable_declaration', 'formal_parameter']),
|
||||
);
|
||||
if (!declaration) return null;
|
||||
|
||||
if (declaration.type === 'field_declaration') {
|
||||
const classBody = nearestAncestor(declaration, new Set(['class_body']));
|
||||
if (!classBody) return null;
|
||||
return { scope: classBody, declarationEnd: 0 };
|
||||
}
|
||||
|
||||
if (declaration.type === 'formal_parameter') {
|
||||
const callable = nearestAncestor(
|
||||
declaration,
|
||||
new Set(['method_declaration', 'constructor_declaration']),
|
||||
);
|
||||
if (!callable) return null;
|
||||
return { scope: callable, declarationEnd: 0 };
|
||||
}
|
||||
|
||||
const block = nearestAncestor(declaration, new Set(['block']));
|
||||
if (!block) return null;
|
||||
return { scope: block, declarationEnd: declaration.endIndex };
|
||||
}
|
||||
|
||||
function resolveServiceForReceiver(
|
||||
bindings: VariableBinding[],
|
||||
receiver: string,
|
||||
callNode: Parser.SyntaxNode,
|
||||
): VariableBinding | null {
|
||||
const callStart = callNode.startIndex;
|
||||
const candidates = bindings.filter(
|
||||
(binding) =>
|
||||
binding.name === receiver &&
|
||||
binding.scopeStart <= callStart &&
|
||||
callStart <= binding.scopeEnd &&
|
||||
binding.declarationEnd <= callStart,
|
||||
);
|
||||
candidates.sort((a, b) => {
|
||||
if (a.scopeSize !== b.scopeSize) return a.scopeSize - b.scopeSize;
|
||||
return b.declarationEnd - a.declarationEnd;
|
||||
});
|
||||
return candidates[0] ?? null;
|
||||
}
|
||||
|
||||
export const JAVA_THRIFT_PLUGIN: ThriftLanguagePlugin = {
|
||||
name: 'java-thrift',
|
||||
language: Java,
|
||||
scan(tree) {
|
||||
const out: ThriftDetection[] = [];
|
||||
const bindings: VariableBinding[] = [];
|
||||
|
||||
for (const match of runCompiledPatterns(VARIABLE_PATTERNS, tree)) {
|
||||
const typeNode = match.captures.type;
|
||||
const varNode = match.captures.var;
|
||||
if (!typeNode || !varNode) continue;
|
||||
const service = serviceFromType(typeNode.text);
|
||||
if (!service) continue;
|
||||
const scope = bindingScope(varNode);
|
||||
if (!scope) continue;
|
||||
bindings.push({
|
||||
name: varNode.text,
|
||||
serviceName: service.serviceName,
|
||||
usesGeneratedServiceMember: service.usesGeneratedServiceMember,
|
||||
scopeStart: scope.scope.startIndex,
|
||||
scopeEnd: scope.scope.endIndex,
|
||||
declarationEnd: scope.declarationEnd,
|
||||
scopeSize: scope.scope.endIndex - scope.scope.startIndex,
|
||||
});
|
||||
}
|
||||
|
||||
for (const match of runCompiledPatterns(CALL_PATTERNS, tree)) {
|
||||
const receiver = match.captures.receiver?.text;
|
||||
const methodName = match.captures.method?.text;
|
||||
const callNode = match.captures.receiver?.parent;
|
||||
if (!receiver || !methodName) continue;
|
||||
if (!callNode) continue;
|
||||
const binding = resolveServiceForReceiver(bindings, receiver, callNode);
|
||||
if (!binding) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName: binding.serviceName,
|
||||
methodName,
|
||||
symbolName: `${receiver}.${methodName}`,
|
||||
source: 'java_thrift_consumer',
|
||||
confidenceWithIdl: 0.75,
|
||||
confidenceWithoutIdl: 0.45,
|
||||
usesGeneratedServiceMember: binding.usesGeneratedServiceMember,
|
||||
});
|
||||
}
|
||||
|
||||
const emittedProviders = new Set<string>();
|
||||
for (const match of runCompiledPatterns(PROVIDER_PATTERNS, tree)) {
|
||||
const typeNode = match.captures.type;
|
||||
const bodyNode = match.captures.body;
|
||||
if (!typeNode || !bodyNode) continue;
|
||||
const service = serviceFromType(typeNode.text);
|
||||
if (!service) continue;
|
||||
|
||||
for (const methodName of methodNamesInClassBody(bodyNode)) {
|
||||
const key = `${service.serviceName}.${methodName}`;
|
||||
if (emittedProviders.has(key)) continue;
|
||||
emittedProviders.add(key);
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName: service.serviceName,
|
||||
methodName,
|
||||
symbolName: `${service.serviceName}.${methodName}`,
|
||||
source: 'java_thrift_provider',
|
||||
confidenceWithIdl: 0.8,
|
||||
confidenceWithoutIdl: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
import type Parser from 'tree-sitter';
|
||||
|
||||
export type ThriftRole = 'provider' | 'consumer';
|
||||
|
||||
export interface ThriftDetection {
|
||||
role: ThriftRole;
|
||||
serviceName: string;
|
||||
methodName: string;
|
||||
symbolName: string;
|
||||
source: string;
|
||||
confidenceWithIdl: number;
|
||||
confidenceWithoutIdl: number;
|
||||
usesGeneratedServiceMember?: boolean;
|
||||
}
|
||||
|
||||
export interface ThriftLanguagePlugin {
|
||||
name: string;
|
||||
language: unknown;
|
||||
scan(tree: Parser.Tree): ThriftDetection[];
|
||||
}
|
||||
@@ -10,8 +10,8 @@ export interface WildcardMatchResult {
|
||||
remaining: StoredContract[];
|
||||
}
|
||||
|
||||
function isGrpcWildcard(cid: string): boolean {
|
||||
return cid.startsWith('grpc::') && cid.endsWith('/*');
|
||||
function isServiceWildcard(cid: string): boolean {
|
||||
return (cid.startsWith('grpc::') || cid.startsWith('thrift::')) && cid.endsWith('/*');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -69,8 +69,9 @@ export function normalizeContractId(id: string): string {
|
||||
}
|
||||
return id;
|
||||
}
|
||||
case 'grpc': {
|
||||
// Canonical form: `grpc::<lowercased-package-or-service>[/<method>]`.
|
||||
case 'grpc':
|
||||
case 'thrift': {
|
||||
// Canonical form: `<type>::<lowercased-package-or-service>[/<method>]`.
|
||||
//
|
||||
// The package/service segment is lowercased because gRPC package
|
||||
// names are effectively case-insensitive across language bindings
|
||||
@@ -84,27 +85,30 @@ export function normalizeContractId(id: string): string {
|
||||
// as DISTINCT canonical forms: `grpc::userservice` does not match
|
||||
// `grpc::userservice/Login`. That's by design — callers that want
|
||||
// service-level manifest matching against method-level providers
|
||||
// should use the gRPC wildcard form `grpc::UserService/*` which is
|
||||
// should use the service wildcard form `grpc::UserService/*` or
|
||||
// `thrift::UserService/*` which is
|
||||
// handled by runWildcardMatch below.
|
||||
const slashIdx = rest.indexOf('/');
|
||||
if (slashIdx > 0) {
|
||||
const pkg = rest.substring(0, slashIdx).toLowerCase();
|
||||
const method = rest.substring(slashIdx);
|
||||
return `grpc::${pkg}${method}`;
|
||||
return `${type}::${pkg}${method}`;
|
||||
}
|
||||
if (slashIdx === 0) {
|
||||
// Malformed "/method" with leading slash — keep as-is so two
|
||||
// equally malformed ids can still match each other.
|
||||
return `grpc::${rest}`;
|
||||
return `${type}::${rest}`;
|
||||
}
|
||||
// No slash: package/service only. Lowercase to match the package
|
||||
// segment produced by the pkg/method branch above.
|
||||
return `grpc::${rest.toLowerCase()}`;
|
||||
return `${type}::${rest.toLowerCase()}`;
|
||||
}
|
||||
case 'topic':
|
||||
return `topic::${rest.trim().toLowerCase()}`;
|
||||
case 'lib':
|
||||
return `lib::${rest.toLowerCase()}`;
|
||||
case 'include':
|
||||
return `include::${rest.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase()}`;
|
||||
default:
|
||||
return id;
|
||||
}
|
||||
@@ -125,6 +129,32 @@ function findMatchingKeys(contractId: string, index: Map<string, StoredContract[
|
||||
return matches;
|
||||
}
|
||||
|
||||
if (normalized.startsWith('thrift::')) {
|
||||
const rest = normalized.substring('thrift::'.length);
|
||||
const slashIdx = rest.indexOf('/');
|
||||
if (slashIdx > 0) {
|
||||
const service = rest.substring(0, slashIdx);
|
||||
const method = rest.substring(slashIdx + 1);
|
||||
if (!service.includes('.') && method && method !== '*') {
|
||||
const matches: string[] = [];
|
||||
for (const key of index.keys()) {
|
||||
if (!key.startsWith('thrift::') || key.endsWith('/*')) continue;
|
||||
const providerRest = key.substring('thrift::'.length);
|
||||
const providerSlashIdx = providerRest.indexOf('/');
|
||||
if (providerSlashIdx < 0) continue;
|
||||
const providerService = providerRest.substring(0, providerSlashIdx);
|
||||
const providerMethod = providerRest.substring(providerSlashIdx + 1);
|
||||
if (providerMethod !== method) continue;
|
||||
if (providerService === service || providerService.endsWith('.' + service)) {
|
||||
matches.push(key);
|
||||
}
|
||||
}
|
||||
matches.sort();
|
||||
return matches.length === 1 ? matches : [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -152,8 +182,9 @@ export function runExactMatch(
|
||||
const isNoisy = buildNoisyContractFilter(matchingConfig);
|
||||
const index = providerIndex ?? buildProviderIndex(contracts, matchingConfig);
|
||||
|
||||
// Skip service wildcard consumers — they go to wildcard pass only
|
||||
const consumers = contracts.filter(
|
||||
(c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId) && !isNoisy(c.contractId),
|
||||
(c) => c.role === 'consumer' && !isServiceWildcard(c.contractId) && !isNoisy(c.contractId),
|
||||
);
|
||||
|
||||
const matched: CrossLink[] = [];
|
||||
@@ -198,15 +229,15 @@ export function runExactMatch(
|
||||
|
||||
// normalUnmatched: contracts that weren't matched in exact pass
|
||||
const normalUnmatched = contracts.filter((c) => {
|
||||
if (isGrpcWildcard(c.contractId)) return false; // excluded from exact, handled separately
|
||||
if (isServiceWildcard(c.contractId)) return false; // excluded from exact, handled separately
|
||||
if (isNoisy(c.contractId)) return false; // excluded from matching — don't surface as unmatched
|
||||
const id = `${c.repo}::${c.contractId}`;
|
||||
return c.role === 'provider' ? !matchedProviderIds.has(id) : !matchedConsumerIds.has(id);
|
||||
});
|
||||
|
||||
// Re-add gRPC wildcard contracts — they were never in exact matching
|
||||
const grpcWildcards = contracts.filter((c) => isGrpcWildcard(c.contractId));
|
||||
const unmatched = [...normalUnmatched, ...grpcWildcards];
|
||||
// Re-add service wildcard contracts — they were never in exact matching
|
||||
const serviceWildcards = contracts.filter((c) => isServiceWildcard(c.contractId));
|
||||
const unmatched = [...normalUnmatched, ...serviceWildcards];
|
||||
|
||||
return { matched, unmatched };
|
||||
}
|
||||
@@ -216,21 +247,28 @@ export function runWildcardMatch(
|
||||
providerIndex: Map<string, StoredContract[]>,
|
||||
): WildcardMatchResult {
|
||||
const wildcardConsumers = unmatched.filter(
|
||||
(c) => c.role === 'consumer' && isGrpcWildcard(c.contractId),
|
||||
(c) => c.role === 'consumer' && isServiceWildcard(c.contractId),
|
||||
);
|
||||
const matched: CrossLink[] = [];
|
||||
const matchedConsumerIds = new Set<string>();
|
||||
|
||||
for (const consumer of wildcardConsumers) {
|
||||
const normalized = normalizeContractId(consumer.contractId);
|
||||
const typeEnd = normalized.indexOf('::');
|
||||
const consumerType = normalized.slice(0, typeEnd);
|
||||
// "grpc::com.example.userservice/*" → "com.example.userservice"
|
||||
// "grpc::userservice/*" → "userservice"
|
||||
const fqService = normalized.slice(normalized.indexOf('::') + 2, -2); // strip "grpc::" and "/*"
|
||||
// "thrift::userservice/*" → "userservice"
|
||||
const fqService = normalized.slice(typeEnd + 2, -2); // strip "<type>::" and "/*"
|
||||
const candidateProviders: StoredContract[] = [];
|
||||
const matchedProviderServices = new Set<string>();
|
||||
|
||||
for (const [key, providers] of providerIndex) {
|
||||
// Only match against non-wildcard gRPC providers (method-level IDs)
|
||||
if (!key.startsWith('grpc::') || key.endsWith('/*')) continue;
|
||||
const afterPrefix = key.slice(6); // strip "grpc::"
|
||||
// Only match against non-wildcard same-type providers (method-level IDs).
|
||||
const keyTypeEnd = key.indexOf('::');
|
||||
if (keyTypeEnd < 0 || key.endsWith('/*')) continue;
|
||||
const providerType = key.slice(0, keyTypeEnd);
|
||||
if (providerType !== consumerType) continue;
|
||||
const afterPrefix = key.slice(keyTypeEnd + 2); // strip "<type>::"
|
||||
const slashIdx = afterPrefix.indexOf('/');
|
||||
if (slashIdx < 0) continue;
|
||||
const providerFqService = afterPrefix.slice(0, slashIdx);
|
||||
@@ -242,39 +280,46 @@ export function runWildcardMatch(
|
||||
|
||||
if (!isMatch) continue;
|
||||
|
||||
for (const provider of providers) {
|
||||
// Skip same-repo same-service (same logic as runExactMatch)
|
||||
if (provider.repo === consumer.repo) {
|
||||
if (!provider.service || !consumer.service || provider.service === consumer.service) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
matchedProviderServices.add(providerFqService);
|
||||
candidateProviders.push(...providers);
|
||||
}
|
||||
|
||||
matched.push({
|
||||
from: {
|
||||
repo: consumer.repo,
|
||||
service: consumer.service,
|
||||
symbolUid: consumer.symbolUid,
|
||||
symbolRef: consumer.symbolRef,
|
||||
},
|
||||
to: {
|
||||
repo: provider.repo,
|
||||
service: provider.service,
|
||||
symbolUid: provider.symbolUid,
|
||||
symbolRef: provider.symbolRef,
|
||||
},
|
||||
type: consumer.type,
|
||||
contractId: consumer.contractId, // consumer's wildcard ID
|
||||
matchType: 'wildcard',
|
||||
confidence: Math.min(provider.confidence, consumer.confidence),
|
||||
});
|
||||
matchedConsumerIds.add(`${consumer.repo}::${consumer.contractId}`);
|
||||
if (consumerType === 'thrift' && !fqService.includes('.') && matchedProviderServices.size > 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const provider of candidateProviders) {
|
||||
// Skip same-repo same-service (same logic as runExactMatch)
|
||||
if (provider.repo === consumer.repo) {
|
||||
if (!provider.service || !consumer.service || provider.service === consumer.service) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
matched.push({
|
||||
from: {
|
||||
repo: consumer.repo,
|
||||
service: consumer.service,
|
||||
symbolUid: consumer.symbolUid,
|
||||
symbolRef: consumer.symbolRef,
|
||||
},
|
||||
to: {
|
||||
repo: provider.repo,
|
||||
service: provider.service,
|
||||
symbolUid: provider.symbolUid,
|
||||
symbolRef: provider.symbolRef,
|
||||
},
|
||||
type: consumer.type,
|
||||
contractId: consumer.contractId, // consumer's wildcard ID
|
||||
matchType: 'wildcard',
|
||||
confidence: Math.min(provider.confidence, consumer.confidence),
|
||||
});
|
||||
matchedConsumerIds.add(`${consumer.repo}::${consumer.contractId}`);
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = unmatched.filter((c) => {
|
||||
if (c.role !== 'consumer' || !isGrpcWildcard(c.contractId)) return true;
|
||||
if (c.role !== 'consumer' || !isServiceWildcard(c.contractId)) return true;
|
||||
return !matchedConsumerIds.has(`${c.repo}::${c.contractId}`);
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
} from './group-path-utils.js';
|
||||
import { getDefaultGitnexusDir, getGroupDir, listGroups, readContractRegistry } from './storage.js';
|
||||
import { syncGroup } from './sync.js';
|
||||
import { logger } from '../logger.js';
|
||||
import type {
|
||||
ContractRegistry,
|
||||
CrossLink,
|
||||
@@ -64,6 +65,15 @@ export interface GroupToolPort {
|
||||
relationTypes: string[];
|
||||
minConfidence: number;
|
||||
includeTests: boolean;
|
||||
// Optional cancellation signal. Callers (notably the cross-impact
|
||||
// Phase-2 fanout) wrap this call in a Promise.race against a
|
||||
// setTimeout-driven AbortController so a single hung neighbor
|
||||
// cannot exceed the request's clamped timeout budget. Implementors
|
||||
// may honor the signal cooperatively or simply let the caller's
|
||||
// race resolve the await — the latter is sufficient for the
|
||||
// resource-exhaustion mitigation. When the signal is absent or
|
||||
// already aborted at call time, behavior is unchanged.
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<unknown | null>;
|
||||
context(
|
||||
@@ -170,11 +180,11 @@ async function loadContractRegistryResilient(
|
||||
contracts.push(row);
|
||||
} else {
|
||||
skippedCorrupt++;
|
||||
console.warn('[group] skipping corrupt contract row in contracts.json');
|
||||
logger.warn('[group] skipping corrupt contract row in contracts.json');
|
||||
}
|
||||
} catch {
|
||||
skippedCorrupt++;
|
||||
console.warn('[group] skipping corrupt contract row in contracts.json');
|
||||
logger.warn('[group] skipping corrupt contract row in contracts.json');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -187,11 +197,11 @@ async function loadContractRegistryResilient(
|
||||
crossLinks.push(row);
|
||||
} else {
|
||||
skippedCorrupt++;
|
||||
console.warn('[group] skipping corrupt crossLinks row in contracts.json');
|
||||
logger.warn('[group] skipping corrupt crossLinks row in contracts.json');
|
||||
}
|
||||
} catch {
|
||||
skippedCorrupt++;
|
||||
console.warn('[group] skipping corrupt crossLinks row in contracts.json');
|
||||
logger.warn('[group] skipping corrupt crossLinks row in contracts.json');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,18 @@ import * as fs from 'node:fs';
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import type { ContractRegistry } from './types.js';
|
||||
import { retryRename } from './bridge-db.js';
|
||||
|
||||
/**
|
||||
* Build an unpredictable suffix for atomic-write tmp files. Replaces the
|
||||
* previous `Date.now()` pattern which CodeQL flagged as
|
||||
* js/insecure-temporary-file: a guessable suffix in a writable directory
|
||||
* lets a co-located attacker pre-create or symlink the tmp path before the
|
||||
* write lands.
|
||||
*/
|
||||
const tmpSuffix = (): string => randomBytes(8).toString('hex');
|
||||
|
||||
const CONTRACTS_FILE = 'contracts.json';
|
||||
|
||||
@@ -34,10 +45,28 @@ export async function writeContractRegistry(
|
||||
registry: ContractRegistry,
|
||||
): Promise<void> {
|
||||
const targetPath = path.join(groupDir, CONTRACTS_FILE);
|
||||
const tmpPath = `${targetPath}.tmp.${Date.now()}`;
|
||||
const tmpPath = `${targetPath}.tmp.${tmpSuffix()}`;
|
||||
|
||||
await fsp.writeFile(tmpPath, JSON.stringify(registry, null, 2), 'utf-8');
|
||||
await fsp.rename(tmpPath, targetPath);
|
||||
// O_EXCL via `'wx'` flag + explicit `0o600` mode — closes both halves
|
||||
// of the CodeQL js/insecure-temporary-file finding: `'wx'` rejects a
|
||||
// pre-planted symlink at the path, and `0o600` (user-only) prevents
|
||||
// the file from being created group/world readable while it briefly
|
||||
// contains contract data en route to the rename. The query's
|
||||
// `isSecureMode` predicate inspects ONLY the mode argument, not the
|
||||
// flags, so the explicit mode is what credits the fix.
|
||||
const handle = await fsp.open(tmpPath, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(JSON.stringify(registry, null, 2), 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
// retryRename absorbs the documented Windows EPERM/EBUSY/EACCES race that
|
||||
// fires when AV scanners or another concurrent rename briefly hold the
|
||||
// destination handle between rename calls. Same helper bridge-db.ts uses
|
||||
// (lines 304, 583, 587, 595, 605, 677) for the bridge.lbug atomic swap —
|
||||
// single source of truth for the Windows-rename pattern across the group
|
||||
// package.
|
||||
await retryRename(tmpPath, targetPath);
|
||||
}
|
||||
|
||||
export async function readContractRegistry(groupDir: string): Promise<ContractRegistry | null> {
|
||||
@@ -106,6 +135,38 @@ matching:
|
||||
# exclude_links_paths: [/ping, /health, /healthcheck]
|
||||
# exclude_links_param_only_paths: false
|
||||
`;
|
||||
await fsp.writeFile(path.join(groupDir, 'group.yaml'), template, 'utf-8');
|
||||
// Always write group.yaml with O_EXCL via `fsp.open(..., 'wx')` —
|
||||
// refuses to follow a pre-planted symlink at the target path, closing
|
||||
// the TOCTOU window between the existence check (line ~98) and the
|
||||
// write that CodeQL js/insecure-temporary-file flags. Under
|
||||
// `force=true` we unlink the existing file first (best-effort, no-op
|
||||
// when absent) so the subsequent O_EXCL open succeeds AND the same
|
||||
// symlink-rejection guarantee holds — this is strictly safer than
|
||||
// the previous `flag: force ? 'w' : 'wx'` shape, which silently
|
||||
// followed symlinks under force. CodeQL's rule does not recognize
|
||||
// the `writeFile(path, content, { flag: 'wx' })` shape as O_EXCL;
|
||||
// the explicit open() handle below is what credits the mitigation.
|
||||
const yamlPath = path.join(groupDir, 'group.yaml');
|
||||
if (force) {
|
||||
try {
|
||||
await fsp.unlink(yamlPath);
|
||||
} catch (err) {
|
||||
// ENOENT (file absent) is expected on first run; rethrow anything
|
||||
// else so we don't silently mask permission/EBUSY failures.
|
||||
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
|
||||
}
|
||||
}
|
||||
// `'wx'` rejects a pre-planted symlink at the path; `0o600` is
|
||||
// user-only (no group/world bits) — gitnexus storage is per-user
|
||||
// (`~/.gitnexus/...`), so any "other user wants to read this" case is
|
||||
// a misconfiguration, not a feature. Keeping the file user-only also
|
||||
// satisfies CodeQL's `isSecureMode` predicate (low 6 bits == 0) and
|
||||
// closes the js/insecure-temporary-file alert at this site.
|
||||
const handle = await fsp.open(yamlPath, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(template, 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return groupDir;
|
||||
}
|
||||
|
||||
@@ -6,15 +6,19 @@ import { readRegistry, type RegistryEntry } from '../../storage/repo-manager.js'
|
||||
import type { GroupConfig, RepoHandle, RepoSnapshot, StoredContract, CrossLink } from './types.js';
|
||||
import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
|
||||
import { GrpcExtractor } from './extractors/grpc-extractor.js';
|
||||
import { ThriftExtractor } from './extractors/thrift-extractor.js';
|
||||
import { TopicExtractor } from './extractors/topic-extractor.js';
|
||||
import { IncludeExtractor } from './extractors/include-extractor.js';
|
||||
import { ManifestExtractor } from './extractors/manifest-extractor.js';
|
||||
import { discoverWorkspaceLinks } from './extractors/workspace-extractor.js';
|
||||
import { runExactMatch } from './matching.js';
|
||||
import { buildProviderIndex, runExactMatch, runWildcardMatch } from './matching.js';
|
||||
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
|
||||
import type { CypherExecutor } from './contract-extractor.js';
|
||||
import { writeContractRegistry } from './storage.js';
|
||||
import { writeBridge } from './bridge-db.js';
|
||||
import type { ContractRegistry } from './types.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
export interface SyncOptions {
|
||||
extractorOverride?:
|
||||
| ((repo: RepoHandle) => Promise<StoredContract[]>)
|
||||
@@ -96,7 +100,9 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
const resolve = opts?.resolveRepoHandle ?? defaultResolveHandle(entries);
|
||||
const httpEx = new HttpRouteExtractor();
|
||||
const grpcEx = new GrpcExtractor();
|
||||
const thriftEx = new ThriftExtractor();
|
||||
const topicEx = new TopicExtractor();
|
||||
const includeEx = new IncludeExtractor();
|
||||
dbExecutors = new Map<string, CypherExecutor>();
|
||||
const openPoolIds: string[] = [];
|
||||
|
||||
@@ -143,6 +149,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
}
|
||||
}
|
||||
|
||||
if (config.detect.thrift) {
|
||||
const extracted = await thriftEx.extract(executor, handle.repoPath, handle);
|
||||
for (const c of extracted) {
|
||||
autoContracts.push({
|
||||
...c,
|
||||
repo: groupPath,
|
||||
service: assignService(c.symbolRef.filePath, boundaries),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (config.detect.topics) {
|
||||
const extracted = await topicEx.extract(executor, handle.repoPath, handle);
|
||||
for (const c of extracted) {
|
||||
@@ -154,6 +171,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
}
|
||||
}
|
||||
|
||||
if (config.detect.includes) {
|
||||
const extracted = await includeEx.extract(executor, handle.repoPath, handle);
|
||||
for (const c of extracted) {
|
||||
autoContracts.push({
|
||||
...c,
|
||||
repo: groupPath,
|
||||
service: assignService(c.symbolRef.filePath, boundaries),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const metaPath = path.join(handle.storagePath, 'meta.json');
|
||||
try {
|
||||
const raw = await fs.readFile(metaPath, 'utf-8');
|
||||
@@ -198,7 +226,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
allLinks = [...allLinks, ...wsResult.links];
|
||||
if (opts?.verbose) {
|
||||
for (const s of wsResult.stats) {
|
||||
console.log(
|
||||
logger.info(
|
||||
` workspace-deps: discovered ${s.linkCount} cross-${s.ecosystem.toLowerCase()} links from ${s.projectCount} ${s.ecosystem} projects`,
|
||||
);
|
||||
}
|
||||
@@ -217,7 +245,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
for (const link of allLinks) {
|
||||
const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
|
||||
if (dangling.length > 0) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`,
|
||||
);
|
||||
}
|
||||
@@ -228,19 +256,21 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
autoContracts.push(...manifestResult.contracts);
|
||||
manifestCrossLinks = manifestResult.crossLinks;
|
||||
if (opts?.verbose) {
|
||||
console.log(
|
||||
logger.info(
|
||||
` manifest: ${manifestCrossLinks.length} cross-links from ${allLinks.length} links (${config.links.length} declared + ${allLinks.length - config.links.length} discovered)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const { matched, unmatched } = runExactMatch(autoContracts, undefined, config.matching);
|
||||
const providerIndex = buildProviderIndex(autoContracts, config.matching);
|
||||
const { matched, unmatched } = runExactMatch(autoContracts, providerIndex, config.matching);
|
||||
const wildcard = runWildcardMatch(unmatched, providerIndex);
|
||||
|
||||
// Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
|
||||
// manifest-declared link can also emit a matchType:'exact' CrossLink with the
|
||||
// same endpoints. Prefer the manifest version — it reflects operator intent
|
||||
// and carries matchType:'manifest' which downstream consumers may rely on.
|
||||
const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
|
||||
const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched, ...wildcard.matched]);
|
||||
const allContracts: StoredContract[] = autoContracts;
|
||||
|
||||
const registry: ContractRegistry = {
|
||||
@@ -254,12 +284,34 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
||||
|
||||
if (opts?.groupDir && !opts.skipWrite) {
|
||||
await writeContractRegistry(opts.groupDir, registry);
|
||||
// writeBridge failure (disk full, schema error, permission denied) must
|
||||
// not mask the registry — contracts.json was just written successfully
|
||||
// and is the canonical source of truth. A stale or absent bridge
|
||||
// degrades impact queries to empty results, which is recoverable on
|
||||
// the next sync. Surface the failure as a warning so operators can
|
||||
// act, but do not propagate it.
|
||||
// (PR #1156 follow-up review: writeBridge error in sync.ts propagates
|
||||
// uncaught.)
|
||||
try {
|
||||
await writeBridge(opts.groupDir, {
|
||||
contracts: allContracts,
|
||||
crossLinks,
|
||||
repoSnapshots,
|
||||
missingRepos,
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
logger.warn(
|
||||
{ err: msg, groupDir: opts.groupDir },
|
||||
'⚠️ writeBridge failed; contracts.json is intact but bridge.lbug is stale. Re-run `gitnexus group sync` to retry.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
contracts: allContracts,
|
||||
crossLinks,
|
||||
unmatched,
|
||||
unmatched: wildcard.remaining,
|
||||
missingRepos,
|
||||
repoSnapshots,
|
||||
};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type ContractType = 'http' | 'grpc' | 'topic' | 'lib' | 'custom';
|
||||
export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include';
|
||||
export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding';
|
||||
export type ContractRole = 'provider' | 'consumer';
|
||||
|
||||
@@ -24,9 +24,11 @@ export interface GroupManifestLink {
|
||||
export interface DetectConfig {
|
||||
http: boolean;
|
||||
grpc: boolean;
|
||||
thrift: boolean;
|
||||
topics: boolean;
|
||||
shared_libs: boolean;
|
||||
embedding_fallback: boolean;
|
||||
includes: boolean;
|
||||
workspace_deps: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import Parser from 'tree-sitter';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
/**
|
||||
* Minimal structural shape consumers need when reading Trees back
|
||||
* through a phase-dependency boundary. Declared here so phases that
|
||||
@@ -49,7 +50,7 @@ export const createASTCache = (maxSize: number = 50): ASTCache => {
|
||||
// will hand freed memory to scope-resolution.
|
||||
(tree as unknown as { delete?: () => void }).delete?.();
|
||||
} catch (e) {
|
||||
console.warn('Failed to delete tree from WASM memory', e);
|
||||
logger.warn({ e }, 'Failed to delete tree from WASM memory');
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
@@ -75,6 +75,7 @@ import { extractReturnTypeName, stripNullable } from './type-extractors/shared.j
|
||||
import type { LiteralTypeInferrer } from './type-extractors/types.js';
|
||||
import type { SyntaxNode } from './utils/ast-helpers.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
/** Per-file resolved type bindings for exported symbols.
|
||||
* Populated during call processing, consumed by Phase 14 re-resolution pass. */
|
||||
export type ExportedTypeMap = Map<string, Map<string, string>>;
|
||||
@@ -768,9 +769,10 @@ export const processCalls = async (
|
||||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
continue;
|
||||
@@ -784,7 +786,7 @@ export const processCalls = async (
|
||||
const query = new Parser.Query(lang, queryStr);
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch (queryError) {
|
||||
console.warn(`Query error for ${file.path}:`, queryError);
|
||||
logger.warn({ queryError }, `Query error for ${file.path}:`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1391,7 +1393,7 @@ export const processCalls = async (
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in call processing — ${lang} parser not available.`,
|
||||
);
|
||||
}
|
||||
@@ -3279,9 +3281,10 @@ export const extractFetchCallsFromFiles = async (
|
||||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
import { CommunityNode } from './community-processor.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
@@ -128,7 +129,7 @@ export const enrichClusters = async (
|
||||
enrichments.set(community.id, enrichment);
|
||||
} catch (error) {
|
||||
// On error, fallback to heuristic
|
||||
console.warn(`Failed to enrich cluster ${community.id}:`, error);
|
||||
logger.warn({ error }, `Failed to enrich cluster ${community.id}:`);
|
||||
enrichments.set(community.id, {
|
||||
name: community.heuristicLabel,
|
||||
keywords: [],
|
||||
@@ -210,7 +211,7 @@ Output JSON array:
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Batch enrichment failed, falling back to heuristics:', error);
|
||||
logger.warn({ error }, 'Batch enrichment failed, falling back to heuristics:');
|
||||
// Fallback for this batch
|
||||
for (const community of batch) {
|
||||
enrichments.set(community.id, {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { logger } from '../../logger.js';
|
||||
/**
|
||||
* COBOL COPY statement expansion engine.
|
||||
*
|
||||
@@ -454,7 +455,7 @@ export function expandCopies(
|
||||
if (visited.has(resolvedPath)) {
|
||||
if (!warnedCircular.has(resolvedPath)) {
|
||||
warnedCircular.add(resolvedPath);
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[cobol-copy-expander] Circular COPY detected: ${cs.target} (${resolvedPath}) ` +
|
||||
`includes itself. Skipping expansion.`,
|
||||
);
|
||||
@@ -464,7 +465,7 @@ export function expandCopies(
|
||||
|
||||
// Max depth exceeded — keep unexpanded
|
||||
if (depth >= maxDepth) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[cobol-copy-expander] Max expansion depth (${maxDepth}) reached for ` +
|
||||
`COPY ${cs.target} in ${srcPath}. Skipping expansion.`,
|
||||
);
|
||||
@@ -475,7 +476,7 @@ export function expandCopies(
|
||||
if (++totalExpansions > MAX_TOTAL_EXPANSIONS) {
|
||||
if (!warnedCircular.has('__max_total__')) {
|
||||
warnedCircular.add('__max_total__');
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[cobol-copy-expander] Max total expansions (${MAX_TOTAL_EXPANSIONS}) reached ` +
|
||||
`in ${srcPath}. Skipping further expansions.`,
|
||||
);
|
||||
|
||||
@@ -369,9 +369,20 @@ const RE_USE_AFTER =
|
||||
/\bUSE\s+(?:AFTER\s+)?(?:STANDARD\s+)?(?:EXCEPTION|ERROR)\s+ON\s+([A-Z][A-Z0-9-]+|INPUT|OUTPUT|I-O|EXTEND)\b/i;
|
||||
|
||||
// SET statement (condition, index)
|
||||
const RE_SET_TO_TRUE = /\bSET\s+((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE\b/i;
|
||||
const RE_SET_INDEX =
|
||||
/\bSET\s+((?:[A-Z][A-Z0-9-]+\s+)+)(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
|
||||
//
|
||||
// Catastrophic-backtracking note (CodeQL js/redos): the previous shape
|
||||
// `((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE`
|
||||
// nested `\s+` quantifiers across alternations and was exponential on
|
||||
// inputs like "SET a OF a OF a ... TO TRUE". Replaced with a lazy
|
||||
// dot-match bounded by the explicit `\s+TO\s+TRUE` suffix — `.+?` is
|
||||
// O(n) with the trailing anchor, and the captured group is parsed
|
||||
// downstream the same way as before.
|
||||
// Exported so the U8 ReDoS regression test can pin the exact production
|
||||
// pattern. Direct import is the only way to ensure the test's
|
||||
// pathological-input timing assertion exercises the production regex
|
||||
// instead of an inline copy that drifts.
|
||||
export const RE_SET_TO_TRUE = /\bSET\s+(.+?)\s+TO\s+TRUE\b/i;
|
||||
export const RE_SET_INDEX = /\bSET\s+(.+?)\s+(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
|
||||
|
||||
// INITIALIZE statement — data reset (captures targets before REPLACING/WITH clause)
|
||||
const RE_INITIALIZE = /\bINITIALIZE\s+([\s\S]*?)(?=\bREPLACING\b|\bWITH\b|\.\s*$|$)/i;
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Unreal Engine reflection-macro preprocessor for C++ source.
|
||||
*
|
||||
* Tree-sitter does not expand C preprocessor macros, so Unreal's reflection
|
||||
* markers (`UCLASS(...)`, `UFUNCTION(...)`, `MODULENAME_API`, ...) are parsed
|
||||
* verbatim. The result is mis-parsed declarations: in `class BRAWLUI_API
|
||||
* UMyClass : public UObject`, tree-sitter-cpp captures `BRAWLUI_API` as the
|
||||
* class name and the rest of the declaration becomes structurally wrong.
|
||||
*
|
||||
* This module elides those macros from the source text BEFORE tree-sitter
|
||||
* parses it. Replacement is **length-preserving** (each elided byte becomes
|
||||
* a space, newlines preserved) so byte offsets and line/column positions
|
||||
* tree-sitter reports remain identical to the original file. Symbol
|
||||
* locations in the graph stay accurate.
|
||||
*
|
||||
* A cheap detection guard short-circuits files that don't look like UE
|
||||
* sources, so non-UE C++ codebases pay no cost.
|
||||
*
|
||||
* Pure function — no tree-sitter dependency, safe for worker threads.
|
||||
*/
|
||||
/**
|
||||
* Strong UE markers — reflection macros that only Unreal Engine projects use.
|
||||
* Presence of one of these is sufficient evidence that the file is a UE source
|
||||
* and that `MODULENAME_API` tokens in it are intended as export macros.
|
||||
*
|
||||
* Importantly, `_API` tokens are NOT in this guard — `REST_API`, `HTTP_API`,
|
||||
* `MY_LIB_API` and similar identifiers appear in plenty of non-UE C++ codebases
|
||||
* as constants/enums/parameter names. We must not erase them just because the
|
||||
* file mentions an `_API` token.
|
||||
*/
|
||||
const HAS_UE_HINT =
|
||||
/\b(?:UCLASS|UFUNCTION|UPROPERTY|USTRUCT|UENUM|UINTERFACE|GENERATED_BODY|GENERATED_[A-Z_]+_BODY|UE_DEPRECATED|DECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE)/;
|
||||
|
||||
const SIMPLE_MACROS_NO_ARGS: readonly string[] = [
|
||||
'GENERATED_BODY',
|
||||
'GENERATED_UCLASS_BODY',
|
||||
'GENERATED_USTRUCT_BODY',
|
||||
'GENERATED_UINTERFACE_BODY',
|
||||
'GENERATED_IINTERFACE_BODY',
|
||||
'DECLARE_CLASS',
|
||||
'GENERATED_BODY_LEGACY',
|
||||
];
|
||||
|
||||
const PARENTHESIZED_MACROS: readonly string[] = [
|
||||
'UCLASS',
|
||||
'UFUNCTION',
|
||||
'UPROPERTY',
|
||||
'USTRUCT',
|
||||
'UENUM',
|
||||
'UINTERFACE',
|
||||
'UMETA',
|
||||
'UE_DEPRECATED',
|
||||
];
|
||||
|
||||
const DELEGATE_MACRO_RE =
|
||||
/\bDECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE(?:_(?:RetVal_OneParam|RetVal_TwoParams|RetVal_ThreeParams|RetVal_FourParams|RetVal_FiveParams|RetVal_SixParams|RetVal_SevenParams|RetVal_EightParams|RetVal_NineParams|RetVal|OneParam|TwoParams|ThreeParams|FourParams|FiveParams|SixParams|SevenParams|EightParams|NineParams|TenParams))?(?=\s*\()/g;
|
||||
|
||||
/**
|
||||
* Module export tokens like `BRAWLUI_API`, `ENGINE_API`, `COREUOBJECT_API`.
|
||||
* Pattern: ALL_CAPS identifier ending in `_API`. The leading word boundary
|
||||
* (`\b`) prevents matching mid-identifier.
|
||||
*/
|
||||
const API_MACRO_RE = /\b[A-Z][A-Z0-9_]*_API\b/g;
|
||||
|
||||
/** Replace `[start, end)` of `chars` with spaces, preserving newlines. */
|
||||
function eraseRange(chars: string[], start: number, end: number): void {
|
||||
for (let i = start; i < end; i++) {
|
||||
if (chars[i] !== '\n' && chars[i] !== '\r') {
|
||||
chars[i] = ' ';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the matching close paren for an opening paren at index `openIdx`.
|
||||
* Returns the index of `)` (inclusive end), or -1 if unbalanced.
|
||||
*
|
||||
* Handles nested parens and string/char literals so commas/parens inside
|
||||
* strings don't throw off the match. Does not attempt to handle raw string
|
||||
* literals (`R"(...)"`); UE reflection-macro arguments do not use them in
|
||||
* practice.
|
||||
*/
|
||||
function findMatchingParen(source: string, openIdx: number): number {
|
||||
if (source.charCodeAt(openIdx) !== 0x28) return -1;
|
||||
let depth = 1;
|
||||
let i = openIdx + 1;
|
||||
const len = source.length;
|
||||
while (i < len && depth > 0) {
|
||||
const ch = source.charCodeAt(i);
|
||||
// String literal
|
||||
if (ch === 0x22) {
|
||||
i++;
|
||||
while (i < len) {
|
||||
const c = source.charCodeAt(i);
|
||||
if (c === 0x5c) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === 0x22) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Char literal
|
||||
if (ch === 0x27) {
|
||||
i++;
|
||||
while (i < len) {
|
||||
const c = source.charCodeAt(i);
|
||||
if (c === 0x5c) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === 0x27) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Line comment
|
||||
if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2f) {
|
||||
while (i < len && source.charCodeAt(i) !== 0x0a) i++;
|
||||
continue;
|
||||
}
|
||||
// Block comment
|
||||
if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2a) {
|
||||
i += 2;
|
||||
while (i < len) {
|
||||
if (source.charCodeAt(i) === 0x2a && source.charCodeAt(i + 1) === 0x2f) {
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === 0x28) depth++;
|
||||
else if (ch === 0x29) {
|
||||
depth--;
|
||||
if (depth === 0) return i;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Match a whole-word identifier at `idx`. Returns the byte after the identifier, or -1 on miss. */
|
||||
function matchIdentifierAt(source: string, idx: number, name: string): number {
|
||||
if (idx > 0) {
|
||||
const prev = source.charCodeAt(idx - 1);
|
||||
if (
|
||||
(prev >= 0x30 && prev <= 0x39) ||
|
||||
(prev >= 0x41 && prev <= 0x5a) ||
|
||||
(prev >= 0x61 && prev <= 0x7a) ||
|
||||
prev === 0x5f
|
||||
) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
for (let k = 0; k < name.length; k++) {
|
||||
if (source.charCodeAt(idx + k) !== name.charCodeAt(k)) return -1;
|
||||
}
|
||||
const after = idx + name.length;
|
||||
if (after < source.length) {
|
||||
const next = source.charCodeAt(after);
|
||||
if (
|
||||
(next >= 0x30 && next <= 0x39) ||
|
||||
(next >= 0x41 && next <= 0x5a) ||
|
||||
(next >= 0x61 && next <= 0x7a) ||
|
||||
next === 0x5f
|
||||
) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Skip ASCII whitespace forward from `idx`. Returns the next non-whitespace byte index. */
|
||||
function skipWhitespace(source: string, idx: number): number {
|
||||
const len = source.length;
|
||||
while (idx < len) {
|
||||
const ch = source.charCodeAt(idx);
|
||||
if (ch === 0x20 || ch === 0x09 || ch === 0x0a || ch === 0x0d) {
|
||||
idx++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip Unreal Engine reflection macros from C++ source, length-preserving.
|
||||
*
|
||||
* Returns the original string unchanged if no strong UE marker is detected,
|
||||
* so non-UE C++ files (including ones that contain `*_API`-suffixed
|
||||
* identifiers like `REST_API` or `HTTP_API`) incur only a single regex test.
|
||||
*
|
||||
* The `_filePath` parameter is part of the `LanguageProvider.preprocessSource`
|
||||
* contract but is unused — UE detection is purely content-based. Accepted and
|
||||
* ignored here so the function matches the hook signature exactly.
|
||||
*/
|
||||
export function stripUeMacros(source: string, _filePath?: string): string {
|
||||
if (!HAS_UE_HINT.test(source)) return source;
|
||||
|
||||
const chars: string[] = source.split('');
|
||||
|
||||
for (const macro of PARENTHESIZED_MACROS) {
|
||||
let searchFrom = 0;
|
||||
while (true) {
|
||||
const hit = source.indexOf(macro, searchFrom);
|
||||
if (hit < 0) break;
|
||||
searchFrom = hit + 1;
|
||||
const after = matchIdentifierAt(source, hit, macro);
|
||||
if (after < 0) continue;
|
||||
const parenIdx = skipWhitespace(source, after);
|
||||
if (source.charCodeAt(parenIdx) !== 0x28) continue;
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close < 0) continue;
|
||||
eraseRange(chars, hit, close + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const macro of SIMPLE_MACROS_NO_ARGS) {
|
||||
let searchFrom = 0;
|
||||
while (true) {
|
||||
const hit = source.indexOf(macro, searchFrom);
|
||||
if (hit < 0) break;
|
||||
searchFrom = hit + 1;
|
||||
const after = matchIdentifierAt(source, hit, macro);
|
||||
if (after < 0) continue;
|
||||
const parenIdx = skipWhitespace(source, after);
|
||||
if (source.charCodeAt(parenIdx) === 0x28) {
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close < 0) continue;
|
||||
eraseRange(chars, hit, close + 1);
|
||||
} else {
|
||||
eraseRange(chars, hit, after);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const re of [DELEGATE_MACRO_RE, API_MACRO_RE]) {
|
||||
re.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = re.exec(source)) !== null) {
|
||||
const start = match.index;
|
||||
let end = start + match[0].length;
|
||||
if (re === DELEGATE_MACRO_RE) {
|
||||
const parenIdx = skipWhitespace(source, end);
|
||||
if (source.charCodeAt(parenIdx) === 0x28) {
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close >= 0) end = close + 1;
|
||||
}
|
||||
}
|
||||
eraseRange(chars, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
return chars.join('');
|
||||
}
|
||||
@@ -9,6 +9,11 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js';
|
||||
|
||||
const CSHARP_VIS = new Set<FieldVisibility>(['public', 'private', 'protected', 'internal']);
|
||||
|
||||
const extractCsharpDeclaredType = (typeNode: SyntaxNode): string | undefined => {
|
||||
if (typeNode.type === 'generic_name') return typeNode.text.trim();
|
||||
return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* C# field extraction config.
|
||||
*
|
||||
@@ -53,17 +58,17 @@ export const csharpConfig: FieldExtractionConfig = {
|
||||
const child = node.namedChild(i);
|
||||
if (child?.type === 'variable_declaration') {
|
||||
const typeNode = child.childForFieldName('type');
|
||||
if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
if (typeNode) return extractCsharpDeclaredType(typeNode);
|
||||
// fallback: first child that is a type
|
||||
const first = child.firstNamedChild;
|
||||
if (first && first.type !== 'variable_declarator') {
|
||||
return extractSimpleTypeName(first) ?? first.text?.trim();
|
||||
return extractCsharpDeclaredType(first);
|
||||
}
|
||||
}
|
||||
}
|
||||
// property_declaration: type is first named child
|
||||
const typeNode = node.childForFieldName('type');
|
||||
if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
if (typeNode) return extractCsharpDeclaredType(typeNode);
|
||||
return undefined;
|
||||
},
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import path from 'path';
|
||||
import { glob } from 'glob';
|
||||
import { createIgnoreFilter } from '../../config/ignore-service.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
export interface FileEntry {
|
||||
path: string;
|
||||
content: string;
|
||||
@@ -74,10 +75,10 @@ export const walkRepositoryPaths = async (
|
||||
if (skippedLarge > 0) {
|
||||
const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES;
|
||||
const suffix = isDefault ? ', likely generated/vendored' : '';
|
||||
console.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
|
||||
logger.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
|
||||
if (isVerboseIngestionEnabled()) {
|
||||
for (const p of skippedLargePaths) {
|
||||
console.warn(` - ${p}`);
|
||||
logger.warn(` - ${p}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import type { ResolutionContext } from './model/resolution-context.js';
|
||||
import { TIER_CONFIDENCE } from './model/resolution-context.js';
|
||||
import type { HeritageInfo } from './heritage-types.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
/**
|
||||
* Derive the heritage-resolution strategy for a language from its
|
||||
* `LanguageProvider`. This is the production wiring that `buildHeritageMap`
|
||||
@@ -218,9 +219,13 @@ export const processHeritage = async (
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
// Use larger bufferSize for files > 32KB
|
||||
// Per-language source preprocessor (length-preserving, e.g. UE macro
|
||||
// stripping for C++). MUST mirror parsing-processor on cache miss so
|
||||
// re-parses see the same input as the cached AST.
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
// Skip files that can't be parsed
|
||||
@@ -237,7 +242,7 @@ export const processHeritage = async (
|
||||
query = new Parser.Query(treeSitterLang, queryStr);
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch (queryError) {
|
||||
console.warn(`Heritage query error for ${file.path}:`, queryError);
|
||||
logger.warn({ queryError }, `Heritage query error for ${file.path}:`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -267,7 +272,7 @@ export const processHeritage = async (
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in heritage processing — ${lang} parser not available.`,
|
||||
);
|
||||
}
|
||||
@@ -412,9 +417,10 @@ export async function extractExtractedHeritageFromFiles(
|
||||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
|
||||
@@ -27,6 +27,7 @@ import type { SyntaxNode } from './utils/ast-helpers.js';
|
||||
import { isDev } from './utils/env.js';
|
||||
import { isRegistryPrimary } from './registry-primary-flag.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
// Type: Map<FilePath, Set<ResolvedFilePath>>
|
||||
// Stores all files that a given file imports from
|
||||
export type ImportMap = Map<string, Set<string>>;
|
||||
@@ -304,9 +305,10 @@ export const processImports = async (
|
||||
let wasReparsed = false;
|
||||
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
continue;
|
||||
@@ -324,14 +326,18 @@ export const processImports = async (
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch (queryError: any) {
|
||||
if (isDev) {
|
||||
console.group(`🔴 Query Error: ${file.path}`);
|
||||
console.log('Language:', language);
|
||||
console.log('Query (first 200 chars):', queryStr.substring(0, 200) + '...');
|
||||
console.log('Error:', queryError?.message || queryError);
|
||||
console.log('File content (first 300 chars):', file.content.substring(0, 300));
|
||||
console.log('AST root type:', tree.rootNode?.type);
|
||||
console.log('AST has errors:', tree.rootNode?.hasError);
|
||||
console.groupEnd();
|
||||
logger.error(
|
||||
{
|
||||
file: file.path,
|
||||
language,
|
||||
err: queryError?.message || queryError,
|
||||
queryPreview: queryStr.substring(0, 200) + '...',
|
||||
contentPreview: file.content.substring(0, 300),
|
||||
astRootType: tree.rootNode?.type,
|
||||
astHasError: tree.rootNode?.hasError,
|
||||
},
|
||||
'tree-sitter query error',
|
||||
);
|
||||
}
|
||||
|
||||
if (wasReparsed) (tree as unknown as { delete?: () => void }).delete?.();
|
||||
@@ -346,7 +352,7 @@ export const processImports = async (
|
||||
const sourceNode = captureMap['import.source'];
|
||||
if (!sourceNode) {
|
||||
if (isDev) {
|
||||
console.log(`⚠️ Import captured but no source node in ${file.path}`);
|
||||
logger.info(`⚠️ Import captured but no source node in ${file.path}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -399,14 +405,14 @@ export const processImports = async (
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in import processing — ${lang} parser not available.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`📊 Import processing complete: ${getResolvedCount()}/${totalImportsFound} imports resolved to graph edges`,
|
||||
);
|
||||
}
|
||||
@@ -498,7 +504,7 @@ export const processImportsFromExtracted = async (
|
||||
);
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`📊 Import processing (fast path): ${getResolvedCount()}/${totalImportsFound} imports resolved to graph edges`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { ImportConfigs } from './import-resolvers/types.js';
|
||||
|
||||
import { isDev } from './utils/env.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
// ============================================================================
|
||||
// LANGUAGE-SPECIFIC CONFIG TYPES
|
||||
// ============================================================================
|
||||
@@ -82,7 +83,7 @@ export async function loadTsconfigPaths(repoRoot: string): Promise<TsconfigPaths
|
||||
|
||||
if (aliases.size > 0) {
|
||||
if (isDev) {
|
||||
console.log(`📦 Loaded ${aliases.size} path aliases from ${filename}`);
|
||||
logger.info(`📦 Loaded ${aliases.size} path aliases from ${filename}`);
|
||||
}
|
||||
return { aliases, baseUrl };
|
||||
}
|
||||
@@ -104,7 +105,7 @@ export async function loadGoModulePath(repoRoot: string): Promise<GoModuleConfig
|
||||
const match = content.match(/^module\s+(\S+)/m);
|
||||
if (match) {
|
||||
if (isDev) {
|
||||
console.log(`📦 Loaded Go module path: ${match[1]}`);
|
||||
logger.info(`📦 Loaded Go module path: ${match[1]}`);
|
||||
}
|
||||
return { modulePath: match[1] };
|
||||
}
|
||||
@@ -132,7 +133,7 @@ export async function loadComposerConfig(repoRoot: string): Promise<ComposerConf
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log(`📦 Loaded ${psr4.size} PSR-4 mappings from composer.json`);
|
||||
logger.info(`📦 Loaded ${psr4.size} PSR-4 mappings from composer.json`);
|
||||
}
|
||||
return { psr4 };
|
||||
} catch {
|
||||
@@ -178,7 +179,7 @@ export async function loadCSharpProjectConfig(repoRoot: string): Promise<CSharpP
|
||||
const projectDir = path.relative(repoRoot, dir).replace(/\\/g, '/');
|
||||
configs.push({ rootNamespace, projectDir });
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`📦 Loaded C# project: ${entry.name} (namespace: ${rootNamespace}, dir: ${projectDir})`,
|
||||
);
|
||||
}
|
||||
@@ -217,7 +218,7 @@ export async function loadSwiftPackageConfig(repoRoot: string): Promise<SwiftPac
|
||||
|
||||
if (targets.size > 0) {
|
||||
if (isDev) {
|
||||
console.log(`📦 Loaded ${targets.size} Swift package targets`);
|
||||
logger.info(`📦 Loaded ${targets.size} Swift package targets`);
|
||||
}
|
||||
return { targets };
|
||||
}
|
||||
|
||||
@@ -116,6 +116,39 @@ interface LanguageProviderConfig {
|
||||
* Required for tree-sitter languages; empty string for standalone processors. */
|
||||
readonly treeSitterQueries: string;
|
||||
|
||||
/**
|
||||
* Optional source-text transform that runs **before** tree-sitter parses the file.
|
||||
*
|
||||
* Used to elide language constructs that confuse the grammar without affecting
|
||||
* source-position fidelity — e.g., Unreal Engine reflection macros (`UCLASS`,
|
||||
* `UFUNCTION`, `MODULENAME_API`) in C++ headers that prevent the parser from
|
||||
* recognising class/function names correctly.
|
||||
*
|
||||
* **Length / position preservation:** the returned string MUST have the same
|
||||
* JavaScript `.length` as the input AND preserve every newline (`\n`/`\r`)
|
||||
* position byte-for-byte. Implementations replace elided characters with
|
||||
* ASCII spaces while leaving newlines untouched. With this contract:
|
||||
*
|
||||
* - tree-sitter's reported `startPosition.row`/`startPosition.column`
|
||||
* match the original file exactly (line/column come from newline counts)
|
||||
* - `startIndex`/`endIndex` byte offsets match the original file exactly
|
||||
* **when the elided range is pure ASCII** (UTF-16 `.length` equals UTF-8
|
||||
* byte length only for ASCII).
|
||||
*
|
||||
* Implementations targeting languages where elided ranges may contain
|
||||
* non-ASCII content must therefore preserve byte length, not just `.length`,
|
||||
* if downstream code uses `startIndex` to slice the original UTF-8 bytes.
|
||||
* The current C++ UE-macro preprocessor relies on the practical fact that
|
||||
* UE reflection macros and module-export tokens are ASCII-only.
|
||||
*
|
||||
* Must be a pure function — same input always yields the same output. Called
|
||||
* once per file, on every code path that re-parses (parsing-processor, import
|
||||
* processor, heritage processor, call processor, parse worker).
|
||||
*
|
||||
* Default: undefined (no preprocessing — `file.content` is parsed verbatim).
|
||||
*/
|
||||
readonly preprocessSource?: (sourceText: string, filePath: string) => string;
|
||||
|
||||
// ── Core (required) ───────────────────────────────────────────────
|
||||
/** Type extraction: declarations, initializers, for-loop bindings */
|
||||
readonly typeConfig: LanguageTypeConfig;
|
||||
|
||||
@@ -45,6 +45,7 @@ import { cVariableConfig, cppVariableConfig } from '../variable-extractors/confi
|
||||
import { createCallExtractor } from '../call-extractors/generic.js';
|
||||
import { cCallConfig, cppCallConfig } from '../call-extractors/configs/c-cpp.js';
|
||||
import { createHeritageExtractor } from '../heritage-extractors/generic.js';
|
||||
import { stripUeMacros } from '../cpp-ue-preprocessor.js';
|
||||
|
||||
const C_BUILT_INS: ReadonlySet<string> = new Set([
|
||||
'printf',
|
||||
@@ -410,6 +411,7 @@ export const cppProvider = defineLanguage({
|
||||
},
|
||||
] satisfies AstFrameworkPatternConfig[],
|
||||
treeSitterQueries: CPP_QUERIES,
|
||||
preprocessSource: stripUeMacros,
|
||||
typeConfig: cCppConfig,
|
||||
exportChecker: cCppExportChecker,
|
||||
importResolver: createImportResolver(cppImportConfig),
|
||||
|
||||
@@ -42,6 +42,39 @@ const FUNCTION_NODE_TYPES = [
|
||||
'local_function_statement',
|
||||
] as const;
|
||||
|
||||
const BUILTIN_TYPE_NAMES = new Set([
|
||||
'bool',
|
||||
'byte',
|
||||
'char',
|
||||
'decimal',
|
||||
'double',
|
||||
'float',
|
||||
'int',
|
||||
'long',
|
||||
'object',
|
||||
'sbyte',
|
||||
'short',
|
||||
'string',
|
||||
'uint',
|
||||
'ulong',
|
||||
'ushort',
|
||||
'void',
|
||||
]);
|
||||
|
||||
function shouldEmitReadMember(memberNode: SyntaxNode): boolean {
|
||||
const parent = memberNode.parent;
|
||||
if (parent === null) return true;
|
||||
|
||||
switch (parent.type) {
|
||||
case 'invocation_expression':
|
||||
return parent.childForFieldName('function')?.id !== memberNode.id;
|
||||
case 'assignment_expression':
|
||||
return parent.childForFieldName('left')?.id !== memberNode.id;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export function emitCsharpScopeCaptures(
|
||||
sourceText: string,
|
||||
_filePath: string,
|
||||
@@ -94,6 +127,14 @@ export function emitCsharpScopeCaptures(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (grouped['@reference.read.member'] !== undefined) {
|
||||
const anchor = grouped['@reference.read.member'];
|
||||
const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_access_expression');
|
||||
if (memberNode === null || !shouldEmitReadMember(memberNode)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize `this` / `base` receiver type-bindings on every
|
||||
// instance method-like. Tree-sitter can't cleanly express "the
|
||||
// implicit receiver of a non-static member of a class/struct/
|
||||
@@ -209,9 +250,63 @@ export function emitCsharpScopeCaptures(
|
||||
}
|
||||
}
|
||||
|
||||
out.push(...synthesizeGenericTypeArgumentReferences(tree.rootNode));
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function synthesizeGenericTypeArgumentReferences(root: SyntaxNode): CaptureMatch[] {
|
||||
const out: CaptureMatch[] = [];
|
||||
// Treat all generic type arguments as static type references, including
|
||||
// declaration signatures and call-site generic instantiations.
|
||||
visit(root, (node) => {
|
||||
if (node.type !== 'generic_name') return;
|
||||
const args = findNamedChild(node, 'type_argument_list');
|
||||
if (args === null) return;
|
||||
|
||||
for (const arg of args.namedChildren) {
|
||||
if (arg === null) continue;
|
||||
const nameNode = terminalTypeNameNode(arg);
|
||||
if (nameNode === null) continue;
|
||||
if (BUILTIN_TYPE_NAMES.has(nameNode.text)) continue;
|
||||
out.push({
|
||||
'@reference.type': nodeToCapture('@reference.type', nameNode),
|
||||
'@reference.name': nodeToCapture('@reference.name', nameNode),
|
||||
});
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function terminalTypeNameNode(node: SyntaxNode): SyntaxNode | null {
|
||||
switch (node.type) {
|
||||
case 'identifier':
|
||||
return node;
|
||||
case 'nullable_type':
|
||||
return node.firstNamedChild === null ? null : terminalTypeNameNode(node.firstNamedChild);
|
||||
case 'qualified_name':
|
||||
return node.lastNamedChild;
|
||||
case 'generic_name':
|
||||
return node.childForFieldName('name') ?? node.firstNamedChild;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function findNamedChild(node: SyntaxNode, type: string): SyntaxNode | null {
|
||||
for (const child of node.namedChildren) {
|
||||
if (child !== null && child.type === type) return child;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function visit(node: SyntaxNode, cb: (node: SyntaxNode) => void): void {
|
||||
cb(node);
|
||||
for (const child of node.namedChildren) {
|
||||
if (child !== null) visit(child, cb);
|
||||
}
|
||||
}
|
||||
|
||||
/** C# 12 primary constructor: `class X(a, b) { }` / `record X(a, b)`.
|
||||
* The parameters are a bare `parameter_list` named child of the type
|
||||
* declaration (no `constructor_declaration` node). Emit a synthetic
|
||||
|
||||
@@ -499,6 +499,12 @@ const CSHARP_SCOPE_QUERY = `
|
||||
left: (member_access_expression
|
||||
expression: "base" @reference.receiver
|
||||
name: (identifier) @reference.name)) @reference.write.member
|
||||
|
||||
;; References — field/property reads: \`obj.Name\`
|
||||
;; Emit-side filtering drops call targets and assignment left-hand sides.
|
||||
(member_access_expression
|
||||
expression: (_) @reference.receiver
|
||||
name: (identifier) @reference.name) @reference.read.member
|
||||
`;
|
||||
|
||||
let _parser: Parser | null = null;
|
||||
|
||||
@@ -30,10 +30,10 @@ export function populateGoRangeBindings(
|
||||
|
||||
for (const rangeNode of tree.rootNode.descendantsOfType('for_statement')) {
|
||||
const rangeClause = rangeNode.namedChildren.find((c) => c.type === 'range_clause');
|
||||
if (rangeClause === null) continue;
|
||||
if (rangeClause === undefined) continue;
|
||||
|
||||
const left = rangeClause.namedChildren.find((c) => c.type === 'expression_list');
|
||||
if (left === null) continue;
|
||||
if (left === undefined) continue;
|
||||
|
||||
const rangeExpr = rangeClause.namedChildren.find(
|
||||
(c, idx) => c.type !== 'expression_list' && idx > rangeClause.namedChildren.indexOf(left),
|
||||
|
||||
@@ -50,7 +50,7 @@ export function synthesizeGoTypeBindings(rootNode: SyntaxNode): CaptureMatch[] {
|
||||
const typeArg = args.namedChildren.find((c) =>
|
||||
['type_identifier', 'qualified_type'].includes(c.type),
|
||||
);
|
||||
if (typeArg !== null) {
|
||||
if (typeArg !== undefined) {
|
||||
const typeName = extractSimpleTypeNameText(typeArg);
|
||||
const nameNodes = lhs.namedChildren.filter((c) => c.type === 'identifier');
|
||||
if (nameNodes.length > 0) {
|
||||
@@ -71,13 +71,13 @@ export function synthesizeGoTypeBindings(rootNode: SyntaxNode): CaptureMatch[] {
|
||||
// V1: channel_type not handled — make(chan T) produces no typeBinding.
|
||||
['slice_type', 'map_type'].includes(c.type),
|
||||
);
|
||||
if (sliceOrMap !== null) {
|
||||
if (sliceOrMap !== undefined) {
|
||||
let typeName = '';
|
||||
if (sliceOrMap.type === 'slice_type') {
|
||||
const elem = sliceOrMap.namedChildren.find((c) =>
|
||||
['type_identifier', 'qualified_type'].includes(c.type),
|
||||
);
|
||||
if (elem !== null) typeName = extractSimpleTypeNameText(elem);
|
||||
if (elem !== undefined) typeName = extractSimpleTypeNameText(elem);
|
||||
} else if (sliceOrMap.type === 'map_type') {
|
||||
const typeChildren = sliceOrMap.namedChildren.filter((c) =>
|
||||
['type_identifier', 'qualified_type'].includes(c.type),
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
*/
|
||||
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { logger } from '../../logger.js';
|
||||
import type {
|
||||
MethodExtractor,
|
||||
MethodExtractorContext,
|
||||
@@ -158,7 +159,7 @@ function findBodies(node: SyntaxNode, bodyNodeSet: Set<string>): SyntaxNode[] {
|
||||
// Fallback: body field exists but its type is not in bodyNodeTypes.
|
||||
// This may indicate a config typo — log for debugging if NODE_ENV is development.
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[MethodExtractor] body field type '${bodyField.type}' not in bodyNodeTypes for node '${node.type}'`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
import type { LanguageProvider } from './language-provider.js';
|
||||
import type { ParsedFile } from 'gitnexus-shared';
|
||||
import { WorkerPool } from './workers/worker-pool.js';
|
||||
import { logger } from '../logger.js';
|
||||
import type {
|
||||
ParseWorkerResult,
|
||||
ParseWorkerInput,
|
||||
@@ -191,7 +192,7 @@ const processParsingWithWorkers = async (
|
||||
const summary = Array.from(skippedLanguages.entries())
|
||||
.map(([lang, count]) => `${lang}: ${count}`)
|
||||
.join(', ');
|
||||
console.warn(` Skipped unsupported languages: ${summary}`);
|
||||
logger.warn(` Skipped unsupported languages: ${summary}`);
|
||||
}
|
||||
|
||||
// Final progress
|
||||
@@ -370,6 +371,11 @@ const processParsingSequential = async (
|
||||
isVueSetup = extracted.isSetup;
|
||||
}
|
||||
|
||||
// Per-language source-text transform (e.g., UE macro stripping for C++).
|
||||
// Length-preserving — see LanguageProvider.preprocessSource contract.
|
||||
parseContent =
|
||||
getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent;
|
||||
|
||||
try {
|
||||
await loadLanguage(language, file.path);
|
||||
} catch {
|
||||
@@ -382,7 +388,7 @@ const processParsingSequential = async (
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
console.warn(`Skipping unparseable file: ${file.path}`);
|
||||
logger.warn(`Skipping unparseable file: ${file.path}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -408,7 +414,7 @@ const processParsingSequential = async (
|
||||
query = new Parser.Query(language, queryString);
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch (queryError) {
|
||||
console.warn(`Query error for ${file.path}:`, queryError);
|
||||
logger.warn({ queryError }, `Query error for ${file.path}:`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -701,7 +707,7 @@ const processParsingSequential = async (
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in parsing processing — ${lang} parser not available.`,
|
||||
);
|
||||
}
|
||||
@@ -742,7 +748,7 @@ export const processParsing = async (
|
||||
// in scope-resolution with an empty cache and get re-parsed.
|
||||
// Surfacing this in PROF mode prevents silent perf cliffs when
|
||||
// a repo crosses the worker-pool threshold.
|
||||
console.warn(
|
||||
logger.warn(
|
||||
`[scope-resolution prof] worker pool engaged for ${files.length} files — cross-phase tree cache will be empty; scope-resolution re-parses.`,
|
||||
);
|
||||
}
|
||||
@@ -757,7 +763,7 @@ export const processParsing = async (
|
||||
);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn('Worker pool parsing stopped; continuing with sequential parser:', message);
|
||||
logger.warn({ message }, 'Worker pool parsing stopped; continuing with sequential parser:');
|
||||
reportProgress?.(
|
||||
lastProgress,
|
||||
files.length,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { readFileContents } from '../filesystem-walker.js';
|
||||
import type { StructureOutput } from './structure.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
export interface CobolOutput {
|
||||
programs: number;
|
||||
paragraphs: number;
|
||||
@@ -47,7 +48,7 @@ export const cobolPhase: PipelinePhase<CobolOutput> = {
|
||||
const cobolResult = processCobol(ctx.graph, cobolFiles, allPathSet);
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
` COBOL: ${cobolResult.programs} programs, ${cobolResult.paragraphs} paragraphs, ${cobolResult.sections} sections from ${cobolFiles.length} files`,
|
||||
);
|
||||
if (
|
||||
@@ -55,12 +56,12 @@ export const cobolPhase: PipelinePhase<CobolOutput> = {
|
||||
cobolResult.execCicsBlocks > 0 ||
|
||||
cobolResult.entryPoints > 0
|
||||
) {
|
||||
console.log(
|
||||
logger.info(
|
||||
` COBOL enriched: ${cobolResult.execSqlBlocks} SQL blocks, ${cobolResult.execCicsBlocks} CICS blocks, ${cobolResult.entryPoints} entry points, ${cobolResult.moves} moves, ${cobolResult.fileDeclarations} file declarations`,
|
||||
);
|
||||
}
|
||||
if (cobolResult.jclJobs > 0) {
|
||||
console.log(` JCL: ${cobolResult.jclJobs} jobs, ${cobolResult.jclSteps} steps`);
|
||||
logger.info(` JCL: ${cobolResult.jclJobs} jobs, ${cobolResult.jclSteps} steps`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { StructureOutput } from './structure.js';
|
||||
import { processCommunities, type CommunityDetectionResult } from '../community-processor.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
export interface CommunitiesOutput {
|
||||
communityResult: CommunityDetectionResult;
|
||||
}
|
||||
@@ -47,7 +48,7 @@ export const communitiesPhase: PipelinePhase<CommunitiesOutput> = {
|
||||
});
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import { topologicalLevelSort } from '../utils/graph-sort.js';
|
||||
import type { KnowledgeGraph } from '../../graph/types.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
/** Max AST trees to keep in LRU cache for cross-file binding propagation. */
|
||||
const AST_CACHE_CAP = 50;
|
||||
|
||||
@@ -60,7 +61,7 @@ export async function runCrossFileBindingPropagation(
|
||||
const { levels, cycleCount } = topologicalLevelSort(ctx.importMap);
|
||||
|
||||
if (isDev && cycleCount > 0) {
|
||||
console.log(`🔄 ${cycleCount} files in import cycles (processed last in undefined order)`);
|
||||
logger.info(`🔄 ${cycleCount} files in import cycles (processed last in undefined order)`);
|
||||
}
|
||||
|
||||
let filesWithGaps = 0;
|
||||
@@ -88,7 +89,7 @@ export async function runCrossFileBindingPropagation(
|
||||
const gapRatio = totalFiles > 0 ? filesWithGaps / totalFiles : 0;
|
||||
if (gapRatio < CROSS_FILE_SKIP_THRESHOLD && filesWithGaps < gapThreshold) {
|
||||
if (isDev) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`⏭️ Cross-file re-resolution skipped (${filesWithGaps}/${totalFiles} files, ${(gapRatio * 100).toFixed(1)}% < ${CROSS_FILE_SKIP_THRESHOLD * 100}% threshold)`,
|
||||
);
|
||||
}
|
||||
@@ -193,7 +194,7 @@ export async function runCrossFileBindingPropagation(
|
||||
|
||||
if (crossFileResolved >= MAX_CROSS_FILE_REPROCESS) {
|
||||
if (isDev)
|
||||
console.log(`⚠️ Cross-file re-resolution capped at ${MAX_CROSS_FILE_REPROCESS} files`);
|
||||
logger.info(`⚠️ Cross-file re-resolution capped at ${MAX_CROSS_FILE_REPROCESS} files`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -204,7 +205,7 @@ export async function runCrossFileBindingPropagation(
|
||||
const elapsed = Date.now() - crossFileStart;
|
||||
const totalElapsed = Date.now() - pipelineStart;
|
||||
const reResolutionPct = totalElapsed > 0 ? ((elapsed / totalElapsed) * 100).toFixed(1) : '0';
|
||||
console.log(
|
||||
logger.info(
|
||||
`🔗 Cross-file re-resolution: ${crossFileResolved} candidates re-processed` +
|
||||
` in ${elapsed}ms (${reResolutionPct}% of total ingestion time so far)`,
|
||||
);
|
||||
|
||||
@@ -36,6 +36,7 @@ import type { ParseOutput } from './parse.js';
|
||||
import { runCrossFileBindingPropagation } from './cross-file-impl.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
export interface CrossFileOutput {
|
||||
/** Number of files re-processed during cross-file propagation. */
|
||||
filesReprocessed: number;
|
||||
@@ -59,11 +60,11 @@ export const crossFilePhase: PipelinePhase<CrossFileOutput> = {
|
||||
if (isDev) {
|
||||
if (bindingAccumulator.totalBindings > 0) {
|
||||
const memKB = Math.round(bindingAccumulator.estimateMemoryBytes() / 1024);
|
||||
console.log(
|
||||
logger.info(
|
||||
`📦 BindingAccumulator: ${bindingAccumulator.totalBindings} bindings across ${bindingAccumulator.fileCount} files (~${memKB} KB)`,
|
||||
);
|
||||
} else if (totalFiles > 0) {
|
||||
console.log(
|
||||
logger.info(
|
||||
`📦 BindingAccumulator: EMPTY — 0 bindings across 0 files despite ${totalFiles} parsed files. If the codebase has typed bindings, this indicates an upstream regression.`,
|
||||
);
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user