fix(install): vendor tree-sitter-dart source (#1125)

Avoid remote git/SSH downloads for the Dart grammar during Docker and npm installs by resolving tree-sitter-dart from vendored source and building it during postinstall. Made-with: Cursor
2026-04-27 21:15:37 +01:00
parent 38ccf7ceb1
commit 780ee83d52
22 changed files with 227807 additions and 28 deletions
@@ -65,7 +65,7 @@
      "optionalDependencies": {
        "node-addon-api": "^8.0.0",
        "node-gyp-build": "^4.8.0",
-        "tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
+        "tree-sitter-dart": "file:./vendor/tree-sitter-dart",
        "tree-sitter-kotlin": "^0.3.8",
        "tree-sitter-proto": "file:./vendor/tree-sitter-proto",
        "tree-sitter-swift": "^0.6.0"
@@ -5100,31 +5100,8 @@
      }
    },
    "node_modules/tree-sitter-dart": {
-      "version": "1.0.0",
-      "resolved": "git+ssh://git@github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
-      "integrity": "sha512-Bs/1wAOIJ2akPEXlE/XVpuES19Oo3NqoSJRJ/0N2r38qAd9nTXdqmaGHQ44/JXnA6QHcbgD2YzCCc4wUc98cyQ==",
-      "hasInstallScript": true,
-      "license": "ISC",
-      "optional": true,
-      "dependencies": {
-        "node-addon-api": "^7.1.0",
-        "node-gyp-build": "^4.8.0"
-      },
-      "peerDependencies": {
-        "tree-sitter": "^0.21.0"
-      },
-      "peerDependenciesMeta": {
-        "tree_sitter": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/tree-sitter-dart/node_modules/node-addon-api": {
-      "version": "7.1.1",
-      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
-      "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
-      "license": "MIT",
-      "optional": true
+      "resolved": "vendor/tree-sitter-dart",
+      "link": true
    },
    "node_modules/tree-sitter-go": {
      "version": "0.23.4",
@@ -5768,6 +5745,19 @@
        "zod": "^3.25.28 || ^4"
      }
    },
+    "vendor/tree-sitter-dart": {
+      "version": "1.0.0",
+      "license": "ISC",
+      "optional": true,
+      "peerDependencies": {
+        "tree-sitter": "^0.21.0"
+      },
+      "peerDependenciesMeta": {
+        "tree_sitter": {
+          "optional": true
+        }
+      }
+    },
    "vendor/tree-sitter-proto": {
      "version": "0.4.1",
      "license": "MIT",
@@ -47,7 +47,7 @@
    "test:integration": "vitest run test/integration",
    "test:watch": "vitest",
    "test:coverage": "vitest run --coverage",
-    "postinstall": "node scripts/patch-tree-sitter-swift.cjs && node scripts/build-tree-sitter-proto.cjs",
+    "postinstall": "node scripts/patch-tree-sitter-swift.cjs && node scripts/build-tree-sitter-dart.cjs && node scripts/build-tree-sitter-proto.cjs",
    "prepare": "node scripts/build.js",
    "prepack": "node scripts/build.js"
  },
@@ -88,7 +88,7 @@
  "optionalDependencies": {
    "node-addon-api": "^8.0.0",
    "node-gyp-build": "^4.8.0",
-    "tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
+    "tree-sitter-dart": "file:./vendor/tree-sitter-dart",
    "tree-sitter-kotlin": "^0.3.8",
    "tree-sitter-proto": "file:./vendor/tree-sitter-proto",
    "tree-sitter-swift": "^0.6.0"
@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart');
+const bindingGyp = path.join(dartDir, 'binding.gyp');
+const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node');
+
+try {
+  if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) {
+    process.exit(0);
+  }
+
+  try {
+    require.resolve('node-addon-api');
+    require.resolve('node-gyp-build');
+  } catch (resolveErr) {
+    console.warn(
+      '[tree-sitter-dart] Skipping build: hoisted build deps not resolvable (%s).',
+      resolveErr.message,
+    );
+    console.warn(
+      '[tree-sitter-dart] Dart parsing will be unavailable. Install without --no-optional and with scripts enabled to build.',
+    );
+    process.exit(0);
+  }
+
+  console.log('[tree-sitter-dart] Building native binding...');
+  execSync('npx node-gyp rebuild', {
+    cwd: dartDir,
+    stdio: 'pipe',
+    timeout: 180000,
+  });
+  console.log('[tree-sitter-dart] Native binding built successfully');
+} catch (err) {
+  console.warn('[tree-sitter-dart] Could not build native binding:', err.message);
+  console.warn(
+    '[tree-sitter-dart] Dart parsing will be unavailable. Non-Dart functionality is unaffected.',
+  );
+  process.exit(0);
+}
@@ -41,6 +41,15 @@ describe('CLI commands', () => {
    });
  });

+  describe('optional parser dependencies', () => {
+    it('uses vendored source for tree-sitter-dart instead of a remote dependency', async () => {
+      const pkg = await import('../../package.json', { with: { type: 'json' } });
+      expect(pkg.default.optionalDependencies['tree-sitter-dart']).toBe(
+        'file:./vendor/tree-sitter-dart',
+      );
+    });
+  });
+
  describe('analyzeCommand', () => {
    it('is a function', async () => {
      const { analyzeCommand } = await import('../../src/cli/analyze.js');
@@ -0,0 +1,2 @@
+build/
+node_modules/
@@ -0,0 +1,18 @@
+# tree-sitter-dart
+This is a tree-sitter grammar written for the dart programming language. We attempt to adhere as closely as possible to the dart language spec. Initially it was started with a copy of the tree-sitter Java grammar which is why there may be a few relics included in here. For the sake of simplifying the syntax tree, many items were made inlined with tree-sitter's "underscore" method of writing rules.
+
+# Getting Started
+- Go to the project directory
+- run `npm install` (first time)
+- run `npm run build_init` (first time) `npm run build` (subsequent times)
+- run `npm run test`
+
+# To test a single highlight file
+- run `tree-sitter highlight test/highlight/types.dart`
+
+# To test a single test file
+- run `tree-sitter test -f 'testcasefilter'`
+- for example `tree-sitter test -f 'dart string literals'`
+
+# To show the output of a parse for a sample file (for example while debugging highlight issues)
+- run `tree-sitter parse path/to/file.dart`
@@ -0,0 +1,31 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_dart_binding",
+      "dependencies": [
+        "<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
+      ],
+      "include_dirs": [
+        "src",
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+        # NOTE: if your language has an external scanner, add it here.
+        "src/scanner.c",
+      ],
+      "conditions": [
+        ["OS!='win'", {
+          "cflags_c": [
+            "-std=c11",
+          ],
+        }, { # OS == "win"
+          "cflags_c": [
+            "/std:c11",
+            "/utf-8",
+          ],
+        }],
+      ],
+    }
+  ]
+}
@@ -0,0 +1,20 @@
+#include <napi.h>
+
+typedef struct TSLanguage TSLanguage;
+
+extern "C" TSLanguage *tree_sitter_dart();
+
+// "tree-sitter", "language" hashed with BLAKE2
+const napi_type_tag LANGUAGE_TYPE_TAG = {
+  0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
+};
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+    exports["name"] = Napi::String::New(env, "dart");
+    auto language = Napi::External<TSLanguage>::New(env, tree_sitter_dart());
+    language.TypeTag(&LANGUAGE_TYPE_TAG);
+    exports["language"] = language;
+    return exports;
+}
+
+NODE_API_MODULE(tree_sitter_dart_binding, Init)
@@ -0,0 +1,28 @@
+type BaseNode = {
+  type: string;
+  named: boolean;
+};
+
+type ChildNode = {
+  multiple: boolean;
+  required: boolean;
+  types: BaseNode[];
+};
+
+type NodeInfo =
+  | (BaseNode & {
+      subtypes: BaseNode[];
+    })
+  | (BaseNode & {
+      fields: { [name: string]: ChildNode };
+      children: ChildNode[];
+    });
+
+type Language = {
+  name: string;
+  language: unknown;
+  nodeTypeInfo: NodeInfo[];
+};
+
+declare const language: Language;
+export = language;
@@ -0,0 +1,7 @@
+const root = require("path").join(__dirname, "..", "..");
+
+module.exports = require("node-gyp-build")(root);
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}
@@ -0,0 +1,18 @@
+{
+  "name": "tree-sitter-dart",
+  "version": "1.0.0",
+  "description": "Dart grammar attempt for tree-sitter",
+  "repository": "https://github.com/UserNobody14/tree-sitter-dart",
+  "license": "ISC",
+  "main": "bindings/node",
+  "types": "bindings/node",
+  "_vendoredBy": "gitnexus - pinned to UserNobody14/tree-sitter-dart commit 80e23c07b64494f7e21090bb3450223ef0b192f4. Build deps are hoisted into gitnexus/package.json optionalDependencies, and native compilation is performed by gitnexus/scripts/build-tree-sitter-dart.cjs at gitnexus postinstall.",
+  "peerDependencies": {
+    "tree-sitter": "^0.21.0"
+  },
+  "peerDependenciesMeta": {
+    "tree_sitter": {
+      "optional": true
+    }
+  }
+}
@@ -0,0 +1,246 @@
+; Variable
+(identifier) @variable
+
+; Keywords
+; --------------------
+[
+    (assert_builtin)
+    (break_builtin)
+    (const_builtin)
+    (part_of_builtin)
+    (rethrow_builtin)
+    (void_type)
+    "abstract"
+    "as"
+    "async"
+    "async*"
+    "await"
+    "base"
+    "case"
+    "catch"
+    "class"
+    "continue"
+    "covariant"
+    "default"
+    "deferred"
+    "do"
+    "dynamic"
+    "else"
+    "enum"
+    "export"
+    "extends"
+    "extension"
+    "external"
+    "factory"
+    "final"
+    "finally"
+    "for"
+    "Function"
+    "get"
+    "hide"
+    "if"
+    "implements"
+    "import"
+    "in"
+    "interface"
+    "is"
+    "late"
+    "library"
+    "mixin"
+    "new"
+    "on"
+    "operator"
+    "part"
+    "required"
+    "return"
+    "sealed"
+    "set"
+    "show"
+    "static"
+    "super"
+    "switch"
+    "sync*"
+    "throw"
+    "try"
+    "typedef"
+    "var"
+    "when"
+    "while"
+    "with"
+    "yield"
+] @keyword
+
+; Methods
+; --------------------
+
+; NOTE: This query is a bit of a work around for the fact that the dart grammar doesn't
+; specifically identify a node as a function call
+(((identifier) @function (#match? @function "^_?[a-z]"))
+ . (selector . (argument_part))) @function
+
+; Annotations
+; --------------------
+(annotation
+  name: (identifier) @attribute)
+
+; Operators and Tokens
+; --------------------
+(template_substitution
+  "$" @punctuation.special
+  "{" @punctuation.special
+  "}" @punctuation.special
+) @none
+
+(template_substitution
+  "$" @punctuation.special
+  (identifier_dollar_escaped) @variable
+) @none
+
+(escape_sequence) @string.escape
+
+[
+ "@"
+ "=>"
+ ".."
+ "??"
+ "=="
+ "?"
+ ":"
+ "&&"
+ "%"
+ "<"
+ ">"
+ "="
+ ">="
+ "<="
+ "||"
+ "~/"
+ (increment_operator)
+ (is_operator)
+ (prefix_operator)
+ (equality_operator)
+ (additive_operator)
+] @operator
+
+(type_arguments
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+
+(type_parameters
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+]  @punctuation.bracket
+
+; Delimiters
+; --------------------
+[
+  ";"
+  "."
+  ","
+] @punctuation.delimiter
+
+; Types
+; --------------------
+(type_identifier) @type
+((type_identifier) @type.builtin
+  (#match? @type.builtin "^(int|double|String|bool|List|Set|Map|Runes|Symbol)$"))
+(class_definition
+  name: (identifier) @type)
+(constructor_signature
+  name: (identifier) @type)
+(scoped_identifier
+  scope: (identifier) @type)
+(function_signature
+  name: (identifier) @function)
+(getter_signature
+  (identifier) @function)
+(setter_signature
+  name: (identifier) @function)
+
+((scoped_identifier
+  scope: (identifier) @type
+  name: (identifier) @type)
+ (#match? @type "^[a-zA-Z]"))
+
+; Enums
+; -------------------
+(enum_declaration
+  name: (identifier) @type)
+(enum_constant
+  name: (identifier) @identifier.constant)
+
+; Variables
+; --------------------
+; var keyword
+(inferred_type) @keyword
+
+((identifier) @type
+ (#match? @type "^_?[A-Z].*[a-z]"))
+
+("Function" @type)
+
+(this) @variable.builtin
+
+; properties
+
+(unconditional_assignable_selector
+  (identifier) @property)
+
+(conditional_assignable_selector
+  (identifier) @property)
+
+(cascade_section
+  (cascade_selector
+    (identifier) @property))
+
+((selector
+  (unconditional_assignable_selector (identifier) @function))
+  (selector (argument_part (arguments)))
+)
+
+(cascade_section
+  (cascade_selector (identifier) @function)
+  (argument_part (arguments))
+)
+
+; assignments
+(assignment_expression
+  left: (assignable_expression) @variable)
+
+(this) @variable.builtin
+
+; Parameters
+; --------------------
+(formal_parameter
+    name: (identifier) @identifier.parameter)
+
+(named_argument
+  (label (identifier) @identifier.parameter))
+
+; Literals
+; --------------------
+[
+    (hex_integer_literal)
+    (decimal_integer_literal)
+    (decimal_floating_point_literal)
+    ; TODO: inaccessbile nodes
+    ; (octal_integer_literal)
+    ; (hex_floating_point_literal)
+] @number
+
+(string_literal) @string
+(symbol_literal (identifier) @constant) @constant
+(true) @boolean
+(false) @boolean
+(null_literal) @constant.null
+
+(documentation_comment) @comment
+(comment) @comment
@@ -0,0 +1,92 @@
+
+(class_definition
+  name: (identifier) @name) @definition.class
+
+(method_signature
+  (function_signature)) @definition.method
+
+(type_alias
+  (type_identifier) @name) @definition.type
+
+(method_signature
+(getter_signature
+  name: (identifier) @name)) @definition.method
+
+(method_signature
+(setter_signature
+  name: (identifier) @name)) @definition.method 
+
+(method_signature
+  (function_signature
+  name: (identifier) @name)) @definition.method
+
+(method_signature
+  (factory_constructor_signature
+    (identifier) @name)) @definition.method
+
+(method_signature
+  (constructor_signature
+  name: (identifier) @name)) @definition.method
+
+(method_signature
+  (operator_signature)) @definition.method
+
+(method_signature) @definition.method
+
+(mixin_declaration
+  (mixin)
+  (identifier) @name) @definition.mixin
+
+(extension_declaration
+  name: (identifier) @name) @definition.extension
+
+
+(new_expression
+  (type_identifier) @name) @reference.class
+
+(enum_declaration
+  name: (identifier) @name) @definition.enum
+
+(function_signature
+  name: (identifier) @name) @definition.function 
+
+(initialized_variable_definition
+  name: (identifier)
+  value: (identifier) @name 
+  value: (selector
+	"!"?
+	(argument_part 
+	  (arguments
+	    (argument)*))?)?) @reference.class
+
+(assignment_expression
+  left: (assignable_expression 
+		  (identifier)
+		  (unconditional_assignable_selector
+			"."
+			(identifier) @name))) @reference.call
+
+(assignment_expression
+  left: (assignable_expression 
+		  (identifier)
+		  (conditional_assignable_selector
+			"?."
+			(identifier) @name))) @reference.call
+
+((identifier) @name
+ (selector
+    "!"?
+    (conditional_assignable_selector
+      "?." (identifier) @name)?
+    (unconditional_assignable_selector
+      "."? (identifier) @name)?
+    (argument_part
+      (arguments
+        (argument)*))?)*
+	(cascade_section
+	  (cascade_selector
+		(identifier)) @name 
+	  (argument_part 
+		(arguments
+		  (argument)*))?)?) @reference.call
+
@@ -0,0 +1 @@
+(if_statement (block))
@@ -0,0 +1,130 @@
+#include "tree_sitter/parser.h"
+#include <wctype.h>
+
+enum TokenType {
+  TEMPLATE_CHARS_SINGLE,
+  TEMPLATE_CHARS_DOUBLE,
+  TEMPLATE_CHARS_SINGLE_SINGLE,
+  TEMPLATE_CHARS_DOUBLE_SINGLE,
+  TEMPLATE_CHARS_RAW_SLASH,
+  BLOCK_COMMENT,
+  DOCUMENTATION_BLOCK_COMMENT,
+};
+
+void *tree_sitter_dart_external_scanner_create() { return NULL; }
+void tree_sitter_dart_external_scanner_destroy(void *p) {}
+void tree_sitter_dart_external_scanner_reset(void *p) {}
+unsigned tree_sitter_dart_external_scanner_serialize(void *p, char *buffer) { return 0; }
+void tree_sitter_dart_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+static bool scan_multiline_comments(TSLexer *lexer) {
+
+    bool documentation_comment = false;
+    advance(lexer);
+    if (lexer->lookahead != '*') return false;
+    advance(lexer);
+    if (lexer->lookahead == '*') documentation_comment = true;
+
+    bool after_star = false;
+    unsigned nesting_depth = 1;
+    for (;;) {
+      switch (lexer->lookahead) {
+        case '\0':
+          return false;
+        case '*':
+          advance(lexer);
+          after_star = true;
+          break;
+        case '/':
+          if (after_star) {
+            advance(lexer);
+            after_star = false;
+            nesting_depth--;
+            if (nesting_depth == 0) {
+              if (!documentation_comment) {
+                lexer->result_symbol = BLOCK_COMMENT;
+              } else {
+                lexer->result_symbol = DOCUMENTATION_BLOCK_COMMENT;
+              }
+              return true;
+            }
+          } else {
+            advance(lexer);
+            after_star = false;
+            if (lexer->lookahead == '*') {
+              nesting_depth++;
+              advance(lexer);
+            }
+          }
+          break;
+        default:
+          advance(lexer);
+          after_star = false;
+          break;
+      }
+    }
+  return false;
+}
+
+
+static bool scan_templates(TSLexer *lexer, const bool *valid_symbols) {
+  if(valid_symbols[TEMPLATE_CHARS_DOUBLE]) {
+              lexer->result_symbol = TEMPLATE_CHARS_DOUBLE;
+  } else if (valid_symbols[TEMPLATE_CHARS_SINGLE]) {
+              lexer->result_symbol = TEMPLATE_CHARS_SINGLE;
+  } else if (valid_symbols[TEMPLATE_CHARS_SINGLE_SINGLE]) {
+              lexer->result_symbol = TEMPLATE_CHARS_SINGLE_SINGLE;
+  } else {
+              lexer->result_symbol = TEMPLATE_CHARS_DOUBLE_SINGLE;
+  }
+  for (bool has_content = false;; has_content = true) {
+    lexer->mark_end(lexer);
+    switch (lexer->lookahead) {
+      case '\'':
+      case '"':
+        return has_content;
+      case '\n':
+        if (valid_symbols[TEMPLATE_CHARS_DOUBLE_SINGLE] || valid_symbols[TEMPLATE_CHARS_SINGLE_SINGLE]) return false;
+        advance(lexer);
+        break;
+      case '\0':
+        return false;
+      case '$':
+        return has_content;
+      case '\\':
+        if (valid_symbols[TEMPLATE_CHARS_RAW_SLASH]) {
+            lexer->result_symbol = TEMPLATE_CHARS_RAW_SLASH;
+            advance(lexer);
+        } else {
+            return has_content;
+        }
+        break;
+      default:
+        advance(lexer);
+    }
+  }
+  return true;
+}
+
+
+
+bool tree_sitter_dart_external_scanner_scan(void *payload, TSLexer *lexer,
+                                                  const bool *valid_symbols) {
+  if (
+      valid_symbols[TEMPLATE_CHARS_DOUBLE] ||
+      valid_symbols[TEMPLATE_CHARS_SINGLE] ||
+      valid_symbols[TEMPLATE_CHARS_DOUBLE_SINGLE] ||
+      valid_symbols[TEMPLATE_CHARS_SINGLE_SINGLE]
+  ) {
+    return scan_templates(lexer, valid_symbols);
+  }
+  while (iswspace(lexer->lookahead)) lexer->advance(lexer, true);
+
+  if (lexer->lookahead == '/') {
+    return scan_multiline_comments(lexer);
+  }
+  return false;
+}
@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t);
+extern void *(*ts_current_calloc)(size_t, size_t);
+extern void *(*ts_current_realloc)(void *, size_t);
+extern void (*ts_current_free)(void *);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_
@@ -0,0 +1,290 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count); \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(default : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_
@@ -0,0 +1,265 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSFieldMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+};
+
+static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_