refactor: replace mdFixer with finalizer and reducer, add ngram script

2026-03-22 07:29:35 +08:00 · 2025-06-11 17:02:33 -07:00
parent 7965ce1167
commit 1fef3c26d9
9 changed files with 298 additions and 28 deletions
--- a/jina-ai/config.json
+++ b/jina-ai/config.json
@@ -59,7 +59,8 @@
          "maxTokens": 8000,
          "model": "gemini-2.0-flash-lite"
        },
-        "mdFixer": {}
+        "finalizer": {},
+        "reducer": {"maxTokens": 16000}
      }
    },
    "openai": {
@@ -87,7 +88,8 @@
        "fallback": {
          "temperature": 0
        },
-        "mdFixer": {}
+        "finalizer": {},
+        "reducer": {"maxTokens": 16000}
      }
    }
  }