Added hook to flush Jupyter notebook outputs

hooks/pre-commit

@@ -0,0 +1,13 @@
+#!/bin/bash
+for f in $(git diff --name-only --cached); do
+    if [[ $f == *.ipynb ]]; then
+        jupyter nbconvert --clear-output --inplace $f
+        git add $f
+    fi
+done
+
+if git diff --name-only --cached --exit-code
+then
+    echo "No changes detected after removing notebook output"
+    exit 1
+fi

raft/raft_ollama_dataset.ipynb

@@ -582,10 +582,88 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "id": "74493d61",
    "metadata": {},
    "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080\">Synthesizing for: corpus/</span><span style=\"color: #000080; text-decoration-color: #000080\">topic</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">2__part</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">066__n</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">60.</span><span style=\"color: #000080; text-decoration-color: #000080\">txt </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">(</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">24</span><span style=\"color: #000080; text-decoration-color: #000080\"> chunks</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">)</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[34mSynthesizing for: corpus/\u001b[0m\u001b[34mtopic\u001b[0m\u001b[34m=\u001b[0m\u001b[34m2__part\u001b[0m\u001b[34m=\u001b[0m\u001b[34m066__n\u001b[0m\u001b[34m=\u001b[0m\u001b[1;34m60\u001b[0m\u001b[1;34m.\u001b[0m\u001b[34mtxt \u001b[0m\u001b[1;34m(\u001b[0m\u001b[1;34m24\u001b[0m\u001b[34m chunks\u001b[0m\u001b[1;34m)\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080\">Progress: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"color: #008080; text-decoration-color: #008080\">/</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">265</span><span style=\"color: #008080; text-decoration-color: #008080\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span><span style=\"color: #008080; text-decoration-color: #008080\">%</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">)</span><span style=\"color: #008080; text-decoration-color: #008080\"> completed</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[36mProgress: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[36m/\u001b[0m\u001b[1;36m265\u001b[0m\u001b[36m \u001b[0m\u001b[1;36m(\u001b[0m\u001b[1;36m0.4\u001b[0m\u001b[36m%\u001b[0m\u001b[1;36m)\u001b[0m\u001b[36m completed\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080\">Synthesizing for: corpus/</span><span style=\"color: #000080; text-decoration-color: #000080\">topic</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">5__part</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">018__n</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">60.</span><span style=\"color: #000080; text-decoration-color: #000080\">txt </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">(</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">52</span><span style=\"color: #000080; text-decoration-color: #000080\"> chunks</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">)</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[34mSynthesizing for: corpus/\u001b[0m\u001b[34mtopic\u001b[0m\u001b[34m=\u001b[0m\u001b[34m5__part\u001b[0m\u001b[34m=\u001b[0m\u001b[34m018__n\u001b[0m\u001b[34m=\u001b[0m\u001b[1;34m60\u001b[0m\u001b[1;34m.\u001b[0m\u001b[34mtxt \u001b[0m\u001b[1;34m(\u001b[0m\u001b[1;34m52\u001b[0m\u001b[34m chunks\u001b[0m\u001b[1;34m)\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080\">Progress: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"color: #008080; text-decoration-color: #008080\">/</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">265</span><span style=\"color: #008080; text-decoration-color: #008080\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span><span style=\"color: #008080; text-decoration-color: #008080\">%</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">)</span><span style=\"color: #008080; text-decoration-color: #008080\"> completed</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[36mProgress: \u001b[0m\u001b[1;36m2\u001b[0m\u001b[36m/\u001b[0m\u001b[1;36m265\u001b[0m\u001b[36m \u001b[0m\u001b[1;36m(\u001b[0m\u001b[1;36m0.8\u001b[0m\u001b[36m%\u001b[0m\u001b[1;36m)\u001b[0m\u001b[36m completed\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080\">Synthesizing for: corpus/</span><span style=\"color: #000080; text-decoration-color: #000080\">topic</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">7__part</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080\">005__n</span><span style=\"color: #000080; text-decoration-color: #000080\">=</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">60.</span><span style=\"color: #000080; text-decoration-color: #000080\">txt </span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">(</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">49</span><span style=\"color: #000080; text-decoration-color: #000080\"> chunks</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">)</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[34mSynthesizing for: corpus/\u001b[0m\u001b[34mtopic\u001b[0m\u001b[34m=\u001b[0m\u001b[34m7__part\u001b[0m\u001b[34m=\u001b[0m\u001b[34m005__n\u001b[0m\u001b[34m=\u001b[0m\u001b[1;34m60\u001b[0m\u001b[1;34m.\u001b[0m\u001b[34mtxt \u001b[0m\u001b[1;34m(\u001b[0m\u001b[1;34m49\u001b[0m\u001b[34m chunks\u001b[0m\u001b[1;34m)\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080\">Progress: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"color: #008080; text-decoration-color: #008080\">/</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">265</span><span style=\"color: #008080; text-decoration-color: #008080\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.1</span><span style=\"color: #008080; text-decoration-color: #008080\">%</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">)</span><span style=\"color: #008080; text-decoration-color: #008080\"> completed</span>\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\u001b[36mProgress: \u001b[0m\u001b[1;36m3\u001b[0m\u001b[36m/\u001b[0m\u001b[1;36m265\u001b[0m\u001b[36m \u001b[0m\u001b[1;36m(\u001b[0m\u001b[1;36m1.1\u001b[0m\u001b[36m%\u001b[0m\u001b[1;36m)\u001b[0m\u001b[36m completed\u001b[0m\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "ename": "KeyboardInterrupt",
      "evalue": "",
@@ -593,8 +671,8 @@
      "traceback": [
       "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
       "\u001b[31mKeyboardInterrupt\u001b[39m                         Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[13]\u001b[39m\u001b[32m, line 43\u001b[39m\n\u001b[32m     40\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m[green]Wrote \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_target\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m rows -> \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mout_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m[/green]\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m     41\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m out_path\n\u001b[32m---> \u001b[39m\u001b[32m43\u001b[39m OUT_JSONL = \u001b[43msynthesize_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43msamples_per_doc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mSAMPLES_PER_DOC\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     44\u001b[39m OUT_JSONL\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 48\u001b[39m\n\u001b[32m     45\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m[green]Wrote \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_target\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m rows -> \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mout_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m[/green]\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m     46\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m out_path\n\u001b[32m---> \u001b[39m\u001b[32m48\u001b[39m OUT_JSONL = \u001b[43msynthesize_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43msamples_per_doc\u001b[49m\u001b[43m=\u001b[49m\u001b[43mSAMPLES_PER_DOC\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     49\u001b[39m OUT_JSONL\n",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[13]\u001b[39m\u001b[32m, line 20\u001b[39m, in \u001b[36msynthesize_dataset\u001b[39m\u001b[34m(samples_per_doc, out_path)\u001b[39m\n\u001b[32m     18\u001b[39m ctx, ids = build_context(pi, k=TOP_K)\n\u001b[32m     19\u001b[39m user = USER_PROMPT_TEMPLATE.format(context=ctx, n=\u001b[32m3\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m20\u001b[39m raw = \u001b[43mollama_generate\u001b[49m\u001b[43m(\u001b[49m\u001b[43muser\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mGEN_MODEL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m=\u001b[49m\u001b[43mTEMPERATURE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_predict\u001b[49m\u001b[43m=\u001b[49m\u001b[43mMAX_TOKENS_GEN\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     21\u001b[39m rows = parse_llm_jsonl(raw)\n\u001b[32m     22\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m rows:\n\u001b[32m     23\u001b[39m     \u001b[38;5;66;03m# enforce schema & enrich meta\u001b[39;00m\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 25\u001b[39m, in \u001b[36msynthesize_dataset\u001b[39m\u001b[34m(samples_per_doc, out_path)\u001b[39m\n\u001b[32m     23\u001b[39m ctx, ids = build_context(pi, k=TOP_K)\n\u001b[32m     24\u001b[39m user = USER_PROMPT_TEMPLATE.format(context=ctx, n=\u001b[32m3\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m25\u001b[39m raw = \u001b[43mollama_generate\u001b[49m\u001b[43m(\u001b[49m\u001b[43muser\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mGEN_MODEL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m=\u001b[49m\u001b[43mTEMPERATURE\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_predict\u001b[49m\u001b[43m=\u001b[49m\u001b[43mMAX_TOKENS_GEN\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     26\u001b[39m rows = parse_llm_jsonl(raw)\n\u001b[32m     27\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m rows:\n\u001b[32m     28\u001b[39m     \u001b[38;5;66;03m# enforce schema & enrich meta\u001b[39;00m\n",
       "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 28\u001b[39m, in \u001b[36mollama_generate\u001b[39m\u001b[34m(prompt, model, temperature, num_predict)\u001b[39m\n\u001b[32m     17\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mollama_generate\u001b[39m(prompt: \u001b[38;5;28mstr\u001b[39m, model: \u001b[38;5;28mstr\u001b[39m = GEN_MODEL, temperature: \u001b[38;5;28mfloat\u001b[39m = TEMPERATURE, num_predict: \u001b[38;5;28mint\u001b[39m = MAX_TOKENS_GEN) -> \u001b[38;5;28mstr\u001b[39m:\n\u001b[32m     18\u001b[39m     payload = {\n\u001b[32m     19\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mmodel\u001b[39m\u001b[33m\"\u001b[39m: model,\n\u001b[32m     20\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mprompt\u001b[39m\u001b[33m\"\u001b[39m: prompt,\n\u001b[32m   (...)\u001b[39m\u001b[32m     26\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mstream\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m     27\u001b[39m     }\n\u001b[32m---> \u001b[39m\u001b[32m28\u001b[39m     r = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43mGEN_ENDPOINT\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpayload\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     29\u001b[39m     r.raise_for_status()\n\u001b[32m     30\u001b[39m     data = r.json()\n",
       "\u001b[36mFile \u001b[39m\u001b[32m~/repo/jupyter-ai-test/.env/lib/python3.12/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m    103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m    104\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m    105\u001b[39m \n\u001b[32m    106\u001b[39m \u001b[33;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m    112\u001b[39m \u001b[33;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[32m    113\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
       "\u001b[36mFile \u001b[39m\u001b[32m~/repo/jupyter-ai-test/.env/lib/python3.12/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m     55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m     56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m     57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m     58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",