Datasets:
26B-Suite
/
Gemma4-FinetuneKit

Dataset card Data Studio Files
xet
 Community
1
text
stringlengths
1
118
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# QLoRA on Goetia-26B-A4B — plain transformers + peft + trl\n",
"\n",
"This is the notebook I used for `Dark-Goetia-26B-A4B-LoRA` v2, cleaned up\n",
"and with the comments translated.\n",
"\n",
"**Environment it actually ran on:** torch 2.12.0+cu130, transformers 5.14.1,\n",
"single A100-SXM4-80GB (rented). ~51 min for 2 epochs, 1489 sessions.\n",
"\n",
"**Config:** QLoRA nf4, attention-only targets, r=32 / alpha=64, lr 2e-5,\n",
"max_len 2048, batch 1 x grad_accum 8.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Cell 1 — dependencies. After install: Kernel -> Restart.\n",
"!pip install --no-cache-dir -U \"transformers>=4.46\" \"peft>=0.13\" \"trl>=0.12\" \\\n",
" \"bitsandbytes>=0.44\" \"accelerate>=1.0\" datasets sentencepiece protobuf"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Cell 2 — environment sanity check\n",
"import torch, transformers\n",
"print(\"torch\", torch.__version__, \"| CUDA\", torch.version.cuda)\n",
"print(\"GPU count:\", torch.cuda.device_count())\n",
"for i in range(torch.cuda.device_count()):\n",
" p = torch.cuda.get_device_properties(i)\n",
" print(f\" [{i}] {p.name}, {p.total_memory/1e9:.0f} GB\")\n",
"print(\"transformers\", transformers.__version__)\n",
"assert torch.cuda.device_count() >= 1"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Cell 3 — config\n",
"MODEL_ID = \"Naphula/Goetia-26B-A4B-v1.3-Absolute-Heretic-ARA\"\n",
"MODEL_DIR = \"./goetia\"\n",
"DATA_PATH = \"./dataset.jsonl\"\n",
"OUT_DIR = \"./goetia-lora-v2\"\n",
"EPOCHS = 2\n",
"MAX_LEN = 2048 # i tried 3072, but catch OOM\n",
"RANK = 32\n",
"ALPHA = 64"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Cell 5 — dataset: ShareGPT jsonl -> messages format\n",
"import json\n",
"from datasets import Dataset\n",
"\n",
"ROLE = {\"system\": \"system\", \"human\": \"user\", \"gpt\": \"assistant\"}\n",
"rows = []\n",
"with open(DATA_PATH, encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if not line:\n",
" continue\n",
" conv = json.loads(line)[\"conversations\"]\n",
" rows.append({\"messages\": [{\"role\": ROLE[m[\"from\"]], \"content\": m[\"value\"]}\n",
" for m in conv]})\n",
"print(\"sessions:\", len(rows))\n",
"\n",
"# Cheap guard: a session with no assistant turn contributes nothing but\n",
"# still eats a training step.\n",
"assert all(any(m[\"role\"] == \"assistant\" for m in r[\"messages\"]) for r in rows)\n",
"\n",
"split = max(10, len(rows) // 20)\n",
"ds_train = Dataset.from_list(rows[:-split])\n",
"ds_eval = Dataset.from_list(rows[-split:])\n",
"print(\"train\", len(ds_train), \"| eval\", len(ds_eval))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],