File size: 2,377 Bytes
94cbe85 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """Generate, verify, and Modal-prep the Figment v14 delta SFT corpus."""
from __future__ import annotations
import argparse
from pathlib import Path
import sys
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from scripts.generate_finetune_data import V14_NAVIGATOR_COUNTS # noqa: E402
from scripts.generate_v3_full_corpus import main as _run_full_corpus # noqa: E402
DEFAULT_OUTPUT_VERSION = "figment_sft_v14_delta"
DEFAULT_TEACHER_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b"
DEFAULT_NAVIGATOR_COUNT = sum(V14_NAVIGATOR_COUNTS.values())
DEFAULT_REPAIR_COUNT = 0
DEFAULT_ARGS = [
"--dataset-version",
DEFAULT_OUTPUT_VERSION,
"--navigator-count",
str(DEFAULT_NAVIGATOR_COUNT),
"--repair-count",
str(DEFAULT_REPAIR_COUNT),
"--teacher-model-id",
DEFAULT_TEACHER_MODEL_ID,
"--no-teacher-worker",
"--base-start-index",
"240000",
"--rows-per-shard",
"10",
"--parallelism",
"2",
"--timeout-seconds",
"120",
"--teacher-error-retries",
"20",
"--teacher-error-sleep-seconds",
"20",
"--shard-prefix",
"data/finetune/shards/figment_sft_v14_delta_full_shard",
"--output",
"data/finetune/figment_sft_v14_delta.jsonl",
"--case-specs",
"data/finetune/figment_sft_v14_delta_case_specs.jsonl",
"--manifest",
"data/finetune/figment_sft_v14_delta_manifest.json",
"--modal-output-dir",
"data/finetune/modal/figment_sft_v14_delta",
"--seed",
"figment-modal-sft-v14-delta",
]
def build_corpus_args(argv: list[str] | None = None) -> list[str]:
raw_args = list(sys.argv[1:] if argv is None else argv)
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("--navigator-count", type=int, default=DEFAULT_NAVIGATOR_COUNT)
parser.add_argument("--repair-count", type=int, default=DEFAULT_REPAIR_COUNT)
parsed, remaining = parser.parse_known_args(raw_args)
defaults = list(DEFAULT_ARGS)
defaults[defaults.index("--navigator-count") + 1] = str(parsed.navigator_count)
defaults[defaults.index("--repair-count") + 1] = str(parsed.repair_count)
return defaults + remaining
def main(argv: list[str] | None = None) -> int:
return _run_full_corpus(build_corpus_args(argv))
if __name__ == "__main__":
raise SystemExit(main())
|