.eval_results/cron-eval-sakthai-coder-1.5b-2026-07-31-1.yaml

target_model:
  id: Nanthasit/sakthai-coder-1.5b
  pipeline_tag: text-generation
  library_name: transformers
  base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct
  downloads: 93
  likes: 0
  private: false
  gated: false
  last_modified: "2026-07-31T05:21:13.000Z"
  model_age_days: 6.7969
  model_type: llm
  has_weights: true

architecture:
  base_model_type: qwen2
  base_architectures: ["Qwen2ForCausalLM"]
  base_hidden_size: 1536
  base_num_hidden_layers: 28
  base_num_attention_heads: 12
  base_num_key_value_heads: 2
  base_intermediate_size: 8960
  base_vocab_size: 151936
  base_max_position_embeddings: 32768
  base_total_parameters: 1540000000
  base_dtype: bfloat16
  tie_word_embeddings: true
  # GGUF-only repo: no config.json at root; architecture verified from
  # Qwen/Qwen2.5-Coder-1.5B-Instruct config.json (fetched live 2026-07-31)

repo_summary:
  siblings_count: 1569
  total_repo_bytes: 1289288771
  total_gb: 1.289
  has_weights: true
  weight_file_count: 1
  weight_bytes: 1117320768
  weight_files: ["qwen2.5-coder-1.5b-instruct-q4_k_m.gguf"]
  config_present: false
  tokenizer_present: false
  chat_template_present: true
  readme_present: true
  readme_size_bytes: 19380
  weight_note: "Single Q4_K_M GGUF (1.12 GB) for llama.cpp/Ollama; tokenizer and chat template are embedded in the GGUF, hence no standalone config.json/tokenizer files. Repo carries a stray dev environment (.venv/, .pytest_cache/, .hypothesis/, .ruff_cache/, .superpowers/, .usage.json) from an over-eager push — documented in README, cleanup commit planned."

benchmarks:
  model_index_count: 1
  metrics_count: 4
  all_verified: false
  pending_metrics: 4
  entries:
    - dataset: openai_humaneval
      metric: pass@1
      value: 74.4
      verified: false
      note: base-model reference (Qwen2.5-Coder-1.5B-Instruct), not re-run on fine-tune
    - dataset: mbpp
      metric: pass@1
      value: 71.2
      verified: false
      note: base-model reference ceiling
    - dataset: multipl_e
      metric: pass@1
      value: 65.3
      verified: false
      note: base-model reference ceiling
    - dataset: SakThai Coding Suite (internal)
      metric: pass@1
      value: 100
      verified: false
      note: internal single-trial llama.cpp run (5/5 tasks, 2026-07-25)
  notes: >
    All 4 model-index metrics are verified: false (base-model references +
    one internal single-trial suite). Independent 2026-07-31 3-trial llama.cpp
    benchmark (.eval_results/benchmark-20260731_031937.yaml) on tool-calling
    code-search got 0/3 tool calls, 0/3 valid JSON, 2/3 hallucinated files at
    14.5 tps — a real gap vs the card's '5/5 tool tasks' claim that should be
    reconciled. Recommended: sakthai-bench-v2 (500 rows, held-out tools).

training:
  dataset: Nanthasit/sakthai-combined-v6
  dataset_size: 2309
  dataset_note: "v6 + v7 (2,309 train / 115 test, verified 2026-07-31) + irrelevance-supplement (60 rows)"
  training_method: QLoRA (4-bit) → GGUF Q4_K_M
  eval_split: "115 held-out examples"
  lora_config:
    r: 16
    alpha: 32
  hardware: "Free Google Colab GPU (T4), $0 budget"
  key_improvements:
    - "Code + tool-calling in one session"
    - "CPU-friendly Q4_K_M GGUF for llama.cpp / Ollama"

card_quality:
  license: apache-2.0
  base_model_documented: true
  base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct
  tags_count: 16
  tags: [code, coder, qwen2.5, qwen2.5-coder, gguf, llama-cpp, llama.cpp, ollama, code-generation, tool-calling, conversational, cpu-inference, small-language-model, offline, sakthai, house-of-sak]
  datasets_cited: ["Nanthasit/sakthai-combined-v6", "Nanthasit/sakthai-combined-v7", "Nanthasit/sakthai-irrelevance-supplement"]
  model_index_present: true
  readme_size_bytes: 19380
  widget_example: "Write a Python function that checks if a string is a palindrome, handling spaces and punctuation:"
  deductions:
    - "model-index metrics all verified: false (base refs + single-trial internal)"
    - "card claims 5/5 tool tasks but 2026-07-31 benchmark showed 0/3 tool calls"
    - "stray dev environment inflates repo to 1,568 files / 1.29 GB"
  score: 88

health_score:
  overall: 43.8
  components:
    popularity: 0.93
    momentum: 100
    benchmarks: 0
    card_quality: 88
    repo_hygiene: 40
  weights:
    popularity: 0.20
    momentum: 0.20
    benchmarks: 0.25
    card_quality: 0.20
    repo_hygiene: 0.15

sibling_comparison:
  rank_by_downloads: 11
  total_author_models: 19
  max_sibling_downloads: 1599
  models_with_positive_downloads: 11
  velocity_rank: 9
  max_sibling_velocity: 62.58
  our_velocity: 13.68

eval_type: metadata_cron
eval_note: >
  First cron eval snapshot for sakthai-coder-1.5b (run 15 of hf-eval-updater).
  The family's code specialist: Qwen2.5-Coder-1.5B-Instruct QLoRA fine-tune
  shipped as a 1.12 GB Q4_K_M GGUF for CPU use. Mid-pack momentum — 93
  downloads, 13.68 dl/day (9/11 velocity, 11/19 downloads). Card quality is
  high (19 KB README, badges, family table, honest caveats) but the model-index
  carries only unverified base-reference scores, and a fresh 3-trial llama.cpp
  tool-calling run failed to reproduce the card's 5/5 tool claim (0/3 tool
  calls, hallucinated files). Repo hygiene is the weak spot: 40/100 due to a
  stray dev environment. Next cycle: run sakthai-bench-v2, reconcile the
  tool-calling claim, and land the cleanup commit.

eval_metadata:
  model: Nanthasit/sakthai-coder-1.5b
  eval_date: "2026-07-31"
  eval_time: "05:27:13Z"
  schema: llm_cron_v1
  age_days: 6.7969
  days_since_last_update: 0.0042
  download_velocity: 13.68
  cron_run: 15