{ "cells": [ { "cell_type": "code", "execution_count": 6, "id": "19f3dd6b", "metadata": {}, "outputs": [], "source": [ "# Seed examples for task bootstrapping\n", "tasks_with_difficulty = {\n", " # lewis\n", " \"Evaluate models {M_i} on benchmarks {B_i}\": \"Easy\",\n", " \"Train models {M_i} on datasets {D_i} with benchmarks {B_i}\": \"Medium\",\n", " \"Run an ablation for hyperparameter P for model M on dataset D\": \"Hard\",\n", " \"Generate completions with model M on dataset D using engine E\": \"Medium\",\n", " \"Merge models {M_i} using linear averaging to find the best result on benchmarks {B_i}\": \"Hard\",\n", " \"Given datasets {D_i}, ablate the best SFT mixture for model M across benchmarks {B_i}\": \"Very hard\",\n", " \"Decontaminate dataset D against benchmarks {B_i}\": \"Hard\",\n", " \"Benchmark RL framework F for best throughput on G GPUs\": \"Very hard\",\n", " \"Implement post-training algorithm A from paper P in framework F. Validate it runs end-to-end\": \"Very hard\",\n", " \"Implement benchmark B in framework F. Validate it reproduces some published results\": \"Very hard\",\n", " \"Format dataset D for compatibility with framework F on task T\": \"Easy\",\n", "\n", " # abubakar\n", " \"Remove the background from this image: [image path]\": \"Easy\",\n", " \"Transcribe all of the audio files in this directory\": \"Easy\",\n", " \"Transcribe all of the audio files in this directory, choose the model that'll be cheapest and also relatively accurate\": \"Medium (judgment call or interaction needed to figure out what accuracy levels are acceptable)\",\n", " \"Remove the background music from this audio file\": \"Medium (needs to find Gradio Space and call its API0\",\n", " \"Change this video track to be from English to Spanish\": \"Medium (needs to link several models together)\",\n", " \"Translate this flyer from English to Spanish, keeping the layout and images the same\": \"Medium (needs to link several models together)\",\n", "\n", " # leandro\n", " \"What's the best model for X?\": \"Easy\",\n", " \"What datasets are available for X? (X={domain x task x modality})\": \"Easy\",\n", " \"Is there a space to do Y?\": \"Easy\",\n", " \"I have this script and this error - what's the issue?\": \"Medium\",\n", " \"This space is broken, how can i fix it?\": \"Medium\",\n", " \"I built a space but it is super slow. What can I do?\": \"Medium\",\n", " \"How can I run modal X locally?\": \"Medium\",\n", " \"I want to build a space with model Y to do X?\": \"Hard\",\n", " \"How can I serve a model with multiple LoRAs?\": \"Hard\",\n", "\n", " # claude\n", " \"What's the best model for sentiment analysis on financial text?\": \"Easy\",\n", " \"Are there any medical image segmentation datasets on HuggingFace for CT scans?\": \"Easy\",\n", " \"Which text classification models support 4-bit quantization?\": \"Medium\",\n", " \"Are there inference endpoints available for Whisper large-v3?\": \"Easy\",\n", " \"What's the license for the SA-Med2D-20M dataset?\": \"Easy\",\n", " \"Which vision models fit in 8GB VRAM for image segmentation?\": \"Medium\",\n", " \"What datasets are available for 3D medical image segmentation?\": \"Medium\",\n", " \"Is there a space to do text-to-speech with emotion control?\": \"Medium\",\n", " \"I'm getting \\\"CUDA out of memory\\\" when loading Llama-2-7b even though nvidia-smi shows I have 6GB free - what's the issue?\": \"Medium\",\n", " \"My Gradio space shows \\\"Connection errored out\\\" after working fine yesterday, no code changes - how can I fix it?\": \"Medium\",\n", " \"I built a Gradio space for Stable Diffusion but inference takes 5+ minutes on a 4090 - what can I do?\": \"Medium\",\n", " \"My Whisper model outputs different transcriptions after quantization to int8 - why?\": \"Medium\",\n", " \"Getting \\\"RuntimeError: CUDA error: out of memory. Tried to allocate 70.00 MiB\\\" but only 2.87 GiB is allocated - what's happening?\": \"Medium\",\n", " \"My HuggingFace space build fails with \\\"failed to create containerd task\\\" - how to fix?\": \"Medium\",\n", " \"DistilBERT model gives \\\"you should probably train your model\\\" warning even though it's a pretrained model from the Hub\": \"Easy\",\n", " \"Space was working fine but now receiving build errors - receiving this error even with a new space\": \"Medium\",\n", " \"Inference is correct locally but wrong on deployed space\": \"Medium\",\n", " \"Getting CUDA OOM despite having enough memory according to nvidia-smi\": \"Medium\",\n", " \"How can I run Mistral-7B-v0.1 locally with multiple LoRA adapters?\": \"Hard\",\n", " \"How can I serve Llama-2-7b with vLLM and dynamically load multiple LoRA adapters?\": \"Hard\",\n", " \"How do I batch inference requests in my Gradio space for better throughput?\": \"Medium\",\n", " \"Can I run Whisper large-v3 with faster-whisper for 4x speedup?\": \"Medium\",\n", " \"How to run Llama 2 on CPU after fine-tuning with LoRA?\": \"Medium\",\n", " \"Best way to handle 50+ concurrent requests in a Gradio space without OOM?\": \"Hard\",\n", " \"How do I add custom stopping criteria for text generation with Transformers?\": \"Hard\",\n", " \"Can I merge multiple LoRA adapters before inference to reduce latency?\": \"Hard\",\n", " \"How can I optimize my LLM inference with one base LLM and multiple LoRA adapters?\": \"Hard\",\n", "}\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "c7014bef", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "53" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(tasks_with_difficulty)" ] }, { "cell_type": "code", "execution_count": null, "id": "3a8bd7ed", "metadata": {}, "outputs": [], "source": [ "import litellm\n", "import json\n", "from pydantic import BaseModel\n", "from enum import Enum\n", "\n", "\n", "class Difficulty(str, Enum):\n", " EASY = \"Easy\"\n", " MEDIUM = \"Medium\"\n", " HARD = \"Hard\"\n", " VERY_HARD = \"Very hard\"\n", "\n", "\n", "class Task(BaseModel):\n", " description: str\n", " difficulty: Difficulty\n", "\n", "\n", "class GeneratedTasks(BaseModel):\n", " tasks: list[Task]\n", "\n", "\n", "def build_prompt(tasks_dict: dict[str, str]) -> str:\n", " task_descriptions = \"\".join(\n", " [f'- \"{task}\" [{difficulty}]\\n' for task, difficulty in tasks_dict.items()]\n", " )\n", "\n", " return f\"\"\"Given the following examples of tasks (with their estimated difficulty levels in brackets):\n", "\n", "{task_descriptions}\n", "\n", "Generate exactly 10 new unique tasks with their difficulty levels (Easy, Medium, Hard, or Very hard).\n", "The new tasks should be bootstrapped by analogy or creative mutation of the provided ones, but not be direct copies.\n", "Vary the domains, instructions, and scenario details. Write crisp, concrete task phrasing. Preserve variety in both tasks and difficulties.\n", "Do not repeat any of the input tasks verbatim. Create plausible, meaningful tasks relevant to LLM training, evaluation, dataprocessing, issue handling, tooling, etc.\n", "\"\"\"\n", "\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "85ef3dcb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Iteration 1/20: Added 10 new tasks. Total: 63\n", "Iteration 2/20: Added 10 new tasks. Total: 73\n", "Iteration 3/20: Added 10 new tasks. Total: 83\n", "Iteration 4/20: Added 10 new tasks. Total: 93\n", "Iteration 5/20: Added 10 new tasks. Total: 103\n", "Iteration 6/20: Added 10 new tasks. Total: 113\n", "Iteration 7/20: Added 10 new tasks. Total: 123\n", "Iteration 8/20: Added 10 new tasks. Total: 133\n", "Iteration 9/20: Added 10 new tasks. Total: 143\n", "Iteration 10/20: Added 10 new tasks. Total: 153\n", "Iteration 11/20: Added 10 new tasks. Total: 163\n", "Iteration 12/20: Added 10 new tasks. Total: 173\n", "Iteration 13/20: Added 10 new tasks. Total: 183\n", "Iteration 14/20: Added 10 new tasks. Total: 193\n", "Iteration 15/20: Added 10 new tasks. Total: 203\n", "Iteration 16/20: Added 10 new tasks. Total: 213\n", "Iteration 17/20: Added 10 new tasks. Total: 223\n", "Iteration 18/20: Added 10 new tasks. Total: 233\n", "Iteration 19/20: Added 10 new tasks. Total: 243\n", "Iteration 20/20: Added 10 new tasks. Total: 253\n", "\n", "Final task count: 253\n" ] } ], "source": [ "model_name = \"gpt-5\"\n", "\n", "# Number of iterations to generate tasks (10 tasks per iteration)\n", "num_iterations = 20\n", "\n", "# Copy the seed tasks to avoid modifying the original\n", "all_tasks = tasks_with_difficulty.copy()\n", "\n", "for i in range(num_iterations):\n", " prompt = build_prompt(all_tasks)\n", "\n", " # Query LLM using litellm with structured output\n", " response = litellm.completion(\n", " model=model_name,\n", " messages=[\n", " {\n", " \"role\": \"system\",\n", " \"content\": \"You are an expert at generating diverse ML/AI task instructions using products from HuggingFace and can enumerate them with proper difficulty.\",\n", " },\n", " {\"role\": \"user\", \"content\": prompt},\n", " ],\n", " response_format=GeneratedTasks,\n", " )\n", "\n", " # Parse the structured output\n", " generated = GeneratedTasks.model_validate_json(\n", " response.choices[0].message.content\n", " )\n", "\n", " # Add new tasks to the dictionary\n", " new_count = 0\n", " for task in generated.tasks:\n", " if task.description not in all_tasks:\n", " all_tasks[task.description] = task.difficulty.value\n", " new_count += 1\n", "\n", " print(f\"Iteration {i + 1}/{num_iterations}: Added {new_count} new tasks. Total: {len(all_tasks)}\")\n", "\n", "# Save to disk\n", "with open(\"generated_tasks_with_difficulty.json\", \"w\") as f:\n", " json.dump(all_tasks, f, indent=2)\n", "\n", "print(f\"\\nFinal task count: {len(all_tasks)}\")\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "9c0ad570", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Dataset: 253 rows\n", "Sample: Evaluate models {M_i} on benchmarks {B_i} (Easy)\n" ] } ], "source": [ "from datasets import Dataset\n", "\n", "# Convert dict to proper columns\n", "questions = list(all_tasks.keys())\n", "difficulties = list(all_tasks.values())\n", "data = {\"question\": questions, \"difficulty\": difficulties}\n", "\n", "dataset = Dataset.from_dict(data)\n", "print(f\"\\nDataset: {len(dataset)} rows\")\n", "print(f\"Sample: {dataset[0]['question']} ({dataset[0]['difficulty']})\")\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "427a2186", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b038f2a6afe84208820c1997e5d15096", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Uploading the dataset shards: 0%| | 0/1 [00:00