Delete prepare_data.py

prepare_data.py

@@ -1,152 +0,0 @@
-import csv
-import json
-import os
-import sys
-
-# Increase field size limit for large CSV fields
-csv.field_size_limit(sys.maxsize)
-
-def clean_text(text):
-    if not text:
-        return ""
-    return text.strip()
-
-def process_item(system, conversation, output_file):
-    # conversion to text format: 
-    # <system>...
-    # <user>...
-    # <ai>...
-    
-    # Check if we have valid content
-    if not conversation:
-        return
-
-    text_parts = []
-    
-    # Add system if present
-    if system:
-        text_parts.append(f"<system>{clean_text(system)}")
-    
-    # Process conversation turns
-    # conversation is a list of (role, content)
-    # roles: 'user', 'ai' (we map 'assistant'->'ai')
-    
-    for role, content in conversation:
-        content = clean_text(content)
-        if not content:
-            continue
-        
-        if role == 'system':
-            # Handle system in message list if somehow present/overriding
-            text_parts.append(f"<system>{content}")
-        elif role == 'user':
-            text_parts.append(f"<user>{content}")
-        elif role == 'assistant' or role == 'ai':
-            text_parts.append(f"<ai>{content}")
-        else:
-            # Fallback for unknown roles
-            text_parts.append(f"<{role}>{content}")
-            
-    if text_parts:
-        final_str = "\n".join(text_parts)
-        output_file.write(final_str + "\n\n")
-
-def process_csv(filepath, output_path):
-    print(f"Processing CSV: {filepath}")
-    try:
-        with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
-            reader = csv.DictReader(f)
-            with open(output_path, 'a', encoding='utf-8') as out:
-                count = 0
-                for row in reader:
-                    # Mapping logic for this specific CSV structure:
-                    # thread_title -> User context/prompt
-                    # instruction -> System prompt
-                    # message -> AI response
-                    
-                    sys_prompt = row.get('instruction', '')
-                    title = row.get('thread_title', '')
-                    msg = row.get('message', '')
-                    
-                    # If we have mainly 'text' and it looks like it contains everything, we might prefer it?
-                    # But analysis suggested 'text' was just instruction/duplicate in some rows.
-                    # We'll stick to constructing from parts which is safer for structured training.
-                    
-                    conversation = []
-                    if title:
-                        conversation.append(('user', title))
-                    if msg:
-                        conversation.append(('ai', msg))
-                    
-                    if conversation:
-                        process_item(sys_prompt, conversation, out)
-                        count += 1
-                        if count % 10000 == 0:
-                            print(f"CSV Processed {count}...", flush=True)
-            print(f"Finished CSV. Processed {count} rows.")
-    except Exception as e:
-        print(f"Error processing CSV: {e}")
-
-def process_jsonl(filepath, output_path):
-    print(f"Processing JSONL: {filepath}")
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f, open(output_path, 'a', encoding='utf-8') as out:
-            count = 0
-            for line in f:
-                if not line.strip(): continue
-                try:
-                    data = json.loads(line)
-                    messages = data.get('messages', [])
-                    
-                    # Extract system prompt if it exists as a separate field or role
-                    system_prompt = data.get('system', '') 
-                    
-                    conversation = []
-                    
-                    for m in messages:
-                        role = m.get('role', '')
-                        content = m.get('content', '')
-                        
-                        if role == 'system':
-                            # If we hit a system role, treat it as global system or part of flow
-                            # User asked to "add system as <system>"
-                            # I'll just map it directly.
-                            conversation.append(('system', content))
-                        else:
-                            conversation.append((role, content))
-                            
-                    if conversation:
-                        # Pass None for separate system arg since we handle it in loop
-                        process_item(None, conversation, out)
-                        count += 1
-                        
-                    if count % 10000 == 0:
-                        print(f"JSONL Processed {count}...", flush=True)
-                        
-                except json.JSONDecodeError:
-                    continue
-            print(f"Finished JSONL. Processed {count} items.")
-    except Exception as e:
-        print(f"Error processing JSONL: {e}")
-
-def main():
-    data_dir = "data"
-    output_filename = "processed_corpus.txt"
-    output_path = os.path.join(data_dir, output_filename)
-    
-    # Overwrite/Create new
-    with open(output_path, 'w', encoding='utf-8') as f:
-        f.write("")
-        
-    # Process JSONL
-    jsonl_path = os.path.join(data_dir, "dataset.jsonl")
-    if os.path.exists(jsonl_path):
-        process_jsonl(jsonl_path, output_path)
-    
-    # Process CSV
-    csv_path = os.path.join(data_dir, "train.csv")
-    if os.path.exists(csv_path):
-        process_csv(csv_path, output_path)
-
-if __name__ == "__main__":
-    main()