import ast import csv import json import logging import time from pathlib import Path from typing import Dict, Tuple import gradio as gr import numpy as np from fetch_url_util import fetch_image_from_url from huggingface_hub import hf_hub_download from PIL import Image # Configure Logging logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(message)s") logger = logging.getLogger("PixAITagger") logging.getLogger().setLevel(logging.WARNING) logging.getLogger("PixAITagger").setLevel(logging.INFO) # Constants MODEL_REPO = "deepghs/pixai-tagger-v0.9-onnx" MODEL_FILENAME = "model.onnx" TAGS_FILENAME = "selected_tags.csv" INPUT_SIZE = 448 class HybridEngine: """ Handles inference with dynamic backend selection. Priority: OpenVINO INT8 -> OpenVINO FP32 -> ONNX Runtime """ def __init__(self, model_path: str, use_int8: bool = True): self.model_path = model_path self.session = None self.use_openvino = False self.provider_name = None self.use_int8 = use_int8 self._init_backend() def _init_backend(self): # --- Attempt 1 & 2: OpenVINO (INT8 or FP32) --- try: import openvino as ov logger.info("Engine: OpenVINO available, reading model...") core = ov.Core() model = core.read_model(self.model_path) # Logic for INT8 Quantization if self.use_int8: try: import nncf logger.info("Engine: Compressing weights to INT8 using NNCF...") model = nncf.compress_weights(model) self.provider_name = "OpenVINO (INT8 Weights)" except ImportError: logger.warning("Engine: NNCF not installed. Falling back to FP32.") self.provider_name = "OpenVINO (FP32 - NNCF missing)" except Exception as e: logger.warning(f"Engine: NNCF Compression failed ({e}). Falling back to FP32.") self.provider_name = "OpenVINO (FP32 - Compression error)" else: self.provider_name = "OpenVINO (FP32)" # Compile self.session = core.compile_model(model, "CPU") self.use_openvino = True logger.info(f"Engine: Success using {self.provider_name}") return except Exception as e: logger.warning(f"Engine: OpenVINO initialization failed ({e}). Falling back to ONNX Runtime.") # --- Attempt 3: ONNX Runtime (Fallback) --- try: import onnxruntime as ort sess_options = ort.SessionOptions() sess_options.log_severity_level = 3 self.session = ort.InferenceSession( self.model_path, sess_options=sess_options, providers=["CPUExecutionProvider"], ) self.use_openvino = False self.provider_name = f"ONNX Runtime ({self.session.get_providers()[0]})" logger.info("Engine: Using ONNX Runtime backend") except Exception as e: logger.error(f"Engine: FATAL - ONNX Runtime also failed: {e}") self.provider_name = "Error: No backend available" self.session = None def run(self, input_data: np.ndarray, expected_dim: int): if not self.session: raise RuntimeError("Engine not initialized") if self.use_openvino: results = self.session(input_data) outputs = list(results.values()) else: input_name = self.session.get_inputs()[0].name outputs = self.session.run(None, {input_name: input_data}) # Pick output with expected class dimension for out in outputs: if out.shape[1] == expected_dim: return out[0], self.provider_name # Fallback: largest output out = max(outputs, key=lambda x: x.shape[1]) return out[0], self.provider_name class PixAITagger: def __init__(self): self.model_path = None self.tags_list = [] self._load_resources() # State tracking for engine reloading self.engine = None self.current_int8_mode = None def _load_resources(self): logger.info(f"Downloading resources from {MODEL_REPO}...") self.model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILENAME) try: tags_path = hf_hub_download(repo_id=MODEL_REPO, filename=TAGS_FILENAME) self._load_tags_csv(Path(tags_path)) except Exception as e: raise FileNotFoundError(f"Could not load tags file: {e}") def _load_tags_csv(self, csv_path: Path): self.tags_list = [] with csv_path.open("r", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: try: idx = int(row.get("id")) name = row.get("name") category = int(row.get("category", 0)) ips_raw = row.get("ips", "[]") ips = [] if ips_raw and ips_raw != "[]": try: ips = json.loads(ips_raw) except Exception: try: ips = ast.literal_eval(ips_raw) except Exception: pass self.tags_list.append( {"id": idx, "name": name, "is_char": category == 4, "ips": [str(ip) for ip in ips]} ) except ValueError: continue self.tags_list.sort(key=lambda x: x["id"]) self.id_to_tag = {} self.char_indices = [] self.gen_indices = [] self.mapping = {} for item in self.tags_list: idx = item["id"] name = item["name"] self.id_to_tag[idx] = name if item["is_char"]: self.char_indices.append(idx) if item["ips"]: self.mapping[name] = item["ips"] else: self.gen_indices.append(idx) self.num_classes = len(self.tags_list) logger.info(f"Loaded {self.num_classes} tags.") def preprocess(self, image: Image.Image) -> np.ndarray: if image.mode != "RGB": image = image.convert("RGB") image = image.resize((INPUT_SIZE, INPUT_SIZE), Image.BICUBIC) img = np.array(image).astype(np.float32) / 255.0 img = (img - 0.5) / 0.5 img = img.transpose(2, 0, 1) return np.expand_dims(img, 0) def predict( self, image: Image.Image, gen_threshold: float, char_threshold: float, resolve_mapping: bool, use_int8_weights: bool ) -> Tuple[Dict, Dict, str, str, float]: # Reload engine if the INT8 preference changed or engine doesn't exist if self.engine is None or self.current_int8_mode != use_int8_weights: logger.info(f"Reloading engine. New mode INT8: {use_int8_weights}") self.engine = HybridEngine(str(self.model_path), use_int8=use_int8_weights) self.current_int8_mode = use_int8_weights input_tensor = self.preprocess(image) infer_start = time.time() logits, provider_name = self.engine.run(input_tensor, self.num_classes) infer_time = time.time() - infer_start # Sigmoid probs = 1 / (1 + np.exp(-logits)) # General Tags gen_tags = {} for idx in self.gen_indices: if idx < len(probs): score = float(probs[idx]) if score >= gen_threshold: gen_tags[self.id_to_tag[idx]] = score # Character Tags & IPs char_tags = {} detected_ips = set() for idx in self.char_indices: if idx < len(probs): score = float(probs[idx]) if score >= char_threshold: name = self.id_to_tag[idx] char_tags[name] = score if resolve_mapping and name in self.mapping: for ip in self.mapping[name]: detected_ips.add(ip) gen_tags = dict(sorted(gen_tags.items(), key=lambda x: x[1], reverse=True)) char_tags = dict(sorted(char_tags.items(), key=lambda x: x[1], reverse=True)) ip_text = ", ".join(sorted(list(detected_ips))) if detected_ips else "" return gen_tags, char_tags, ip_text, provider_name, infer_time # --- UI Setup --- tagger_instance = None def get_tagger(): global tagger_instance if tagger_instance is None: tagger_instance = PixAITagger() return tagger_instance def init_app(): """Warms up the model loader.""" get_tagger() return None def run_inference(image, gen_thresh, char_thresh, resolve_mapping, use_int8): if image is None: return "", "", "", {}, {}, "" try: model = get_tagger() start_time = time.time() gen_tags, char_tags, ip_str, provider, infer_time = model.predict( image, gen_thresh, char_thresh, resolve_mapping, use_int8 ) char_str = ", ".join(char_tags.keys()).replace("_", " ") gen_str = ", ".join(gen_tags.keys()).replace("_", " ") if not resolve_mapping: ip_disp = "" elif not ip_str: ip_disp = "" else: ip_disp = ip_str.replace("_", " ") time_disp = f"- **Provider:** {provider} | **Inference time:** {infer_time:.4f}s" return char_str, ip_disp, gen_str, char_tags, gen_tags, time_disp except Exception as e: logger.error(f"Inference Error: {e}") return "", f"Error: {str(e)}", "", {}, {}, f"Error: {str(e)}" with gr.Blocks(title="PixAI Tagger v0.9 ONNX") as demo: gr.Markdown( 'PixAI Tagger' ' is an iteration on top of ' 'SmilingWolf/wd-eva02-large-tagger-v3' ' with an updated dataset (2025-01). \n' 'It should be noted that PixAI Tagger may be worse in accuracy over eva02 large. See the PixAI page for details' ) # Header Row with gr.Row(elem_classes=["container"]): with gr.Column(scale=1, elem_classes=["header-col"]): gr.Markdown("### Input") with gr.Column(scale=1, elem_classes=["header-col"]): gr.Markdown("### Configuration & Results") with gr.Row(elem_classes=["container"]): # LEFT COLUMN with gr.Column(scale=1): url_input = gr.Textbox( label="Enter Image URL (not all may work) or upload an image below", placeholder="https://example.com/image.jpg", ) input_img = gr.Image(type="pil", label="", show_label=False, elem_classes=["image-container"]) # RIGHT COLUMN - Controls and Outputs with gr.Column(scale=1): # Action Buttons with gr.Row(): run_btn = gr.Button("Run (Image Upload)", variant="primary") url_btn = gr.Button("Run (URL)", variant="secondary") # Output Textboxes with gr.Group(): with gr.Row(elem_id="d-row-container"): char_box = gr.Textbox(label="Character Tags", interactive=False, buttons=["copy"]) ip_box = gr.Textbox(label="Character - Copyright Mapping", interactive=False, buttons=["copy"]) gen_box = gr.Textbox(label="General Tags", interactive=False, buttons=["copy"]) time_info = gr.Markdown(elem_id="time-display") # Configuration Section with gr.Group(): with gr.Row(elem_id="d-row-container"): char_slider = gr.Slider(0.0, 1.0, value=0.75, step=0.05, label="Character Threshold") gen_slider = gr.Slider(0.0, 1.0, value=0.30, step=0.05, label="General Threshold") # Checkboxes Row with gr.Row(): map_checkbox = gr.Checkbox(value=True, label="Resolve Copyright Mapping") int8_checkbox = gr.Checkbox(value=True, label="INT8 Weights") # Confidence Plots gr.Markdown("### Confidence Scores") char_plot = gr.Label(label="Character Probabilities", num_top_classes=50) gen_plot = gr.Label(label="General Probabilities", num_top_classes=500) gr.Markdown( 'Model sourced from ' 'deepghs/pixai-tagger-v0.9-onnx. \n' 'OpenVINO™ will be used to accelerate CPU inference with ONNX CPUExecutionProvider as fallback. \n' 'INT8 weights option may improve inference times with a deviation of roughly +-0.005 in scores.' ) # Click Logic # Added int8_checkbox to inputs url_btn.click(fn=fetch_image_from_url, inputs=[url_input], outputs=[input_img]).then( fn=run_inference, inputs=[input_img, gen_slider, char_slider, map_checkbox, int8_checkbox], outputs=[char_box, ip_box, gen_box, char_plot, gen_plot, time_info], ) run_btn.click( fn=run_inference, inputs=[input_img, gen_slider, char_slider, map_checkbox, int8_checkbox], outputs=[char_box, ip_box, gen_box, char_plot, gen_plot, time_info], ) # Load logic demo.load(fn=init_app) if __name__ == "__main__": demo.launch( theme=gr.themes.Base(), css=""" * { box-sizing: border-box; } @media (max-width: 1022px) { #d-row-container { flex-direction: column !important; } #d-row-container > * { width: 100% !important; } #d-row-container .block { width: 100% !important; } }""", )