""" X-ray Bone Tumor Annotator ========================== A Gradio app that, given: 1) an X-ray image (possibly tilted, sitting on a white/grey background), and 2) a LabelMe-style JSON file containing a bounding-box (rectangle) shape and a polygon (mask) shape for a bone tumor, will: - Detect the actual X-ray film region and remove the surrounding white/grey background. - Straighten (deskew) the tilted film so it is axis-aligned. - Resize the result to 640x640. - Draw the tumor mask (polygon) in a user-chosen color, draw a bounding box around it, and place a text label (typed by the user) on top of the box. Run: pip install -r requirements.txt python app.py """ import json import os import tempfile import traceback import cv2 import numpy as np import gradio as gr # Hugging Face Spaces ZeroGPU compatibility. # If a Space's hardware is set to "ZeroGPU", HF requires at least one # function decorated with @spaces.GPU to be registered at startup, or it # fails with: "No @spaces.GPU function detected during startup". # This app does no GPU work at all (pure OpenCV/CPU), so the decorator # below is a no-op wrapper everywhere except on a ZeroGPU Space, where it # satisfies that startup requirement. If you don't need ZeroGPU hardware, # the simpler fix is to switch the Space's hardware to "CPU basic" instead. try: import spaces HAS_SPACES = True except ImportError: spaces = None HAS_SPACES = False def gpu_compatible(fn): """Wrap fn with @spaces.GPU when running on a HF ZeroGPU Space; otherwise return fn unchanged so local/non-Spaces runs are unaffected.""" if HAS_SPACES: return spaces.GPU(fn) return fn # -------------------------------------------------------------------------- # Config # -------------------------------------------------------------------------- OUTPUT_SIZE = 640 # final width/height of the output image # 9 preset colors (BGR order, since we draw with OpenCV) shown to the user by name. COLOR_MAP = { "Red": (0, 0, 255), "Green": (0, 200, 0), "Blue": (255, 0, 0), "Yellow": (0, 220, 255), "Cyan": (255, 255, 0), "Magenta": (255, 0, 255), "Orange": (0, 140, 255), "Purple": (170, 0, 130), "White": (255, 255, 255), } MASK_ALPHA = 0.40 # transparency of the filled polygon mask BOX_THICKNESS = 3 # bounding-box line thickness MASK_OUTLINE_THICKNESS = 2 # polygon outline thickness FONT = cv2.FONT_HERSHEY_SIMPLEX FONT_SCALE = 0.7 FONT_THICKNESS = 2 LABEL_PADDING = 6 # -------------------------------------------------------------------------- # Step 1: Parse the LabelMe-style JSON # -------------------------------------------------------------------------- def parse_labelme_json(json_path): """Return (rectangles, polygons) as lists of numpy arrays of (x, y) points, read from a LabelMe-style JSON file.""" with open(json_path, "r") as f: data = json.load(f) rectangles = [] polygons = [] for shape in data.get("shapes", []): pts = np.array(shape["points"], dtype=np.float32) shape_type = shape.get("shape_type", "polygon") if shape_type == "rectangle": rectangles.append(pts) else: polygons.append(pts) return rectangles, polygons # -------------------------------------------------------------------------- # Step 2: Detect the tilted film region, straighten it, and crop away the # surrounding white/grey background. # -------------------------------------------------------------------------- def order_points(pts): """Order 4 points as top-left, top-right, bottom-right, bottom-left.""" rect = np.zeros((4, 2), dtype=np.float32) s = pts.sum(axis=1) rect[0] = pts[np.argmin(s)] # top-left -> smallest x+y rect[2] = pts[np.argmax(s)] # bottom-right -> largest x+y diff = np.diff(pts, axis=1).reshape(-1) rect[1] = pts[np.argmin(diff)] # top-right -> smallest x-y rect[3] = pts[np.argmax(diff)] # bottom-left -> largest x-y return rect def find_film_corners(gray): """ Try to find the 4 corners of the tilted X-ray film against its white/grey background. Returns a (4,2) float32 array of corners, or None if nothing reasonable was found (caller should then skip deskew/crop and just use the raw image). """ h, w = gray.shape[:2] total_area = float(h * w) blurred = cv2.GaussianBlur(gray, (7, 7), 0) # Figure out which side of the intensity histogram is "background". # The background (white/grey paper/table the film sits on) is what # touches the image border, since the film itself is fully inside the # frame. Sampling a thin strip around the border gives a reliable # estimate of the background brightness, regardless of whether the # film happens to be mostly dark or mostly bright overall. border = int(max(h, w) * 0.02) or 1 border_pixels = np.concatenate([ blurred[:border, :].ravel(), blurred[-border:, :].ravel(), blurred[:, :border].ravel(), blurred[:, -border:].ravel(), ]) background_is_bright = np.median(border_pixels) > 127 # If the background is bright, the film (darker overall, because of # its exposed black regions) should be isolated with an inverted # threshold (dark pixels -> foreground). If the background is dark, # do the opposite. invert = background_is_bright flag = cv2.THRESH_BINARY_INV if invert else cv2.THRESH_BINARY _, th = cv2.threshold(blurred, 0, 255, flag + cv2.THRESH_OTSU) k = max(15, (max(h, w) // 60) | 1) # odd kernel size, scales with image kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k)) closed = cv2.morphologyEx(th, cv2.MORPH_CLOSE, kernel, iterations=2) contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: return None contour = max(contours, key=cv2.contourArea) area = cv2.contourArea(contour) ratio = area / total_area # A believable film region: clearly present, but not literally the # entire frame (which would mean detection failed / there is no # distinguishable background to remove). if not (0.05 < ratio < 0.97): return None rect = cv2.minAreaRect(contour) box = cv2.boxPoints(rect) # 4 (x, y) points # Sanity check: the detected region should actually be noticeably # darker/lighter (whichever we expected) than the border background; # otherwise this is likely a spurious detection and we bail out. mask = np.zeros_like(gray) cv2.drawContours(mask, [contour], -1, 255, thickness=-1) inside_mean = gray[mask == 255].mean() border_mean = border_pixels.mean() if background_is_bright and inside_mean >= border_mean: return None if not background_is_bright and inside_mean <= border_mean: return None return order_points(box.astype(np.float32)) def deskew_and_crop(image_bgr, all_points): """ Detect the film region, straighten it, and crop away the background. Parameters ---------- image_bgr : np.ndarray, original image (H, W, 3) all_points : list of np.ndarray, all annotation point sets that must be transformed the same way as the image. Returns ------- warped_image : np.ndarray transformed_points : list of np.ndarray (same structure as all_points) used_deskew : bool (False if we fell back to the original image untouched) """ gray = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY) corners = find_film_corners(gray) if corners is None: # Fall back: no background/tilt correction possible, pass image through. return image_bgr.copy(), [pts.copy() for pts in all_points], False (tl, tr, br, bl) = corners width_top = np.linalg.norm(tr - tl) width_bottom = np.linalg.norm(br - bl) height_left = np.linalg.norm(bl - tl) height_right = np.linalg.norm(br - tr) out_w = int(max(width_top, width_bottom)) out_h = int(max(height_left, height_right)) out_w = max(out_w, 2) out_h = max(out_h, 2) dst = np.array( [[0, 0], [out_w - 1, 0], [out_w - 1, out_h - 1], [0, out_h - 1]], dtype=np.float32, ) M = cv2.getPerspectiveTransform(corners, dst) warped = cv2.warpPerspective(image_bgr, M, (out_w, out_h)) transformed_points = [] for pts in all_points: pts_reshaped = pts.reshape(-1, 1, 2).astype(np.float32) new_pts = cv2.perspectiveTransform(pts_reshaped, M).reshape(-1, 2) transformed_points.append(new_pts) return warped, transformed_points, True # -------------------------------------------------------------------------- # Step 3: Resize to 640x640 (points scaled to match) # -------------------------------------------------------------------------- def resize_to_square(image_bgr, all_points, size=OUTPUT_SIZE): h, w = image_bgr.shape[:2] resized = cv2.resize(image_bgr, (size, size), interpolation=cv2.INTER_AREA) sx = size / float(w) sy = size / float(h) scaled_points = [] for pts in all_points: scaled = pts.copy() scaled[:, 0] *= sx scaled[:, 1] *= sy scaled_points.append(scaled) return resized, scaled_points # -------------------------------------------------------------------------- # Step 4: Draw the mask, bounding box, and label # -------------------------------------------------------------------------- def _text_color_for(bg_color_bgr): """Pick black or white text for best contrast against a given BGR color.""" b, g, r = bg_color_bgr luminance = 0.114 * b + 0.587 * g + 0.299 * r return (0, 0, 0) if luminance > 150 else (255, 255, 255) def draw_annotations(image_bgr, rectangles, polygons, color_bgr, label_text): output = image_bgr.copy() overlay = image_bgr.copy() # --- filled + outlined polygon mask(s) --- for poly in polygons: poly_i = poly.astype(np.int32) cv2.fillPoly(overlay, [poly_i], color_bgr) output = cv2.addWeighted(overlay, MASK_ALPHA, output, 1 - MASK_ALPHA, 0) for poly in polygons: poly_i = poly.astype(np.int32) cv2.polylines(output, [poly_i], isClosed=True, color=color_bgr, thickness=MASK_OUTLINE_THICKNESS, lineType=cv2.LINE_AA) # --- bounding box(es) + label --- for rect_pts in rectangles: xs = rect_pts[:, 0] ys = rect_pts[:, 1] x1, y1 = int(round(xs.min())), int(round(ys.min())) x2, y2 = int(round(xs.max())), int(round(ys.max())) cv2.rectangle(output, (x1, y1), (x2, y2), color_bgr, BOX_THICKNESS, cv2.LINE_AA) if label_text: text = label_text.strip().upper() (text_w, text_h), baseline = cv2.getTextSize(text, FONT, FONT_SCALE, FONT_THICKNESS) label_x1 = x1 label_y2 = y1 # bottom of label sits at top of the box label_y1 = label_y2 - text_h - 2 * LABEL_PADDING label_x2 = label_x1 + text_w + 2 * LABEL_PADDING # Keep the label on-canvas if the box is near the top edge. if label_y1 < 0: label_y1 = y2 label_y2 = label_y1 + text_h + 2 * LABEL_PADDING h, w = output.shape[:2] label_x2 = min(label_x2, w - 1) label_y2 = min(label_y2, h - 1) cv2.rectangle(output, (label_x1, label_y1), (label_x2, label_y2), color_bgr, thickness=-1) text_color = _text_color_for(color_bgr) text_org = (label_x1 + LABEL_PADDING, label_y2 - LABEL_PADDING - baseline // 2) cv2.putText(output, text, text_org, FONT, FONT_SCALE, text_color, FONT_THICKNESS, cv2.LINE_AA) return output # -------------------------------------------------------------------------- # Full pipeline # -------------------------------------------------------------------------- @gpu_compatible def process(image_path, json_path, color_name, label_text): if image_path is None: raise gr.Error("Please upload an X-ray image.") if json_path is None: raise gr.Error("Please upload the corresponding LabelMe JSON file.") if not label_text or not label_text.strip(): raise gr.Error("Please type the name of the bone tumor to use as the label.") image_bgr = cv2.imread(image_path, cv2.IMREAD_COLOR) if image_bgr is None: raise gr.Error("Could not read the uploaded image file.") try: rectangles, polygons = parse_labelme_json(json_path) except Exception: raise gr.Error(f"Could not parse the JSON file:\n{traceback.format_exc()}") if not rectangles and not polygons: raise gr.Error("No 'rectangle' or 'polygon' shapes were found in the JSON file.") # Keep track of how many points belong to each shape so we can split the # transformed point arrays back into rectangles/polygons afterwards. all_points = rectangles + polygons n_rect = len(rectangles) warped_img, warped_points, used_deskew = deskew_and_crop(image_bgr, all_points) final_img, final_points = resize_to_square(warped_img, warped_points, OUTPUT_SIZE) final_rectangles = final_points[:n_rect] final_polygons = final_points[n_rect:] color_bgr = COLOR_MAP.get(color_name, (0, 140, 255)) annotated_bgr = draw_annotations(final_img, final_rectangles, final_polygons, color_bgr, label_text) annotated_rgb = cv2.cvtColor(annotated_bgr, cv2.COLOR_BGR2RGB) # Save to a temp file so the user can download it. out_dir = tempfile.mkdtemp(prefix="xray_annot_") out_path = os.path.join(out_dir, "annotated_640x640.png") cv2.imwrite(out_path, annotated_bgr) status = ( "Background removed & image straightened successfully." if used_deskew else "Note: could not confidently detect a tilted film boundary against the " "background, so the image was used as-is (only resized/annotated)." ) return annotated_rgb, out_path, status # -------------------------------------------------------------------------- # Gradio UI # -------------------------------------------------------------------------- with gr.Blocks(title="X-ray Bone Tumor Annotator") as demo: gr.Markdown( """ # 🦴 X-ray Bone Tumor Annotator Upload an X-ray image and its matching LabelMe JSON (containing a `rectangle` and a `polygon` shape). The app will remove the white/grey background, straighten a tilted film, resize to **640x640**, and draw the tumor mask + bounding box + label in the color you choose. """ ) with gr.Row(): with gr.Column(): image_input = gr.Image(label="X-ray Image", type="filepath") json_input = gr.File(label="LabelMe JSON File", file_types=[".json"], type="filepath") color_input = gr.Radio( choices=list(COLOR_MAP.keys()), value="Orange", label="Mask / Box Color", ) label_input = gr.Textbox( label="Bone Tumor Name (used as the label on the box)", placeholder="e.g. Giant Cell Tumor", ) run_button = gr.Button("Process", variant="primary") with gr.Column(): output_image = gr.Image(label="Annotated Result (640x640)") output_file = gr.File(label="Download Annotated Image") status_box = gr.Textbox(label="Status", interactive=False) run_button.click( fn=process, inputs=[image_input, json_input, color_input, label_input], outputs=[output_image, output_file, status_box], ) if __name__ == "__main__": demo.launch(share=True)