Duplicate from polejowska/vicellst-att

.gitattributes

@@ -0,0 +1,70 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_036_053248_048128_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_008192_025600_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_009216_022528_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_009216_025600_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_013312_024576_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_013312_025600_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_096_053248_075776_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_046080_050176_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_047104_048128_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_047104_049152_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_048128_050176_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_048128_051200_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_048128_052224_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_050176_048128_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_050176_049152_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_050176_050176_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_090112_066560_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_091136_006144_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_091136_062464_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_092160_008192_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_092160_062464_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_096256_027648_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_097280_022528_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_097280_024576_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_097280_025600_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_097280_026624_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_281_097280_027648_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_059_092160_016384_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_080_044032_053248_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_080_101376_038912_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_101_057344_037888_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_124_019456_009216_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_205_010240_018432_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_228_089088_062464_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_253_054272_057344_HE.png filter=lfs diff=lfs merge=lfs -text
+cd45rb_test_imgs/CD45RB_Leukocyte_253_055296_057344_HE.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Vicellst
+emoji: 🦀
+colorFrom: blue
+colorTo: gray
+sdk: gradio
+sdk_version: 3.18.0
+app_file: app.py
+pinned: false
+duplicated_from: polejowska/vicellst-att
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,146 @@
+import pathlib
+from constants import MODELS_REPO, MODELS_NAMES
+
+import gradio as gr
+import torch
+
+from transformers import AutoFeatureExtractor, DetrForObjectDetection
+from visualization import visualize_attention_map, visualize_prediction
+from style import css, description, title
+
+from PIL import Image
+
+
+
+def make_prediction(img, feature_extractor, model):
+    inputs = feature_extractor(img, return_tensors="pt")
+    outputs = model(**inputs)
+    img_size = torch.tensor([tuple(reversed(img.size))])
+    processed_outputs = feature_extractor.post_process(outputs, img_size)
+    print(outputs.keys())
+    return (
+        processed_outputs[0],
+        outputs["decoder_attentions"],
+        outputs["encoder_attentions"],
+    )
+
+
+def detect_objects(model_name, image_input, threshold, display_mask=False, img_input_mask=None):
+    feature_extractor = AutoFeatureExtractor.from_pretrained(MODELS_REPO[model_name])
+
+    if "DETR" in model_name:
+        model = DetrForObjectDetection.from_pretrained(MODELS_REPO[model_name])
+        model_details = "DETR details"
+
+    (
+        processed_outputs,
+        decoder_attention_map,
+        encoder_attention_map,
+    ) = make_prediction(image_input, feature_extractor, model)
+
+    viz_img = visualize_prediction(
+        pil_img=image_input,
+        output_dict=processed_outputs,
+        threshold=threshold,
+        id2label=model.config.id2label,
+        display_mask=display_mask,
+        mask=img_input_mask
+    )
+    decoder_attention_map_img = visualize_attention_map(
+        image_input, decoder_attention_map
+    )
+    encoder_attention_map_img = visualize_attention_map(
+        image_input, encoder_attention_map
+    )
+
+    return (
+        viz_img,
+        decoder_attention_map_img,
+        encoder_attention_map_img,
+        model_details
+    )
+
+
+def set_example_image(example: list):
+    print(f"Set example image to: {example[0]}")
+    print(f"Set example image mask to: {example[1]}")
+    return gr.Image.update(value=example[0]), gr.Image.update(value=example[1])
+
+
+with gr.Blocks(css=css) as app:
+    gr.Markdown(title)
+
+    with gr.Tabs():
+        with gr.TabItem("Image upload and detections visualization"):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        img_input = gr.Image(type="pil")
+                    img_input_mask = gr.Image(type="pil", visible=False)
+                    with gr.Row():
+                        example_images = gr.Dataset(
+                            components=[img_input, img_input_mask],
+                            samples=[
+                                [path.as_posix(), path.as_posix().replace("_HE", "_mask")]
+                                for path in sorted(
+                                    pathlib.Path("cd45rb_test_imgs").rglob("*_HE.png")
+                                )
+                            ],
+                            samples_per_page=2,
+                        )
+                with gr.Column():
+                    with gr.Row():
+                        options = gr.Dropdown(
+                            value=MODELS_NAMES[0],
+                            choices=MODELS_NAMES,
+                            label="Select an object detection model",
+                            show_label=True,
+                        )
+                    with gr.Row():
+                        slider_input = gr.Slider(
+                            minimum=0.2, maximum=1, value=0.7, label="Prediction threshold"
+                        )
+                    with gr.Row():
+                        display_mask = gr.Checkbox(
+                            label="Display masks", default=False
+                        )
+                    with gr.Row():
+                        detect_button = gr.Button("Detect leukocytes")
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(
+                        """The selected image with detected bounding boxes by the model"""
+                    )
+                    img_output_from_upload = gr.Image(shape=(800, 800))
+        with gr.TabItem("Attentions visualization"):
+            gr.Markdown("""Encoder attentions""")
+            with gr.Row():
+                encoder_att_map_output = gr.Image(shape=(850, 850))
+            gr.Markdown("""Decoder attentions""")
+            with gr.Row():
+                decoder_att_map_output = gr.Image(shape=(850, 850))
+        with gr.TabItem("Model details"):
+            with gr.Row():
+                model_details = gr.Markdown(""" """)
+        with gr.TabItem("Dataset details"):
+            with gr.Row():
+                gr.Markdown(description)
+
+    detect_button.click(
+        detect_objects,
+        inputs=[options, img_input, slider_input, display_mask, img_input_mask],
+        outputs=[
+            img_output_from_upload,
+            decoder_att_map_output,
+            encoder_att_map_output,
+            # cross_att_map_output,
+            model_details,
+        ],
+        queue=True,
+    )
+    example_images.click(
+        fn=set_example_image, inputs=[example_images], outputs=[img_input, img_input_mask],
+        show_progress=True
+    )
+
+    app.launch(enable_queue=True)

constants.py

@@ -0,0 +1,19 @@
+COLORS = [
+    [0.000, 0.447, 0.741],
+    [0.850, 0.325, 0.098],
+    [0.929, 0.694, 0.125],
+    [0.494, 0.184, 0.556],
+    [0.466, 0.674, 0.188],
+    [0.301, 0.745, 0.933],
+    [0.351, 0.760, 0.903],
+]
+
+MODELS_REPO = {
+    "DETR-RESNET-50": "polejowska/detr-resnet-50-CD45RB-1000-att",
+    "DEFORMABLE-DETR": "polejowska/deformable-detr-resnet50-leuk",
+    "CONDITIONAL-DETR": "polejowska/cdetr-cd45rb-s",
+    "DETR-RESNET-101": "polejowska/detr-resnet-101-CD45RB-1000-att",
+    "DETR-RESNET-50-TEST": "polejowska/detr-resnet50-leuk",
+}
+
+MODELS_NAMES = ["DETR-RESNET-50", "DEFORMABLE-DETR", "CONDITIONAL-DETR", "DETR-RESNET-101", "DETR-RESNET-50-TEST"]

requirements.txt

@@ -0,0 +1,8 @@
+beautifulsoup4==4.9.3
+bs4==0.0.1
+requests-file==1.5.1
+torch==1.10.1
+git+https://github.com/huggingface/transformers.git
+validators==0.18.2
+timm==0.5.4
+opencv-python

style.py

@@ -0,0 +1,23 @@
+title = """<h2 id="title">vicellst</h2>"""
+
+description = """
+Citation for the dataset used to train the model and provide examples in this interface:
+> @dataset{komura_daisuke_2022_7412739,
+>  author       = {Komura, Daisuke},
+>  title        = {{Large-scale annotation dataset for cell/tissue 
+>                   segmentation in H\&E-stained images : anti-CD45RB
+>                   (leukocytes)}},
+>  month        = apr,
+>  year         = 2022,
+>  publisher    = {Zenodo},
+>  version      = {0.3},
+>  doi          = {10.5281/zenodo.7412739},
+>  url          = {https://doi.org/10.5281/zenodo.7412739}
+}
+"""
+
+css = """
+h2#title {
+  text-align: left;
+}
+"""

utils.py

@@ -0,0 +1,9 @@
+import io
+from PIL import Image
+
+
+def fig2img(fig):
+    buf = io.BytesIO()
+    fig.savefig(buf)
+    buf.seek(0)
+    return Image.open(buf)

visualization.py

@@ -0,0 +1,108 @@
+from matplotlib import pyplot as plt
+from PIL import Image
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+from constants import COLORS
+from utils import fig2img
+
+
+def visualize_prediction(
+    pil_img, output_dict, threshold=0.7, id2label=None, display_mask=False, mask=None
+):
+    keep = output_dict["scores"] > threshold
+    boxes = output_dict["boxes"][keep].tolist()
+    scores = output_dict["scores"][keep].tolist()
+    labels = output_dict["labels"][keep].tolist()
+    if id2label is not None:
+        labels = [id2label[x] for x in labels]
+
+    fig, ax = plt.subplots(figsize=(12, 12))
+    ax.imshow(pil_img)
+    if display_mask and mask is not None:
+        # Convert the mask image to a numpy array
+        mask_arr = np.asarray(mask)
+        
+        # Create a new mask with white objects and black background
+        new_mask = np.zeros_like(mask_arr)
+        new_mask[mask_arr > 0] = 255
+        
+        # Convert the numpy array back to a PIL Image
+        new_mask = Image.fromarray(new_mask)
+        
+        # Display the new mask as a semi-transparent overlay
+        ax.imshow(new_mask, alpha=0.5, cmap='viridis')
+       
+    colors = COLORS * 100
+    for score, (xmin, ymin, xmax, ymax), label, color in zip(
+        scores, boxes, labels, colors
+    ):
+        ax.add_patch(
+            plt.Rectangle(
+                (xmin, ymin),
+                xmax - xmin,
+                ymax - ymin,
+                fill=False,
+                color=color,
+                linewidth=2,
+            )
+        )
+        ax.text(
+            xmin,
+            ymin,
+            f"{score:0.2f}",
+            fontsize=8,
+            bbox=dict(facecolor="yellow", alpha=0.5),
+        )
+    ax.axis("off")
+    return fig2img(fig)
+
+
+def visualize_attention_map(pil_img, attention_map):
+    # Get the attention map for the last layer
+    attention_map = attention_map[-1].detach().cpu()
+    
+    # Get the number of heads
+    n_heads = attention_map.shape[1]
+    
+    # Calculate the average attention weight for each head
+    avg_attention_weight = torch.mean(attention_map, dim=1).squeeze()
+    
+    # Resize the attention map
+    resized_attention_weight = F.interpolate(
+        avg_attention_weight.unsqueeze(0).unsqueeze(0),
+        size=pil_img.size[::-1],
+        mode="bicubic",
+    ).squeeze().numpy()
+    
+    # Create a grid of subplots
+    fig, axes = plt.subplots(nrows=1, ncols=n_heads, figsize=(n_heads*4, 4))
+    
+    # Loop through the subplots and plot the attention for each head
+    for i, ax in enumerate(axes.flat):
+        ax.imshow(pil_img)
+        ax.imshow(attention_map[0,i,:,:].squeeze(), alpha=0.7, cmap="viridis")
+        ax.set_title(f"Head {i+1}")
+        ax.axis("off")
+    
+    plt.tight_layout()
+    
+    return fig2img(fig)
+    # attention_map = attention_map[-1].detach().cpu()
+    # avg_attention_weight = torch.mean(attention_map, dim=1).squeeze()
+    # avg_attention_weight_resized = (
+    #     F.interpolate(
+    #         avg_attention_weight.unsqueeze(0).unsqueeze(0),
+    #         size=pil_img.size[::-1],
+    #         mode="bicubic",
+    #     )
+    #     .squeeze()
+    #     .numpy()
+    # )
+
+    # plt.imshow(pil_img)
+    # plt.imshow(avg_attention_weight_resized, alpha=0.7, cmap="viridis")
+    # plt.axis("off")
+    # fig = plt.gcf()
+    # return fig2img(fig)