{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "8ma4uIoIZjC1"
},
"source": [
"# benchmark: medpy vs. mikan-rs 🍊"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "L5D9-i91eYUY"
},
"source": [
"Please note that **Colab only provides dual-core CPUs**, so the speedup is limited. You can test on a server with more CPUs to observe mikan's blazingly fast performance."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "y_qn133LaML8",
"outputId": "618ac8ca-5e1f-4519-ee9f-343ca1a64ccc"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: SimpleITK in /usr/local/lib/python3.11/dist-packages (2.4.1)\n",
"Requirement already satisfied: medpy in /usr/local/lib/python3.11/dist-packages (0.5.2)\n",
"Requirement already satisfied: mikan-rs in /usr/local/lib/python3.11/dist-packages (0.1.2)\n",
"Requirement already satisfied: scipy>=1.10 in /usr/local/lib/python3.11/dist-packages (from medpy) (1.13.1)\n",
"Requirement already satisfied: numpy>=1.24 in /usr/local/lib/python3.11/dist-packages (from medpy) (1.26.4)\n"
]
}
],
"source": [
"!pip install SimpleITK medpy mikan-rs"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 181
},
"id": "a_RYW5sPX6mY",
"outputId": "2a3fec85-5b1c-4b4b-82d7-d4cb961d7354"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Downloading...\n",
"From: https://drive.google.com/uc?id=1R6rph1_Wc2HfLhzvGkcNt7hgbQiHWZBY\n",
"To: /content/patients_26_ground_truth.nii.gz\n",
"100%|██████████| 1.17M/1.17M [00:00<00:00, 70.3MB/s]\n",
"Downloading...\n",
"From: https://drive.google.com/uc?id=1cShgX96WgK_j4EbfR4wN2RLgcyVR4ca0\n",
"To: /content/patients_26_segmentation.nii.gz\n",
"100%|██████████| 5.05M/5.05M [00:00<00:00, 28.3MB/s]\n"
]
},
{
"data": {
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
},
"text/plain": [
"'patients_26_segmentation.nii.gz'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import gdown\n",
"\n",
"# We use datasets from seg_metrics\n",
"\n",
"gdth_url_id = \"1R6rph1_Wc2HfLhzvGkcNt7hgbQiHWZBY\"\n",
"pred_url_id = \"1cShgX96WgK_j4EbfR4wN2RLgcyVR4ca0\"\n",
"gdth_fpath = \"patients_26_ground_truth.nii.gz\"\n",
"pred_fpath = \"patients_26_segmentation.nii.gz\"\n",
"\n",
"gdown.download(id=gdth_url_id, output=gdth_fpath, quiet=False)\n",
"gdown.download(id=pred_url_id, output=pred_fpath, quiet=False)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "cxVnoqqEYd6F"
},
"outputs": [],
"source": [
"import time\n",
"import mikan\n",
"import numpy as np\n",
"import SimpleITK as sitk\n",
"from medpy.metric import dc, hd, hd95, assd"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Lmzfs9DaZyaD"
},
"source": [
"## Load datas"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "E3lZFPFNYiSK"
},
"outputs": [],
"source": [
"gt = sitk.ReadImage(rf\"patients_26_ground_truth.nii.gz\", sitk.sitkUInt8)\n",
"pred = sitk.ReadImage(rf\"patients_26_segmentation.nii.gz\", sitk.sitkUInt8)\n",
"\n",
"gt_arr = sitk.GetArrayFromImage(gt)\n",
"pred_arr = sitk.GetArrayFromImage(pred)\n",
"\n",
"# Downsample for faster\n",
"# If you're patient, you can comment out here and wait for medpy to run for 30 minutes 😆\n",
"gt_arr = np.array(gt_arr[::2, ::2, ::2])\n",
"pred_arr = np.array(pred_arr[::2, ::2, ::2])"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "m9vGRSY8Z2v9"
},
"source": [
"## Dice"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "nMxlyYl3YpDO",
"outputId": "c87b5310-88ba-4381-8115-13a7187c3af6"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mikan cost 0.03 s.\n",
"medpy costs 0.27 s.\n",
"DSC: 8.05x faster\n"
]
}
],
"source": [
"# mikan: DSC\n",
"t = time.time()\n",
"evaluator = mikan.ArrayEvaluator(gt_arr, pred_arr, spacing=gt.GetSpacing())\n",
"dsc = evaluator.labels([1,2,3,4,5]).metrics(\"dsc\")\n",
"mikan_costs = time.time() - t\n",
"print(f\"Mikan cost {mikan_costs:.2f} s.\")\n",
"\n",
"# medpy: DSC\n",
"t = time.time()\n",
"for i in (1,2,3,4,5):\n",
" dsc = dc(pred_arr == i, gt_arr == i)\n",
"medpy_costs = time.time() - t\n",
"print(f\"medpy costs {time.time() - t:.2f} s.\")\n",
"print(f\"DSC: {medpy_costs / mikan_costs :.2f}x faster\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "YX0b4t2XZ4z8"
},
"source": [
"## HD"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "hrU3grdDdrCn",
"outputId": "bc289aa3-2001-4455-b7b6-2f6242640b7e"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[3.8066796490554484, 5.875891921349948, 7.18959419428273, 55.03783156368531, 46.23897571237574]\n",
"Mikan has calculated Hausdorff distance and cost 0.93 s.\n",
"Let's waiting for medpy, be patient for a while...\n",
"46.23897571237574\n",
"HD: 71.22x faster\n"
]
}
],
"source": [
"# mikan: HD\n",
"t = time.time()\n",
"evaluator = mikan.ArrayEvaluator(gt_arr, pred_arr, spacing=gt.GetSpacing())\n",
"hausdorff_distance = evaluator.labels([1, 2, 3, 4, 5]).metrics(\"HD\")\n",
"print(hausdorff_distance)\n",
"\n",
"mikan_costs = time.time() - t\n",
"print(f\"Mikan has calculated Hausdorff distance and cost {mikan_costs:.2f} s.\")\n",
"print(f\"Let's waiting for medpy, be patient for a while...\")\n",
"\n",
"# medpy: HD\n",
"t = time.time()\n",
"for i in (1, 2, 3, 4, 5):\n",
" hausdorff_distance = hd(pred_arr == i, gt_arr == i, voxelspacing=gt.GetSpacing()[::-1])\n",
"medpy_costs = time.time() - t\n",
"print(hausdorff_distance)\n",
"print(f\"HD: {medpy_costs / mikan_costs :.2f}x faster\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vC0KgPIlZ6LU"
},
"source": [
"## All Distances"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "3weKKOcMZeLC",
"outputId": "4c87051f-9d4c-4e79-a192-ae8a26ccb68a"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'1': {'hd': 3.8066796490554484, 'hd95': 0.7409999966621399, 'assd': 0.2482734556966591}, '2': {'hd': 5.875891921349948, 'hd95': 0.893913303991663, 'assd': 0.30188703831217684}, '3': {'hd': 7.18959419428273, 'hd95': 0.7409999966621399, 'assd': 0.29096373483885757}, '4': {'hd': 55.03783156368531, 'hd95': 0.7409999966621399, 'assd': 0.25367831066384994}, '5': {'hd': 46.23897571237574, 'hd95': 0.7409999966621399, 'assd': 0.27452537658032045}}\n",
"Mikan has calculated distance and cost 0.96 s.\n",
"Let's waiting for medpy, be patient for a while...\n",
"{1: {'hd': 3.8066796490554484, 'hd95': 0.7409999966621399, 'assd': 0.24827345569665876}, 2: {'hd': 5.875891921349948, 'hd95': 0.893913303991663, 'assd': 0.3018870383121832}, 3: {'hd': 7.18959419428273, 'hd95': 0.7409999966621399, 'assd': 0.2909637348388582}, 4: {'hd': 55.03783156368531, 'hd95': 0.7409999966621399, 'assd': 0.25367831066385055}, 5: {'hd': 46.23897571237574, 'hd95': 0.7409999966621399, 'assd': 0.2745253765803193}}\n",
"Distances: 210.35x faster\n"
]
}
],
"source": [
"# mikan: Distances\n",
"t = time.time()\n",
"evaluator = mikan.ArrayEvaluator(gt_arr, pred_arr, spacing=gt.GetSpacing())\n",
"mikan_distances = evaluator.labels([1,2,3,4,5]).metrics([\"hd\", \"hd95\", \"assd\"])\n",
"mikan_costs = time.time() - t\n",
"print(mikan_distances)\n",
"print(f\"Mikan has calculated distance and cost {mikan_costs:.2f} s.\")\n",
"print(f\"Let's waiting for medpy, be patient for a while...\")\n",
"\n",
"# medpy: Distances\n",
"t = time.time()\n",
"medpy_results = {}\n",
"for i in (1,2,3,4,5):\n",
" hd_ = hd(pred_arr == i, gt_arr == i, voxelspacing=gt.GetSpacing()[::-1])\n",
" hd95_ = hd95(pred_arr == i, gt_arr == i, voxelspacing=gt.GetSpacing()[::-1])\n",
" assd_ = assd(pred_arr == i, gt_arr == i, voxelspacing=gt.GetSpacing()[::-1])\n",
" medpy_results[i] = {\n",
" \"hd\": hd_,\n",
" \"hd95\": hd95_,\n",
" \"assd\": assd_,\n",
" }\n",
"medpy_costs = time.time() - t\n",
"print(medpy_results)\n",
"print(f\"Distances: {medpy_costs / mikan_costs :.2f}x faster\")"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}