{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "5caf83ef2d199c94",
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime\n",
"import os\n",
"import re\n",
"\n",
"FIGSIZE = (6, 3)\n",
"DPI = 300\n",
"\n",
"def new_fig():\n",
" plt.figure(figsize=FIGSIZE, dpi=DPI)\n",
" sns.set(style=\"darkgrid\")\n",
" sns.set_context(\"paper\")\n",
"\n",
"# Get the current directory\n",
"# current_directory = '/home/andrew/Downloads/Predictions3_final_all2'\n",
"current_directory = '/media/andrew/ExternalM2/VIRAT_gray_0.5_3/Predictions_prior'\n",
"\n",
"# Regular expression to extract the [name] portion\n",
"pattern = r'(\\d+)_(.*?)_(.*?)_(.*?)_(\\d{2}_\\d{2}_\\d{4}_\\d{2}_\\d{2}_\\d{2})\\.log'\n",
"\n",
"# Create a dictionary to store filenames by [name]\n",
"name_to_files = {}\n",
"\n",
"# List all files with a .log extension in the current directory\n",
"log_files = [file for file in os.listdir(current_directory) if file.endswith(\".log\")]\n",
"\n",
"\n",
"# Function to parse the timestamp string into a datetime object\n",
"def parse_timestamp(timestamp_str):\n",
" return datetime.strptime(timestamp_str, '%d_%m_%Y_%H_%M_%S')\n",
"\n",
"\n",
"# Iterate through log files and group them by [name]\n",
"for log_file in log_files:\n",
" match = re.search(pattern, log_file)\n",
" if match:\n",
" name = match.group(4)\n",
" timestamp_str = match.group(5)\n",
" timestamp = parse_timestamp(timestamp_str)\n",
"\n",
" if name not in name_to_files:\n",
" name_to_files[name] = []\n",
" name_to_files[name].append((timestamp, log_file))\n",
"\n",
"# Print the grouped filenames\n",
"for name, files in name_to_files.items():\n",
" print(f\"[name]: {name}\")\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" for _timestamp, file in sorted_files:\n",
" print(file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5df199c0db582925",
"metadata": {},
"outputs": [],
"source": [
"from enum import Enum\n",
"import numpy as np\n",
"\n",
"\n",
"class InputSet:\n",
" def __init__(self, height, width, channels):\n",
" self.event_bitrate = None\n",
" self.compressed_bitrate = None\n",
" self.adder_feature_count = None\n",
" self.adder_time = None\n",
" self.cv_feature_count = None\n",
" self.cv_time = None\n",
" self.psnr = None\n",
" self.mse = None\n",
" self.ssim = None\n",
" self.recon_psnr = None\n",
" self.recon_mse = None\n",
" self.recon_ssim = None\n",
" self.precision = None\n",
" self.recall = None\n",
" self.accuracy = None\n",
" self.num_pixels = height*width*channels\n",
" self.num_channels = channels\n",
" self.tps = None\n",
" self.dt_ref = None\n",
"\n",
"\n",
"def process_entry(entry, input_sets):\n",
" if 's' in entry:\n",
" if \"OpenCV\" in entry['s']:\n",
" input_sets[-1].cv_features[entry['y'], entry['x']] = True\n",
" elif \"ADDER\" in entry['s']:\n",
" input_sets[-1].adder_features[entry['y'], entry['x']] = True\n",
" else:\n",
" raise Exception(\"Malformed entry\")\n",
" elif 'psnr' in entry:\n",
" input_sets[-1].psnr = min(entry['psnr'], 60.0)\n",
" input_sets[-1].mse = entry['mse']\n",
" input_sets[-1].ssim = entry['ssim']\n",
" \n",
"def process_recon_entry(entry, input_sets, idx):\n",
" if 'psnr' in entry:\n",
" input_sets[idx].recon_psnr = min(entry['psnr'], 60.0)\n",
" input_sets[idx].recon_mse = entry['mse']\n",
" input_sets[idx].recon_ssim = entry['ssim']\n",
"\n",
"\n",
"import pickle\n",
"\n",
"Stage = Enum('Stage', ['RAW', 'RECON', 'COMPRESS_SIZE'])\n",
"\n",
"def read_logfile(path):\n",
" objects = []\n",
" with (\n",
" open(\n",
" path,\n",
" \"rb\")) as openfile:\n",
" # Read the dimensions\n",
" dims = openfile.readline().decode('UTF-8')\n",
" # print(dims)\n",
" width = int(dims.split('x')[0])\n",
" height = int(dims.split('x')[1])\n",
" channels = int(dims.split('x')[2])\n",
"\n",
" dbg_lines = 0\n",
" input_interval_idx = 0\n",
" input_sets = [InputSet(height, width, channels)]\n",
" recon = False\n",
" input_set_index = 0\n",
" stage = Stage.RAW\n",
" adu_interval = None\n",
"\n",
" while True:\n",
" try:\n",
" entry = pickle.load(openfile)\n",
" if type(entry) == str and \"ADDER FAST:\" in entry:\n",
" # The start of a new interval\n",
" time = entry.split(': ')[1]\n",
" input_sets[-1].adder_time = int(time)\n",
" continue\n",
" elif type(entry) == str and \"bps:\" in entry:\n",
" # The bitrate\n",
" # The end of the interval\n",
" bitrate = float(entry.split(': ')[1]) / 1024.0 / 1024.0\n",
" input_sets[-1].event_bitrate = bitrate\n",
"\n",
" input_sets += [InputSet(height, width, channels)]\n",
" input_interval_idx += 1\n",
" continue\n",
" elif type(entry) == str and \"RECONSTRUCTION\" in entry:\n",
" stage = Stage.RECON\n",
" input_set_index = 0\n",
" elif type(entry) == str and \"Compressed adu\" in entry:\n",
" stage = Stage.COMPRESS_SIZE\n",
" input_set_index = 0\n",
" elif type(entry) == str and \"OpenCV FAST:\" in entry:\n",
" time = entry.split(': ')[1]\n",
" input_sets[-1].cv_time = int(time)\n",
"\n",
" elif type(entry) == str and \"META:\" in entry:\n",
" print(entry)\n",
" continue\n",
" elif type(entry) == str and \"Feature results:\" in entry:\n",
" precision = pickle.load(openfile)\n",
" input_sets[-1].precision = precision\n",
" recall = pickle.load(openfile)\n",
" input_sets[-1].recall = recall\n",
" accuracy = pickle.load(openfile)\n",
" input_sets[-1].accuracy = accuracy\n",
" \n",
" elif type(entry) == int:\n",
" if stage == stage.RAW:\n",
" if input_sets[-1].adder_feature_count is None:\n",
" input_sets[-1].adder_feature_count = entry\n",
" else:\n",
" input_sets[-1].cv_feature_count = entry\n",
" elif stage == stage.COMPRESS_SIZE:\n",
" for i in range(input_set_index, input_set_index + adu_interval):\n",
" input_sets[i].compressed_bitrate = (entry * 8) / 1024 /1024 # Mb/s\n",
" input_set_index += adu_interval\n",
" else:\n",
" raise Exception(\"unexpected int\")\n",
" \n",
" else:\n",
" if stage == Stage.RAW:\n",
" process_entry(entry, input_sets)\n",
" elif stage == Stage.RECON:\n",
" process_recon_entry(entry, input_sets, input_set_index)\n",
" input_set_index += 1\n",
" elif stage == Stage.COMPRESS_SIZE:\n",
" print('here')\n",
" break\n",
"\n",
" # print(entry)\n",
" dbg_lines += 1\n",
" # if dbg_lines == 100000:\n",
" # break\n",
" except pickle.UnpicklingError:\n",
" line = openfile.readline().decode('UTF-8')\n",
" if \"Ticks per second\" in line:\n",
" tps = int(line.split(': ')[2])\n",
" input_sets[-1].tps = tps\n",
" elif \"ticks per source interval\" in line:\n",
" tpf = int(line.split(': ')[2])\n",
" input_sets[-1].dt_ref = tpf\n",
" elif \"t_max\" in line:\n",
" dt_max = int(line.split(': ')[2])\n",
" adu_interval = int(dt_max / tpf)\n",
" # print(line)\n",
" # break\n",
" except EOFError:\n",
" break\n",
"\n",
" # Remove the last item because it might have none values\n",
" input_sets = input_sets[:-1]\n",
" if stage == Stage.RAW:\n",
" raise Exception(\"Did not perform reconstruction!\")\n",
" return input_sets, width, height, channels, tps, tpf\n",
" \n",
"print(sorted_files[0][1])\n",
"input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + sorted_files[0][1])\n",
"\n",
"print('\\nAll done')\n",
"print(input_sets[0].recon_psnr)\n",
"print(input_sets[0].compressed_bitrate)\n",
"print(input_sets[30].compressed_bitrate)\n",
"\n",
"print(sorted_files[1][1])\n",
"input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + sorted_files[1][1])\n",
"\n",
"print('\\nAll done')\n",
"print(input_sets[0].recon_psnr)\n",
"print(input_sets[0].compressed_bitrate)\n",
"print(input_sets[30].compressed_bitrate)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "720e91ca3c22cbce",
"metadata": {},
"outputs": [],
"source": [
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import mpld3\n",
"QUALITY_TO_LABEL = ['Lossless','High','Medium','Low']\n",
"\n",
"def feature_count_plot_all(sets):\n",
" new_fig()\n",
"\n",
" for i, input_sets in enumerate(sets[1::2]):\n",
" adder_feature_count = [obj.adder_feature_count for obj in input_sets]\n",
" if i == 3:\n",
" cv_feature_count = [obj.cv_feature_count for obj in input_sets]\n",
"\n",
"\n",
" \n",
" \n",
" sns.lineplot(x=range(len(input_sets) - 1), y=adder_feature_count[:-1], marker=None, label=\"ADΔER_\" + QUALITY_TO_LABEL[i])\n",
" \n",
" \n",
" sns.lineplot(x=range(len(input_sets) - 1), y=cv_feature_count[:-1], marker=None, label=\"ADΔER_\" + QUALITY_TO_LABEL[i] +'_OpenCV')\n",
"\n",
" plt.ylabel(\"# features detected\")\n",
" plt.xlim(0, 500)\n",
" plt.xlabel(\"Input frame\")\n",
" # plt.title(\"Feature count over time\")\n",
" plt.show()\n",
" # interactive_plot = mpld3.display()\n",
" # interactive_plot\n",
"\n",
"\n",
"def feature_speed_plot_all(sets):\n",
" new_fig()\n",
"\n",
" assert sets[0][4].adder_time is None\n",
" for i, input_sets in enumerate(sets[1::2]):\n",
" adder_times = [obj.adder_time for obj in input_sets]\n",
"\n",
" sns.lineplot(x=range(len(input_sets)), y=adder_times, marker=None, label=\"ADΔER_\" + QUALITY_TO_LABEL[i])\n",
" \n",
" for i, input_sets in enumerate(sets[1::2]):\n",
" if i == 3:\n",
" cv_times = [obj.cv_time for obj in input_sets]\n",
" sns.lineplot(x=range(len(input_sets)), y=cv_times, marker=None, label=\"ADΔER_OpenCV_\" + QUALITY_TO_LABEL[i])\n",
" plt.ylabel(\"Feature detection time (ns)\")\n",
" plt.xlabel(\"Input frame\")\n",
" plt.yscale('log')\n",
" plt.xlim(0, 500)\n",
" plt.ylim(900, 1e7)\n",
" # plt.title(\"Feature detection speed\")\n",
"\n",
" plt.show()\n",
" # interactive_plot = mpld3.display()\n",
" # interactive_plot\n",
"\n",
" cv_mean = np.mean(np.array(cv_times))\n",
" adder_mean = np.mean(np.array(adder_times))\n",
" percent_change = (adder_mean / cv_mean) - 1\n",
" print('OpenCV mean:', cv_mean, ' Median:', np.median(np.array(cv_times)))\n",
" print('ADDER mean:', adder_mean, \"({:.1f}%)\".format(percent_change * 100), ' Median:',\n",
" np.median(np.array(adder_times)))\n",
"\n",
"\n",
"def bitrates_plot_all(sets, width, height, channels, tps, tpf):\n",
" bitrates_plot_partial(sets[::2], width, height, channels, tps, tpf) # without feat detection\n",
" # plt.title(\"Bitrate\")\n",
" \n",
" bitrates_plot_partial(sets[1::2], width, height, channels, tps, tpf) # with feat detection\n",
" # plt.title(\"Bitrate, feature detection\")\n",
" \n",
"def bitrates_plot_partial(sets, width, height, channels, tps, tpf):\n",
" new_fig()\n",
"\n",
" for i, input_sets in enumerate(sets):\n",
" adder_bitrates = [obj.event_bitrate for obj in input_sets]\n",
"\n",
" total_adder_size = np.sum(adder_bitrates) / (tps / tpf)\n",
" print(total_adder_size, \"Mb\")\n",
"\n",
" source_raw_Mbps = (width * height * channels * (tps / tpf) * 8.0) / 1024.0 / 1024.0\n",
" total_raw_size = (width * height * channels * len(adder_bitrates) * 8.0) / 1024.0 / 1024.0\n",
" print(total_raw_size, \"Mb\")\n",
" framed_bitrates = [source_raw_Mbps for obj in input_sets]\n",
"\n",
"\n",
" sns.lineplot(x=range(len(input_sets)), y=adder_bitrates, marker=None, label=\"ADΔER_\" + QUALITY_TO_LABEL[i])\n",
" \n",
" sns.lineplot(x=range(len(input_sets)), y=framed_bitrates, marker=None, label=\"Image frames (8 bits per pixel)\")\n",
" \n",
" plt.ylabel(\"Raw bitrate (Mb/s)\")\n",
" plt.xlabel(\"Input frame\")\n",
" plt.yscale('log')\n",
" plt.legend(loc='lower left')\n",
" plt.ylim(0.0001, 1000)\n",
" # plt.title(\"Bitrate\")\n",
"\n",
" plt.show()\n",
"\n",
"def quality_plot_all(sets):\n",
" quality_plot_partial(sets[::2]) # Without features\n",
" # plt.title(\"Quality metrics-raw_nofeatures\")\n",
" \n",
" quality_plot_partial(sets[1::2]) # With features\n",
" plt.title(\"Quality metrics-raw_features\")\n",
" print(\"recon quality:\")\n",
" quality_plot_partial(sets[::2],recon=True) # Without features\n",
" plt.title(\"Quality metrics-recon_nofeatures\")\n",
" quality_plot_partial(sets[1::2],recon=True) # With features\n",
" plt.title(\"Quality metrics-recon_features\")\n",
" \n",
" plt.show()\n",
"\n",
"def quality_plot_partial(sets, recon=False):\n",
"\n",
" \n",
" # PSNR\n",
" new_fig()\n",
" for i, input_sets in enumerate(sets):\n",
" if recon:\n",
" adder_mse = [obj.recon_mse for obj in input_sets]\n",
" # adder_psnr = [obj.recon_psnr for obj in input_sets]\n",
" else: \n",
" adder_mse = [obj.mse for obj in input_sets]\n",
" # adder_psnr = [obj.psnr for obj in input_sets]\n",
" sns.lineplot(x=range(len(input_sets)), y=adder_mse, marker=None, label=\"ADΔER_\"+ QUALITY_TO_LABEL[i])\n",
" plt.ylabel(\"Mean squared error\")\n",
" plt.xlabel(\"Input frame\")\n",
" # plt.title(\"Quality metrics\")\n",
" \n",
" # # SSIM\n",
" # plt.figure(figsize=(8, 4))\n",
" # for i, input_sets in enumerate(sets):\n",
" # if recon:\n",
" # adder_ssim = [obj.recon_ssim for obj in input_sets]\n",
" # else: \n",
" # adder_ssim = [obj.ssim for obj in input_sets]\n",
" # \n",
" # sns.lineplot(x=range(len(input_sets)), y=adder_ssim, marker=None, label=\"SSIM_\" + str(i))\n",
" # plt.ylabel(\"Score\")\n",
" # plt.xlabel(\"Input frame\")\n",
" # plt.title(\"Quality metrics\")\n",
"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dc863641fb6bc4b9",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "2cdef3edc412e2c5",
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_VIDEO_NUM = 8 # We expect to run 8 permutations: quality 0,3,6,9 both with and without feature detection\n",
"\n",
"for name, files in name_to_files.items():\n",
" print(f\"[name]: {name}\")\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" current_input_sets = []\n",
" \n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
"\n",
" feature_count_plot_all(current_input_sets)\n",
" feature_speed_plot_all(current_input_sets)\n",
" bitrates_plot_all(current_input_sets, width, height, channels, tps, tpf)\n",
" quality_plot_all(current_input_sets)\n",
" \n",
" # plt.figure(figsize=(8, 4))\n",
" # sns.set(style=\"whitegrid\")\n",
" # plot_mean_bitrate(current_input_sets)\n",
" # plt.show()\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "57a80b39a872091",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "77cf1ef098b45a0e",
"metadata": {
"collapsed": false
},
"source": [
"## All mean bitrates"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "49c9c6aa5d2a59f0",
"metadata": {},
"outputs": [],
"source": [
"palette = \"husl\"\n",
"COLOR_MAP = sns.color_palette(palette, 4)\n",
"\n",
"\n",
"def plot_mean_bitrate(sets, feat=False, recon=False, marker='o'):\n",
" if feat:\n",
" sets = sets[1::2]\n",
" else:\n",
" sets = sets[::2]\n",
" return plot_mean_bitrate_partial(sets, recon, marker)\n",
" \n",
"def plot_mean_bitrate_partial(sets, recon=False, marker='o'):\n",
" running_bitrate = 0\n",
"\n",
" mean_bitrates = np.zeros((len(sets)))\n",
" mean_qualities = np.zeros((len(sets)))\n",
"\n",
" for i, input_sets in enumerate(sets):\n",
" # Get the mean adder bitrate\n",
" if recon:\n",
" adder_bitrates = [obj.compressed_bitrate for obj in input_sets if obj.compressed_bitrate is not None]\n",
" adder_quality = [obj.recon_psnr for obj in input_sets if obj.recon_psnr is not None]\n",
" else:\n",
" adder_bitrates = [obj.event_bitrate for obj in input_sets if obj.event_bitrate is not None]\n",
" adder_quality = [obj.psnr for obj in input_sets if obj.psnr is not None]\n",
" \n",
" mean_bitrate = np.sum(adder_bitrates)/ len(adder_bitrates)\n",
" adder_quality = [min(x, 60.0) for x in adder_quality] # TODO: Mention in paper that we cap PSNR at 50 for practical reasons\n",
" mean_quality = np.sum(adder_quality)/ len(adder_quality)\n",
" # if mean_quality < 1.0:\n",
" # mean_quality = 1.0\n",
" running_bitrate += mean_bitrate \n",
" \n",
" # if i > 0:\n",
" # # plot the line\n",
" # plt.plot([last_bitrate, mean_bitrate], [last_quality, mean_quality], c=COLOR_MAP[i], label='Line', alpha=0.5)\n",
" # plot this as a single point\n",
" \n",
" plt.scatter( mean_bitrate, mean_quality, c=[COLOR_MAP[i]], marker=marker, s=40, label='Point', alpha=0.5) \n",
" \n",
" mean_bitrates[i] = mean_bitrate\n",
" mean_qualities[i] = mean_quality\n",
" \n",
" last_bitrate = mean_bitrate\n",
" last_quality = mean_quality\n",
" \n",
" return mean_bitrates, mean_qualities\n",
" \n",
"\n",
" \n",
" \n",
" \n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a03e64096c8b0b46",
"metadata": {},
"outputs": [],
"source": [
"EXPECTED_VIDEO_NUM = 8 # We expect to run 8 permutations: quality 0,3,6,9 both with and without feature detection\n",
"\n",
"def new_fig_setup_bitrate_quality(feat=False, recon=False):\n",
" new_fig()\n",
" sns.set(style=\"darkgrid\")\n",
" sns.set_context(\"paper\")\n",
" #Make the xaxis log\n",
" plt.xscale('log')\n",
" plt.xlim(0.1, 510)\n",
" xticks = [0.1, 1, 10, 100, 500]\n",
" plt.xticks(xticks, xticks)\n",
" plt.xlabel(\"Mean bitrate (Mb/s)\")\n",
" \n",
" plt.ylim(25, 62)\n",
" plt.ylabel(\"Mean PSNR (dB)\")\n",
"\n",
" \n",
" # Set the title\n",
" if feat:\n",
" if recon:\n",
" plt.title(\"Source-modeled compression with feature detection\")\n",
" else:\n",
" plt.title(\"Raw events with feature detection\")\n",
" else:\n",
" if recon:\n",
" plt.title(\"Source-modeled compression\")\n",
" else:\n",
" plt.title(\"Raw events\")\n",
"\n",
" all_mean_bitrates = []\n",
" all_mean_qualities = []\n",
" h265_bitrates = []\n",
"\n",
" for name, files in name_to_files.items():\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" current_input_sets = []\n",
" \n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
" mean_bitrates, mean_qualities = plot_mean_bitrate(current_input_sets, feat=feat, recon=recon)\n",
" \n",
" # Get the size of the h265-encoded mp4 file\n",
" try:\n",
" h265_size = os.path.getsize(current_directory + '/../' + name[:-1] + '.mp4')\n",
" h265_size_per_frame = h265_size / len(input_sets)\n",
" h265_bits_per_sec = h265_size_per_frame * 8 * (tps / tpf)\n",
" h265_bitrates += [h265_bits_per_sec / 1024 / 1024] #Mb/s\n",
" except:\n",
" print(name[:-1] + '.mp4 not found')\n",
" \n",
" all_mean_bitrates += [mean_bitrates]\n",
" all_mean_qualities += [mean_qualities]\n",
" return all_mean_bitrates, all_mean_qualities, h265_bitrates\n",
" \n",
" \n",
"\n",
"mean_bitrates, mean_qualities, _ = new_fig_setup_bitrate_quality(feat=False, recon=False)\n",
"plt.show()\n",
"\n",
"mean_bitrates_feat, mean_qualities_feat, _ = new_fig_setup_bitrate_quality(feat=True, recon=False)\n",
"plt.show()\n",
"\n",
"\n",
"# recon\n",
"mean_bitrates_recon, mean_qualities_recon, _ = new_fig_setup_bitrate_quality(feat=False, recon=True)\n",
"plt.show()\n",
"\n",
"mean_bitrates_feat_recon, mean_qualities_feat_recon, h265_bitrates = new_fig_setup_bitrate_quality(feat=True, recon=True)\n",
"plt.show()\n",
"\n",
"\n",
"# Determine the change in quality and size between regular and reconstruction\n",
"mean_changes_bitrate = (np.array(mean_bitrates_recon)/np.array(mean_bitrates))-1.0\n",
"mean_changes_quality = np.array(mean_qualities_recon) - np.array(mean_qualities)\n",
"print(\"Mean bitrate change % 0:\", np.mean(mean_changes_bitrate[0::4]))\n",
"print(\"Mean bitrate change % 1:\", np.mean(mean_changes_bitrate[1::4]))\n",
"print(\"Mean bitrate change % 2:\", np.mean(mean_changes_bitrate[2::4]))\n",
"print(\"Mean bitrate change % 3:\", np.mean(mean_changes_bitrate[3::4]))\n",
"print(\"Mean quality change 0:\", np.mean(mean_changes_quality[0::4]))\n",
"print(\"Mean quality change 1:\", np.mean(mean_changes_quality[1::4]))\n",
"print(\"Mean quality change 2:\", np.mean(mean_changes_quality[2::4]))\n",
"print(\"Mean quality change 3:\", np.mean(mean_changes_quality[3::4]))\n",
"\n",
"mean_changes_bitrate = (np.array(mean_bitrates_feat_recon)/np.array(mean_bitrates_feat))-1.0\n",
"mean_changes_quality = np.array(mean_qualities_feat_recon) - np.array(mean_qualities_feat)\n",
"print(\"Mean bitrate feat change % 0:\", np.mean(mean_changes_bitrate[0::4]))\n",
"print(\"Mean bitrate feat change % 1:\", np.mean(mean_changes_bitrate[1::4]))\n",
"print(\"Mean bitrate feat change % 2:\", np.mean(mean_changes_bitrate[2::4]))\n",
"print(\"Mean bitrate feat change % 3:\", np.mean(mean_changes_bitrate[3::4]))\n",
"print(\"Mean quality feat change 0:\", np.mean(mean_changes_quality[0::4]))\n",
"print(\"Mean quality feat change 1:\", np.mean(mean_changes_quality[1::4]))\n",
"print(\"Mean quality feat change 2:\", np.mean(mean_changes_quality[2::4]))\n",
"print(\"Mean quality feat change 3:\", np.mean(mean_changes_quality[3::4]))\n",
"\n",
"\n",
"mean_changes_bitrate_h265_to_adder_0 = (np.array(mean_bitrates_recon)[:, 0]/np.array(h265_bitrates))-1.0\n",
"mean_changes_bitrate_h265_to_adder_1 = (np.array(mean_bitrates_recon)[:, 1]/np.array(h265_bitrates))-1.0\n",
"mean_changes_bitrate_h265_to_adder_2 = (np.array(mean_bitrates_recon)[:, 2]/np.array(h265_bitrates))-1.0\n",
"mean_changes_bitrate_h265_to_adder_3 = (np.array(mean_bitrates_recon)[:, 3]/np.array(h265_bitrates))-1.0\n",
"\n",
"print(\"Mean h265 bitrate change % 0:\", np.mean(mean_changes_bitrate_h265_to_adder_0))\n",
"print(\"Mean h265 bitrate change % 1:\", np.mean(mean_changes_bitrate_h265_to_adder_1))\n",
"print(\"Mean h265 bitrate change % 2:\", np.mean(mean_changes_bitrate_h265_to_adder_2))\n",
"print(\"Mean h265 bitrate change % 3:\", np.mean(mean_changes_bitrate_h265_to_adder_3))\n",
"# \n",
"# data = pd.DataFrame(columns=['Category', 'Value'])\n",
"# sns.boxplot(y='Category', x='Value', orient=\"h\", data=data, hue=\"Category\", palette=COLOR_MAP)\n",
"# quartiles = data.groupby('Category')['Value'].quantile([0.25, 0.5, 0.75])\n",
"# print(\"Quartiles for each category:\")\n",
"# print(quartiles)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b7d927272c31223a",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T19:23:14.366703682Z",
"start_time": "2024-01-30T19:22:59.813452645Z"
}
},
"outputs": [],
"source": [
"import matplotlib.lines as mlines\n",
"import warnings\n",
"warnings.simplefilter(action='ignore', category=FutureWarning)\n",
"import pandas as pd\n",
"\n",
"below_bitrate = pd.DataFrame(columns=[\"input_sets\", \"h265_bitrate\"])\n",
"above1x_bitrate = pd.DataFrame(columns=[\"input_sets\", \"h265_bitrate\"])\n",
"above2x_bitrate = pd.DataFrame(columns=[\"input_sets\", \"h265_bitrate\"])\n",
"above3x_bitrate = pd.DataFrame(columns=[\"input_sets\", \"h265_bitrate\"])\n",
"\n",
"def new_fig_setup_bitrate_quality_single_video(video_index):\n",
" global below_bitrate\n",
" global above1x_bitrate\n",
" global above2x_bitrate\n",
" global above3x_bitrate\n",
"\n",
" \n",
" all_mean_bitrates = []\n",
" all_mean_qualities = []\n",
" h265_bitrates = []\n",
"\n",
" name = list(name_to_files.keys())[video_index]\n",
" files = list(name_to_files.values())[video_index]\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" \n",
" current_input_sets = []\n",
" \n",
" \n",
" video_count = 0\n",
" for time, file in sorted_files:\n",
" try:\n",
" # print(file)\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" return\n",
" \n",
" # Get the size of the h265-encoded mp4 file\n",
" try:\n",
" h265_size = os.path.getsize(current_directory + '/../' + name[:-1] + '.mp4')\n",
" h265_size_per_frame = h265_size / len(input_sets)\n",
" h265_bits_per_sec = h265_size_per_frame * 8 * (tps / tpf)\n",
" h265_bitrates += [h265_bits_per_sec / 1024 / 1024] #Mb/s\n",
" \n",
" \n",
" \n",
" except:\n",
" print(name[:-1] + '.mp4 not found')\n",
" return\n",
" mean_low_compressed_bitrate = np.mean(np.array([obj.compressed_bitrate for obj in current_input_sets[-2] if obj.compressed_bitrate is not None]))\n",
" # print(mean_low_compressed_bitrate)\n",
" # print(h265_bitrates[-1])\n",
" \n",
" \n",
" new_row = pd.DataFrame({\"input_sets\": [current_input_sets], \"h265_bitrate\": h265_bitrates[-1]},index=[0])\n",
" if mean_low_compressed_bitrate >= h265_bitrates[-1] * 3:\n",
" return\n",
" above3x_bitrate = pd.concat([above3x_bitrate, new_row], ignore_index=True)\n",
" \n",
" elif mean_low_compressed_bitrate >= h265_bitrates[-1] * 2 and mean_low_compressed_bitrate < h265_bitrates[-1] * 3:\n",
" return\n",
" above2x_bitrate = pd.concat([above2x_bitrate, new_row], ignore_index=True)\n",
" elif mean_low_compressed_bitrate >= h265_bitrates[-1] * 1 and mean_low_compressed_bitrate < h265_bitrates[-1] * 2:\n",
" return\n",
" \n",
" # print(np.mean(np.array([obj.compressed_bitrate for obj in current_input_sets[-1] if obj.compressed_bitrate is not None])))\n",
" # print(h265_bitrates[-1])\n",
" above1x_bitrate = pd.concat([above1x_bitrate, new_row], ignore_index=True)\n",
" elif mean_low_compressed_bitrate < h265_bitrates[-1] * 1:\n",
" # return\n",
" print(mean_low_compressed_bitrate)\n",
" print(h265_bitrates[-1])\n",
" below_bitrate = pd.concat([below_bitrate, new_row], ignore_index=True)\n",
" \n",
" # print(np.mean(np.array([obj.compressed_bitrate for obj in current_input_sets[-1] if obj.compressed_bitrate is not None])))\n",
" # print(h265_bitrates[-1])\n",
" else:\n",
" return\n",
" \n",
" new_fig()\n",
" plt.rcParams['figure.figsize'] = [1, 1]\n",
"\n",
" sns.set(style=\"darkgrid\")\n",
" sns.set_context(\"paper\")\n",
" #Make the xaxis log\n",
" \n",
" plt.xlabel(\"Mean bitrate (Mb/s)\")\n",
" xticks = [0.1, 1, 10, 100, 500]\n",
" plt.ylim(30, 62)\n",
" plt.ylabel(\"Mean PSNR (dB)\")\n",
" xticks = [0.1, round(h265_bitrates[-1], 2), 1, 10, 100, 500]\n",
" plt.axvline(x = h265_bitrates[-1], color = 'b', linestyle = '--')\n",
" \n",
" mean_bitrates, mean_qualities = plot_mean_bitrate(current_input_sets, feat=False, recon=False, marker='o')\n",
" mean_bitrates, mean_qualities = plot_mean_bitrate(current_input_sets, feat=False, recon=True, marker='x')\n",
" mean_bitrates, mean_qualities = plot_mean_bitrate(current_input_sets, feat=True, recon=False, marker='s')\n",
" mean_bitrates, mean_qualities = plot_mean_bitrate(current_input_sets, feat=True, recon=True, marker='+')\n",
" \n",
" circle = mlines.Line2D([], [], color='black', marker='o', linestyle='None',\n",
" label='Standard')\n",
" ex = mlines.Line2D([], [], color='black', marker='x', linestyle='None',\n",
" label='Compression')\n",
" plus = mlines.Line2D([], [], color='black', marker='+', linestyle='None',\n",
" label='Compression\\n+ feature detection')\n",
" ess = mlines.Line2D([], [], color='black', marker='s', linestyle='None',\n",
" label='Feature detection')\n",
" \n",
" color1 = mlines.Line2D([], [], color=COLOR_MAP[0], label='Lossless')\n",
" color2 = mlines.Line2D([], [], color=COLOR_MAP[1], label='High')\n",
" color3 = mlines.Line2D([], [], color=COLOR_MAP[2], label='Medium')\n",
" color4 = mlines.Line2D([], [], color=COLOR_MAP[3], label='Low')\n",
" \n",
" \n",
" \n",
" \n",
" \n",
" vline = mlines.Line2D([], [], color='b', label='H.265 @ CRF 23', linestyle='--')\n",
" plt.legend(handles=[circle, ex, ess, plus, color1, color2, color3, color4, vline])\n",
" plt.title(name)\n",
" \n",
" plt.xscale('log')\n",
" plt.xlim(0.1, 510)\n",
" \n",
" plt.xticks(xticks, xticks)\n",
" \n",
" plt.savefig('/home/andrew/Videos/VIRAT_gray_0.5_2/plots_tmp/' + name + '.png', bbox_inches='tight')\n",
" all_mean_bitrates += [mean_bitrates]\n",
" all_mean_qualities += [mean_qualities]\n",
" \n",
" return all_mean_bitrates, all_mean_qualities, h265_bitrates\n",
"\n",
"for i in range(len(list(name_to_files.keys()))):\n",
" new_fig_setup_bitrate_quality_single_video(i)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a19597ed2498f762",
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"warnings.simplefilter(action='ignore', category=FutureWarning)\n",
"import pandas as pd\n",
"import math\n",
"\n",
"EXPECTED_VIDEO_NUM = 8 # We expect to run 8 permutations: quality 0,3,6,9 both with and without feature detection\n",
"\n",
"def get_feature_bitrate_quality_effect(sets, sets_features, data, recon=False):\n",
" \n",
" # categories = [3, 6, 9]\n",
" \n",
" for (i, input_sets), input_sets_features in zip(enumerate(sets), sets_features):\n",
" # Get the mean adder bitrate\n",
" if recon:\n",
" adder_bitrates = [obj.compressed_bitrate for obj in input_sets if obj.compressed_bitrate is not None]\n",
" adder_quality = [obj.recon_psnr for obj in input_sets if obj.recon_psnr is not None]\n",
" adder_quality_mse = [obj.recon_mse for obj in input_sets if obj.recon_mse is not None]\n",
" adder_bitrates_features = [obj.compressed_bitrate for obj in input_sets_features if obj.compressed_bitrate is not None]\n",
" adder_quality_features = [obj.recon_psnr for obj in input_sets_features if obj.recon_psnr is not None]\n",
" adder_quality_features_mse = [obj.recon_mse for obj in input_sets_features if obj.recon_mse is not None]\n",
" else:\n",
" adder_bitrates = [obj.event_bitrate for obj in input_sets if obj.event_bitrate is not None]\n",
" adder_quality = [obj.psnr for obj in input_sets if obj.psnr is not None]\n",
" adder_quality_mse = [obj.mse for obj in input_sets if obj.mse is not None]\n",
" adder_bitrates_features = [obj.event_bitrate for obj in input_sets_features if obj.event_bitrate is not None]\n",
" adder_quality_features = [obj.psnr for obj in input_sets_features if obj.psnr is not None]\n",
" adder_quality_features_mse = [obj.mse for obj in input_sets_features if obj.mse is not None]\n",
" \n",
" mean_bitrate = np.sum(adder_bitrates)/ len(adder_bitrates)\n",
" adder_quality = [min(x, 60) for x in adder_quality] # TODO: Mention in paper that we cap PSNR at 50 for practical reasons\n",
" mean_quality = np.sum(adder_quality)/ len(adder_quality)\n",
" mean_bitrate_features = np.sum(adder_bitrates_features)/ len(adder_bitrates_features)\n",
" adder_quality_features = [min(x, 60) for x in adder_quality_features] # TODO: Mention in paper that we cap PSNR at 50 for practical reasons\n",
" mean_quality_features = np.sum(adder_quality_features)/ len(adder_quality_features)\n",
" \n",
" mean_quality_mse = np.mean(np.array(adder_quality_mse))\n",
" \n",
" \n",
" if i > 0:\n",
" # plot the line\n",
" # plt.plot([last_bitrate, mean_bitrate], [last_quality, mean_quality], c=COLOR_MAP[i], label='Line', alpha=0.5)\n",
" \n",
" # Get the predicted quality (wihout features) at the bitrate with features\n",
" # Do logarithmic interpolation to predict the PSNR at the bitrate with features\n",
" log_bitrate1 = math.log10(last_bitrate)\n",
" log_bitrate2 = math.log10(mean_bitrate)\n",
" slope = (last_quality - mean_quality) / (log_bitrate1 - log_bitrate2)\n",
" \n",
" \n",
" # slope = (last_quality - mean_quality) / (last_bitrate - mean_bitrate)\n",
" assert slope > 0\n",
" x_diff = math.log10(mean_bitrate_features) - math.log10(mean_bitrate)\n",
" predicted_quality = min((slope * x_diff) + mean_quality, 48.15) # Don't predict aything past our max practical psnr value\n",
" \n",
" # get the difference between our predicted and actual quality with feature detection\n",
" # If this number is positive, then feature detection offers a benefit!\n",
" quality_diff = mean_quality_features - predicted_quality\n",
" # print(quality_diff)\n",
" \n",
" # Add a point to the boxplot\n",
" new_point = {'Category': i, 'Value': quality_diff}\n",
" data = pd.concat([data, pd.DataFrame(new_point, index=[0])], ignore_index=True)\n",
" \n",
" last_bitrate = mean_bitrate\n",
" last_quality = mean_quality \n",
" return data\n",
"\n",
"def new_fig_setup_feature_effect( recon=False):\n",
" new_fig()\n",
" sns.set(style=\"darkgrid\")\n",
" sns.set_context(\"paper\")\n",
" #Make the xaxis log\n",
" # plt.xscale('log')\n",
" # plt.xlim(0.1, 510)\n",
" # xticks = [0.1, 1, 10, 100, 500]\n",
" # plt.xticks(xticks, xticks)\n",
" plt.ylabel(\"Quality level\")\n",
" # \n",
" plt.xlim(-6, 4)\n",
" plt.xlabel(\"Change in PSNR (dB)\")\n",
"\n",
" if recon: \n",
" plt.title(\"Effect of feature detection on quality at same bitrate (reconstruction)\")\n",
" else:\n",
" plt.title(\"Effect of feature detection on quality at same bitrate \")\n",
"\n",
" # Create an empty DataFrame to store the data\n",
" data = pd.DataFrame(columns=['Category', 'Value'])\n",
"\n",
" for name, files in name_to_files.items():\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" current_input_sets = []\n",
" \n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
" data = get_feature_bitrate_quality_effect(current_input_sets[::2], current_input_sets[1::2], data, recon=recon)\n",
" \n",
" sns.boxplot(y='Category', x='Value', orient='h', data=data, hue=\"Category\")\n",
"\n",
" \n",
"\n",
"new_fig_setup_feature_effect(recon=False)\n",
"plt.show()\n",
"\n",
"new_fig_setup_feature_effect(recon=True)\n",
"plt.show()\n"
]
},
{
"cell_type": "markdown",
"id": "3853eb3e907bcd59",
"metadata": {
"collapsed": false
},
"source": [
"## Feature speed change overall (probably just make this a table!!)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9a6f732ad62d101",
"metadata": {},
"outputs": [],
"source": [
"def get_feature_speed_change(sets, data, recon=False):\n",
" \n",
" # categories = ['opencv', 0, 3, 6, 9]\n",
" cv_means = 0 #normalize this across the four video runs, since it's pretty much always constant\n",
" for i, input_sets in enumerate(sets):\n",
" # Get the mean adder bitrate\n",
" adder_times = [obj.adder_time for obj in input_sets]\n",
" cv_times = [obj.cv_time for obj in input_sets]\n",
" # percent_change = (adder_mean / cv_mean) - 1\n",
"\n",
" changes = (np.array(adder_times) / np.array(cv_times)) - 1.0\n",
" changes *= 100.0\n",
" \n",
" cv_mean = np.mean(np.array(cv_times))\n",
" cv_means += cv_mean\n",
" # adder_mean = np.mean(np.array(adder_times))\n",
" adder_mean = np.mean(changes)\n",
" \n",
" # change_sum = (np.sum(np.array(adder_times)) / np.sum(np.array(cv_times))) - 1.0\n",
" # adder_mean = change_sum * 100.0\n",
" \n",
" # Add a point to the boxplot\n",
" # if i > 0: # Don't show 0 quality on here, because it's not very useful\n",
" new_point = {'Category': i, 'Value': adder_mean}\n",
" data = pd.concat([data, pd.DataFrame(new_point, index=[0])], ignore_index=True)\n",
" \n",
" # if i > 0 and adder_mean > cv_mean:\n",
" # print(adder_mean, cv_mean)\n",
" \n",
" # cv_means /= len(sets)\n",
" # new_point = {'Category': 'OpenCV', 'Value': cv_means}\n",
" # data = pd.concat([data, pd.DataFrame(new_point, index=[0])], ignore_index=True)\n",
" return data \n",
" \n",
"\n",
"def new_fig_setup_feature_speed_change():\n",
" new_fig()\n",
" sns.set(style=\"darkgrid\")\n",
" sns.set_context(\"paper\")\n",
" #Make the xaxis log\n",
" # plt.xscale('log')\n",
" plt.xlim(-100, 500)\n",
" \n",
" # xticks = [-100, 0, 100, 200, 400, 600, 800, 1000, 1200, 1400]\n",
" xticks = [-100, 0, 100, 200, 300, 400, 500]\n",
" plt.xticks(xticks, xticks)\n",
" plt.ylabel(\"Quality level\")\n",
" # \n",
" # plt.ylim(-1, 15)\n",
" # y_ticks = [-100, -10, -1, 1, 10, 100, 1000]\n",
" # plt.yticks(y_ticks, y_ticks)\n",
" # plt.yscale('log')\n",
" plt.xlabel(\"Change in execution time (%)\")\n",
" plt.title(\"Effect of ADDER transcode on feature detection speed\")\n",
"\n",
" # Create an empty DataFrame to store the data\n",
" data = pd.DataFrame(columns=['Category', 'Value'])\n",
"\n",
" for name, files in name_to_files.items():\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" current_input_sets = []\n",
" \n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
" data = get_feature_speed_change(current_input_sets[1::2], data)\n",
" \n",
" sns.boxplot(y='Category', x='Value', orient=\"h\", data=data, hue=\"Category\", palette=COLOR_MAP)\n",
" quartiles = data.groupby('Category')['Value'].quantile([0.25, 0.5, 0.75])\n",
" print(\"Quartiles for each category:\")\n",
" print(quartiles)\n",
"\n",
" \n",
"\n",
"new_fig_setup_feature_speed_change()\n",
"plt.show()\n",
" "
]
},
{
"cell_type": "markdown",
"id": "8fbacf7a254efa44",
"metadata": {
"collapsed": false
},
"source": [
"## Feature speed change vs. ADDER raw event rate"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "63020d9507549257",
"metadata": {},
"outputs": [],
"source": [
"def get_feature_speed_change_rate(sets, data, recon=False):\n",
" \n",
" # categories = ['opencv', 0, 3, 6, 9]\n",
" cv_means = 0 #normalize this across the four video runs, since it's pretty much always constant\n",
" for i, input_sets in enumerate(sets):\n",
" # Get the mean adder bitrate\n",
" \n",
" adder_times = [obj.adder_time for obj in input_sets]\n",
" cv_times = [obj.cv_time for obj in input_sets]\n",
" \n",
" \n",
" \n",
" changes = (np.array(adder_times) / np.array(cv_times)) - 1.0\n",
" changes *= 100.0\n",
"\n",
" # Get the mean raw adder bitrate\n",
" adder_bitrates = [obj.event_bitrate for obj in input_sets]\n",
" event_rates = np.array(adder_bitrates) * 1024.0 *1024.0 / 8.0\n",
" event_rates /= np.array([obj.num_pixels for obj in input_sets])\n",
" if input_sets[0].num_channels == 1:\n",
" event_rates /= 9\n",
" else:\n",
" event_rates /= 11\n",
" event_rates /= (input_sets[0].tps / input_sets[0].dt_ref)\n",
" mean_bitrate = np.sum(adder_bitrates)/ len(adder_bitrates)\n",
" \n",
" change_sum = np.sum(np.array(adder_times)) - np.sum(np.array(cv_times))\n",
" # num_raw_samples =input_sets[0].num_pixels * len(input_sets)\n",
" # print(change_sum, str(np.sum(event_rates) > num_raw_samples))\n",
" \n",
" mean_change = np.mean(changes)\n",
" mean_rates = np.mean(event_rates)\n",
" # \n",
" new_point = {'Category': i, 'X': mean_rates, 'Y': mean_change}\n",
" data = pd.concat([data, pd.DataFrame(new_point, index=[0])], ignore_index=True)\n",
" # df = pd.DataFrame({'Category': i,'X': event_rates, 'Y': changes})\n",
" # data = pd.concat([data, df], ignore_index=True)\n",
"\n",
" return data \n",
" \n",
"\n",
"def new_fig_setup_feature_speed_change():\n",
" new_fig()\n",
" \n",
" #Make the xaxis log\n",
" # plt.xscale('log')\n",
" plt.xlim(0, 0.2)\n",
" # xticks = [0.1, 1, 10, 100, 500]\n",
" # plt.xticks(xticks, xticks)\n",
" plt.xlabel(\"Events per pixel, per input frame\")\n",
" # \n",
" plt.ylim(-100, 200)\n",
" # y_ticks = [-100, -10, -1, 1, 10, 100, 1000, 2000]\n",
" # plt.yticks(y_ticks, y_ticks)\n",
" # plt.yscale('log')\n",
" plt.ylabel(\"Change in execution time (%)\")\n",
" # plt.title(\"Effect of ADΔER transcode on feature detection speed\")\n",
"\n",
" # Create an empty DataFrame to store the data\n",
" data = pd.DataFrame(columns=['Category', 'Value'])\n",
"\n",
" for name, files in name_to_files.items():\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" current_input_sets = []\n",
" \n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" current_input_sets = current_input_sets + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
" data = get_feature_speed_change_rate(current_input_sets[1::2], data)\n",
" \n",
" sns.scatterplot(data=data, x=\"X\", y=\"Y\", hue=\"Category\", alpha=0.5, palette=COLOR_MAP)\n",
"\n",
"new_fig_setup_feature_speed_change()\n",
"plt.legend(title = 'ADΔER quality', loc='lower right', labels=QUALITY_TO_LABEL)\n",
"plt.show()\n",
" "
]
},
{
"cell_type": "markdown",
"id": "40358a04d51c854a",
"metadata": {
"collapsed": false
},
"source": [
"## Get the median quality at each CRF level"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9b601dd05c74fe90",
"metadata": {},
"outputs": [],
"source": [
"def quality_metrics(sets):\n",
" psnrs = pd.DataFrame(columns=[\"lossless\", \"high\", \"medium\", \"low\"])\n",
" psnrs_recon = pd.DataFrame(columns=[\"lossless\", \"high\", \"medium\", \"low\"])\n",
" mses = pd.DataFrame(columns=[\"lossless\", \"high\", \"medium\", \"low\"])\n",
" mses_recon = pd.DataFrame(columns=[\"lossless\", \"high\", \"medium\", \"low\"])\n",
" \n",
" \n",
" \n",
" \n",
" for i, input_sets in enumerate(sets):\n",
" current_mean = np.median(np.array([obj.psnr for obj in input_sets if obj.psnr is not None]))\n",
" if i % 4 == 0:\n",
" lossless = current_mean\n",
" elif i % 4 == 1:\n",
" high = current_mean\n",
" elif i % 4 == 2:\n",
" medium = current_mean\n",
" elif i % 4 == 3:\n",
" low = current_mean\n",
" psnrs = psnrs._append(pd.Series([lossless, high, medium, low], index=psnrs.columns), ignore_index=True)\n",
" \n",
" for i, input_sets in enumerate(sets):\n",
" current_mean = np.median(np.array([obj.recon_psnr for obj in input_sets if obj.recon_psnr is not None]))\n",
" if i % 4 == 0:\n",
" lossless = current_mean\n",
" elif i % 4 == 1:\n",
" high = current_mean\n",
" elif i % 4 == 2:\n",
" medium = current_mean\n",
" elif i % 4 == 3:\n",
" low = current_mean\n",
" psnrs_recon = psnrs_recon._append(pd.Series([lossless, high, medium, low], index=psnrs_recon.columns), ignore_index=True)\n",
" \n",
" for i, input_sets in enumerate(sets):\n",
" current_mean = np.median(np.array([obj.mse for obj in input_sets if obj.mse is not None]))\n",
" if i % 4 == 0:\n",
" lossless = current_mean\n",
" elif i % 4 == 1:\n",
" high = current_mean\n",
" elif i % 4 == 2:\n",
" medium = current_mean\n",
" elif i % 4 == 3:\n",
" low = current_mean\n",
" mses = mses._append(pd.Series([lossless, high, medium, low], index=mses.columns), ignore_index=True)\n",
" \n",
" for i, input_sets in enumerate(sets):\n",
" current_mean = np.median(np.array([obj.recon_mse for obj in input_sets if obj.recon_mse is not None]))\n",
" if i % 4 == 0:\n",
" lossless = current_mean\n",
" elif i % 4 == 1:\n",
" high = current_mean\n",
" elif i % 4 == 2:\n",
" medium = current_mean\n",
" elif i % 4 == 3:\n",
" low = current_mean\n",
" mses_recon = mses_recon._append(pd.Series([lossless, high, medium, low], index=mses_recon.columns), ignore_index=True)\n",
" \n",
"\n",
" return psnrs, psnrs_recon, mses, mses_recon\n",
"\n",
"all_input_sets = []\n",
"print(len(name_to_files))\n",
"for name, files in name_to_files.items():\n",
" sorted_files = sorted(files, key=lambda x: x[0]) # Sort by timestamp\n",
" tmp_set = []\n",
" video_count = 0\n",
" for _, file in sorted_files:\n",
" try:\n",
" input_sets, width, height, channels, tps, tpf = read_logfile(current_directory + '/' + file)\n",
" tmp_set = tmp_set + [input_sets]\n",
" video_count += 1\n",
" except Exception as e:\n",
" print(e)\n",
" break\n",
" if video_count < 8:\n",
" print(\"SKIPPING\", name)\n",
" continue\n",
" all_input_sets = all_input_sets + tmp_set\n",
" \n",
"print(len(all_input_sets))\n",
"psnrs, psnrs_recon, mses, mses_recon = quality_metrics(all_input_sets[0::2]) # no feat\n",
"print(np.percentile(psnrs[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"high\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"medium\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"low\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"high\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"medium\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"low\"], [25,50,75]))\n",
"print(np.percentile(mses[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(mses[\"high\"], [25,50,75]))\n",
"print(np.percentile(mses[\"medium\"], [25,50,75]))\n",
"print(np.percentile(mses[\"low\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"high\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"medium\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"low\"], [25,50,75]))\n",
"\n",
"\n",
"psnrs, psnrs_recon, mses, mses_recon = quality_metrics(all_input_sets[1::2]) # feat\n",
"\n",
"\n",
"print('\\n\\nfeat\\n')\n",
"print(np.percentile(psnrs[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"high\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"medium\"], [25,50,75]))\n",
"print(np.percentile(psnrs[\"low\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"high\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"medium\"], [25,50,75]))\n",
"print(np.percentile(psnrs_recon[\"low\"], [25,50,75]))\n",
"print(np.percentile(mses[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(mses[\"high\"], [25,50,75]))\n",
"print(np.percentile(mses[\"medium\"], [25,50,75]))\n",
"print(np.percentile(mses[\"low\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"lossless\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"high\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"medium\"], [25,50,75]))\n",
"print(np.percentile(mses_recon[\"low\"], [25,50,75]))"
]
},
{
"cell_type": "markdown",
"id": "e2eaadcb88d015c7",
"metadata": {
"collapsed": false
},
"source": [
"## Get stats based on the category"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "11e0f35acc130c0e",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T18:55:07.913158662Z",
"start_time": "2024-01-30T18:55:07.838046143Z"
}
},
"outputs": [],
"source": [
"def print_stats(psnrs, psnrs_recon, mses, mses_recon):\n",
" # print('psnrs')\n",
" # print(np.percentile(psnrs[\"lossless\"], [25,50,75]))\n",
" # print(np.percentile(psnrs[\"high\"], [25,50,75]))\n",
" # print(np.percentile(psnrs[\"medium\"], [25,50,75]))\n",
" # print(np.percentile(psnrs[\"low\"], [25,50,75]))\n",
" print('psnrs_recon')\n",
" print(np.percentile(psnrs_recon[\"lossless\"], [25,50,75]))\n",
" print(np.percentile(psnrs_recon[\"high\"], [25,50,75]))\n",
" print(np.percentile(psnrs_recon[\"medium\"], [25,50,75]))\n",
" print(np.percentile(psnrs_recon[\"low\"], [25,50,75]))\n",
" # print('mses')\n",
" # print(np.percentile(mses[\"lossless\"], [25,50,75]))\n",
" # print(np.percentile(mses[\"high\"], [25,50,75]))\n",
" # print(np.percentile(mses[\"medium\"], [25,50,75]))\n",
" # print(np.percentile(mses[\"low\"], [25,50,75]))\n",
" # print('mses_recon')\n",
" # print(np.percentile(mses_recon[\"lossless\"], [25,50,75]))\n",
" # print(np.percentile(mses_recon[\"high\"], [25,50,75]))\n",
" # print(np.percentile(mses_recon[\"medium\"], [25,50,75]))\n",
" # print(np.percentile(mses_recon[\"low\"], [25,50,75]))\n",
" \n",
"def print_stats_bitrate(bitrates_pctchange):\n",
" print('bitrates_pctchange')\n",
" print(np.percentile(bitrates_pctchange[\"lossless\"], [25,50,75]))\n",
" print(np.percentile(bitrates_pctchange[\"high\"], [25,50,75]))\n",
" print(np.percentile(bitrates_pctchange[\"medium\"], [25,50,75]))\n",
" print(np.percentile(bitrates_pctchange[\"low\"], [25,50,75]))\n",
" \n",
"def bitrate_metrics(sets, h265_bitrates):\n",
" bitrates_pctchange = pd.DataFrame(columns=[\"lossless\", \"high\", \"medium\", \"low\"])\n",
" for i, input_sets in enumerate(sets):\n",
" # print(np.array([obj.compressed_bitrate for obj in input_sets if obj.compressed_bitrate is not None]))\n",
" current_mean = np.mean(np.array([obj.compressed_bitrate for obj in input_sets if obj.compressed_bitrate is not None]))\n",
" # print(\"new\")\n",
" # print(current_mean)\n",
" # print(h265_bitrates[i])\n",
" if i % 4 == 0:\n",
" lossless = current_mean\n",
" elif i % 4 == 1:\n",
" high = current_mean\n",
" elif i % 4 == 2:\n",
" medium = current_mean\n",
" elif i % 4 == 3:\n",
" low = current_mean\n",
" \n",
" # Change each to be the percentage change from h265_bitrate\n",
" lossless = ((lossless / h265_bitrates[i]) - 1.0) *100.0\n",
" high = ((high / h265_bitrates[i]) - 1.0) *100.0\n",
" medium = ((medium / h265_bitrates[i]) - 1.0) *100.0\n",
" low = ((low / h265_bitrates[i]) - 1.0) *100.0\n",
" \n",
" \n",
" bitrates_pctchange = bitrates_pctchange._append(pd.Series([lossless, high, medium, low], index=bitrates_pctchange.columns), ignore_index=True)\n",
" \n",
" return bitrates_pctchange\n",
"\n",
"def get_stats(all_sets):\n",
" df_exploded = all_sets.explode(\"input_sets\")\n",
" \n",
" sets = df_exploded['input_sets'].tolist()\n",
" h265_bitrates = df_exploded['h265_bitrate'].tolist()\n",
" # # Group by index and concatenate the lists\n",
" # sets = df_exploded.groupby(df_exploded.index)[\"input_sets\"].agg(list)\n",
" \n",
" psnrs, psnrs_recon, mses, mses_recon = quality_metrics(sets[0::2]) # no feat\n",
" print_stats(psnrs, psnrs_recon, mses, mses_recon)\n",
" psnrs, psnrs_recon, mses, mses_recon = quality_metrics(sets[1::2]) # feat\n",
" print_stats(psnrs, psnrs_recon, mses, mses_recon)\n",
" \n",
" compressed_bitrates_nofeat = bitrate_metrics(sets[0::2], h265_bitrates[0::2]) # no feat\n",
" print_stats_bitrate(compressed_bitrates_nofeat)\n",
" compressed_bitrates_feat = bitrate_metrics(sets[1::2], h265_bitrates[0::2]) # feat\n",
" print_stats_bitrate(compressed_bitrates_feat)\n",
" \n",
" data = pd.DataFrame(columns=['Category', 'Value'])\n",
" \n",
" for i in range(0, len(sets), 8):\n",
" group = sets[i:i + 8]\n",
" data = get_feature_speed_change(group[1::2], data)\n",
" quartiles = data.groupby('Category')['Value'].quantile([0.25, 0.5, 0.75])\n",
" # sns.boxplot(y='Category', x='Value', orient='h', data=data, hue=\"Category\")\n",
"\n",
" print(quartiles)\n",
" \n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7233cd706bfddf4",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T18:55:15.182585912Z",
"start_time": "2024-01-30T18:55:15.051654937Z"
}
},
"outputs": [],
"source": [
"print('below') \n",
"print(below_bitrate)\n",
"get_stats(below_bitrate)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1107015591e85a27",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T18:57:10.663567453Z",
"start_time": "2024-01-30T18:57:10.121076609Z"
}
},
"outputs": [],
"source": [
"print('above1x')\n",
"get_stats(above1x_bitrate)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8759666074a6c24c",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T19:03:13.187207873Z",
"start_time": "2024-01-30T19:03:12.922157053Z"
}
},
"outputs": [],
"source": [
"print('above2x')\n",
"get_stats(above2x_bitrate)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "609509e82d45edb3",
"metadata": {
"ExecuteTime": {
"end_time": "2024-01-30T19:10:54.436307385Z",
"start_time": "2024-01-30T19:10:53.246953389Z"
}
},
"outputs": [],
"source": [
"\n",
"print('above3x')\n",
"get_stats(above3x_bitrate)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}