HuggingAGree/AcmeTrace
Acme Trace This repository hosts the public releases of Acme traces from the Shanghai AI Lab, encompassing workloads spanning from March 2023 to August 2023. We encourage anyone to use the traces for academic purposes, and if you had any questions, feel free to send an email to us, or file an issue on Github. Furthermore, we have conducted a thorough analysis of the Acme workloads, detailed in our NSDI '24 paper titled Characterization of Large Language Model Development in the… See the full description on the dataset page: https://huggingface.co/datasets/HuggingAGree/AcmeTrace.
0151
1{2 "cells": [3 {4 "cell_type": "markdown",5 "metadata": {},6 "source": [7 "#### Analysis"8 ]9 },10 {11 "cell_type": "code",12 "execution_count": 1,13 "metadata": {},14 "outputs": [],15 "source": [16 "from typing import List\n",17 "import os\n",18 "import pickle\n",19 "import squarify\n",20 "\n",21 "import numpy as np\n",22 "import pandas as pd\n",23 "import seaborn as sns\n",24 "import matplotlib\n",25 "import matplotlib.pyplot as plt\n",26 "import matplotlib.patches as mpatches\n",27 "from matplotlib.lines import Line2D\n",28 "\n",29 "SAVEPATH = \"./figure\"\n",30 "TRACEPATH = \"./data/job_trace\"\n",31 "PKLPATH = \"./data/utilization/util_pkl\"\n",32 "\n",33 "sns.set_style(\"ticks\")\n",34 "font = {\n",35 " \"font.family\": \"Roboto\",\n",36 " \"font.size\": 12,\n",37 "}\n",38 "sns.set_style(font)\n",39 "paper_rc = {\n",40 " \"lines.linewidth\": 3,\n",41 " \"lines.markersize\": 10,\n",42 "}\n",43 "sns.set_context(\"paper\", font_scale=2, rc=paper_rc)\n",44 "cmp = sns.color_palette(\"tab10\")\n",45 "\n",46 "\n",47 "def autolabel(rects, ax, prec=1):\n",48 " \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",49 " for rect in rects:\n",50 " height = rect.get_height()\n",51 " ax.annotate(\n",52 " f\"{height:.{prec}f}\",\n",53 " xy=(rect.get_x() + rect.get_width() / 2, height),\n",54 " xytext=(0, 3), # 3 points vertical offset\n",55 " textcoords=\"offset points\",\n",56 " ha=\"center\",\n",57 " va=\"bottom\",\n",58 " size=16,\n",59 " )\n",60 "\n",61 "\n",62 "def calculate_num_cdf_customized_xaxis(df: pd.DataFrame, x_axis: List, key: str):\n",63 " \"\"\"\n",64 " Calculate quantity percentile CDF with customized threshold of x-axis, y-axis: 0-100%,\n",65 " \"\"\"\n",66 " # print(\"Parsing\")\n",67 " data = df[[key]].copy()\n",68 " data.dropna(inplace=True)\n",69 "\n",70 " y = [len(data[data[key] <= x]) / len(data) * 100 for x in x_axis]\n",71 "\n",72 " return y\n",73 "\n",74 "\n",75 "def calculate_sum_cdf_customized_xaxis(df: pd.DataFrame, x_axis: List, key: str, key_to_time=None):\n",76 " \"\"\"\n",77 " Calculate sum CDF with customized threshold of x-axis, y-axis: 0-100%,\n",78 " \"\"\"\n",79 " if key_to_time is not None:\n",80 " data = df[[key, key_to_time]].copy()\n",81 " data[\"new\"] = data[key] * data[key_to_time]\n",82 " else:\n",83 " data = df[[key]].copy()\n",84 " data[\"new\"] = data[key]\n",85 " data.dropna(inplace=True)\n",86 " sum = data[\"new\"].sum()\n",87 "\n",88 " y = [data[data[key] <= x][\"new\"].sum() / sum * 100 for x in x_axis]\n",89 "\n",90 " return y\n",91 "\n",92 "\n",93 "if not os.path.exists(SAVEPATH):\n",94 " os.makedirs(SAVEPATH)\n",95 "\n",96 "\n",97 "data_seren = pd.read_csv(f\"{TRACEPATH}/trace_seren.csv\")\n",98 "data_kalos = pd.read_csv(f\"{TRACEPATH}/trace_kalos.csv\")\n",99 "data_philly = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/philly_trace.csv\")\n",100 "data_helios = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/helios_trace.csv\")\n",101 "data_pai = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/pai_trace.csv\")\n",102 "\n",103 "# A few further process\n",104 "data_pai.rename(columns={\"plan_cpu\": \"cpu_num\", \"plan_gpu\": \"gpu_num\", \"wait_time\": \"queue\", \"status\": \"state\"}, inplace=True)\n",105 "data_pai[[\"cpu_num\", \"gpu_num\"]] /= 100\n",106 "data_pai[\"state\"] = data_pai[\"state\"].map({\"Failed\": \"FAILED\"}) # Not suitable for final state analysis\n",107 "data_philly[\"state\"] = data_philly[\"state\"].map({\"Pass\": \"COMPLETED\", \"Failed\": \"FAILED\", \"Killed\": \"CANCELLED\"})"108 ]109 },110 {111 "cell_type": "markdown",112 "metadata": {},113 "source": [114 "#### CDF: GPU Job Duration & Utilization"115 ]116 },117 {118 "cell_type": "code",119 "execution_count": null,120 "metadata": {},121 "outputs": [],122 "source": [123 "x = [2**i for i in range(0, 22)]\n",124 "y_gpu_seren = calculate_num_cdf_customized_xaxis(data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",125 "y_gpu_kalos = calculate_num_cdf_customized_xaxis(data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",126 "y_gpu_philly = calculate_num_cdf_customized_xaxis(data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",127 "y_gpu_helios = calculate_num_cdf_customized_xaxis(data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",128 "y_gpu_pai = calculate_num_cdf_customized_xaxis(data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",129 "\n",130 "with open(f\"{PKLPATH}/util_gpu_seren.pkl\", \"rb\") as file:\n",131 " x1, y1, _, _, _, _, _, _, _, _ = pickle.load(file)\n",132 "\n",133 "with open(f\"{PKLPATH}/util_gpu_kalos.pkl\", \"rb\") as file:\n",134 " x4, y4, _, _ = pickle.load(file)\n",135 "\n",136 "with open(f\"{PKLPATH}/util_gpu_pai.pkl\", \"rb\") as file: # Collect via Antman\n",137 " x2, y2 = pickle.load(file)\n",138 "\n",139 "with open(f\"{PKLPATH}/util_gpu_philly.pkl\", \"rb\") as file:\n",140 " x3, y3 = pickle.load(file)"141 ]142 },143 {144 "cell_type": "code",145 "execution_count": null,146 "metadata": {},147 "outputs": [],148 "source": [149 "linestyles = [\"-\", \"--\", \":\", \":\", \":\"]\n",150 "grid_params = dict(width_ratios=[1, 1])\n",151 "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",152 "\n",153 "ax1.plot(x, y_gpu_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",154 "ax1.plot(x, y_gpu_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",155 "ax1.plot(x, y_gpu_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",156 "ax1.plot(x, y_gpu_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",157 "ax1.plot(x, y_gpu_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",158 "\n",159 "ax2.plot(x1, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",160 "ax2.plot(x4, y4, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",161 "ax2.plot(x2, y2, linestyles[2], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",162 "ax2.plot(x3, y3, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",163 "\n",164 "ax1.set_xlabel(f\"(a) GPU Job Duration (s)\")\n",165 "ax1.set_ylabel(f\"CDF (%)\")\n",166 "ax1.set_xscale(\"log\")\n",167 "ax1.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",168 "ax1.set_xlim(1, x[-1])\n",169 "ax1.set_ylim(-0.5, 100.8)\n",170 "ax1.grid(linestyle=\":\")\n",171 "\n",172 "ax2.set_xlabel(f\"(b) GPU Utilization (%)\")\n",173 "ax2.set_ylabel(f\"CDF (%)\")\n",174 "ax2.set_xlim(-0.8, 100.8)\n",175 "ax2.set_xticks([0, 25, 50, 75, 100])\n",176 "ax2.set_ylim(0, 100.8)\n",177 "ax2.grid(linestyle=\":\")\n",178 "\n",179 "handles, labels = ax1.get_legend_handles_labels()\n",180 "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.1, 1.145), loc=2, columnspacing=1.5, handletextpad=0.5)\n",181 "\n",182 "sns.despine()\n",183 "fig.savefig(f\"{SAVEPATH}/cdf_job_duration_util.pdf\", bbox_inches=\"tight\")"184 ]185 },186 {187 "cell_type": "markdown",188 "metadata": {},189 "source": [190 "#### CDF: GPU Number"191 ]192 },193 {194 "cell_type": "code",195 "execution_count": null,196 "metadata": {},197 "outputs": [],198 "source": [199 "x = [i for i in range(0, 1025)]\n",200 "y_gpu_seren = calculate_num_cdf_customized_xaxis(data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",201 "y_gpu_kalos = calculate_num_cdf_customized_xaxis(data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",202 "y_gpu_philly = calculate_num_cdf_customized_xaxis(data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",203 "y_gpu_helios = calculate_num_cdf_customized_xaxis(data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",204 "y_gpu_pai = calculate_num_cdf_customized_xaxis(data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",205 "\n",206 "y_gtime_seren = calculate_sum_cdf_customized_xaxis(\n",207 " data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",208 ")\n",209 "y_gtime_kalos = calculate_sum_cdf_customized_xaxis(\n",210 " data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",211 ")\n",212 "y_gtime_philly = calculate_sum_cdf_customized_xaxis(\n",213 " data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",214 ")\n",215 "y_gtime_helios = calculate_sum_cdf_customized_xaxis(\n",216 " data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",217 ")\n",218 "y_gtime_pai = calculate_sum_cdf_customized_xaxis(\n",219 " data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",220 ")"221 ]222 },223 {224 "cell_type": "code",225 "execution_count": null,226 "metadata": {},227 "outputs": [],228 "source": [229 "linestyles = [\"-\", \"--\", \":\", \":\", \":\"]\n",230 "grid_params = dict(width_ratios=[1, 1])\n",231 "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",232 "\n",233 "ax1.plot(x, y_gpu_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",234 "ax1.plot(x, y_gpu_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",235 "ax1.plot(x, y_gpu_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",236 "ax1.plot(x, y_gpu_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",237 "ax1.plot(x, y_gpu_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",238 "\n",239 "ax2.plot(x, y_gtime_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",240 "ax2.plot(x, y_gtime_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",241 "ax2.plot(x, y_gtime_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",242 "ax2.plot(x, y_gtime_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",243 "ax2.plot(x, y_gtime_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",244 "\n",245 "\n",246 "ax1.set_xlabel(f\"(a) Number of GPU\")\n",247 "ax1.set_ylabel(f\"CDF of Jobs (%)\")\n",248 "ax1.set_xscale(\"log\", base=2)\n",249 "ax1.set_xticks([2**i for i in range(0, 11, 2)])\n",250 "ax1.set_xticklabels(\n",251 " [2**i for i in range(0, 10, 2)]\n",252 " + [\n",253 " \"1024+\",\n",254 " ]\n",255 ")\n",256 "ax1.set_xlim(1, x[-1] + 1)\n",257 "ax1.set_ylim(-0.5, 100.8)\n",258 "ax1.grid(linestyle=\":\")\n",259 "\n",260 "ax2.set_xlabel(f\"(b) Number of GPU\")\n",261 "ax2.set_ylabel(f\"CDF of GPU Time (%)\")\n",262 "ax2.set_xscale(\"log\", base=2)\n",263 "ax2.set_xticks([2**i for i in range(0, 11, 2)])\n",264 "ax2.set_xticklabels(\n",265 " [2**i for i in range(0, 10, 2)]\n",266 " + [\n",267 " \"1024+\",\n",268 " ]\n",269 ")\n",270 "ax2.set_xlim(1, x[-1] + 50)\n",271 "ax2.set_ylim(-0.5, 100.8)\n",272 "ax2.grid(linestyle=\":\")\n",273 "\n",274 "handles, labels = ax1.get_legend_handles_labels()\n",275 "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.1, 1.145), loc=2, columnspacing=1.5, handletextpad=0.5)\n",276 "\n",277 "sns.despine()\n",278 "fig.savefig(f\"{SAVEPATH}/cdf_job_gpunum.pdf\", bbox_inches=\"tight\")"279 ]280 },281 {282 "cell_type": "markdown",283 "metadata": {},284 "source": [285 "#### Bar: Job Final State"286 ]287 },288 {289 "cell_type": "code",290 "execution_count": null,291 "metadata": {},292 "outputs": [],293 "source": [294 "df = pd.read_csv(\"./data/cluster_summary.csv\", index_col=\"id\")\n",295 "grid_params = dict(width_ratios=[1, 1])\n",296 "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",297 "\n",298 "x = np.arange(1, 3)\n",299 "width = 0.22\n",300 "p1 = ax1.bar(\n",301 " x - width,\n",302 " df.loc[[\"Seren\", \"Kalos\"], \"complete_rate_gpu\"] * 100,\n",303 " width,\n",304 " label=\"Completed\",\n",305 " alpha=0.8,\n",306 " linewidth=1,\n",307 " edgecolor=\"k\",\n",308 ")\n",309 "p2 = ax1.bar(\n",310 " x, df.loc[[\"Seren\", \"Kalos\"], \"cancel_rate_gpu\"] * 100, width, label=\"Canceled\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",311 ")\n",312 "p3 = ax1.bar(\n",313 " x + width, df.loc[[\"Seren\", \"Kalos\"], \"fail_rate_gpu\"] * 100, width, label=\"Failed\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",314 ")\n",315 "\n",316 "p4 = ax2.bar(\n",317 " x - width,\n",318 " df.loc[[\"Seren\", \"Kalos\"], \"complete_rate_gpu_time\"] * 100,\n",319 " width,\n",320 " label=\"Completed\",\n",321 " alpha=0.8,\n",322 " linewidth=1,\n",323 " edgecolor=\"k\",\n",324 ")\n",325 "p5 = ax2.bar(\n",326 " x, df.loc[[\"Seren\", \"Kalos\"], \"cancel_rate_gpu_time\"] * 100, width, label=\"Canceled\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",327 ")\n",328 "p6 = ax2.bar(\n",329 " x + width,\n",330 " df.loc[[\"Seren\", \"Kalos\"], \"fail_rate_gpu_time\"] * 100,\n",331 " width,\n",332 " label=\"Failed\",\n",333 " alpha=0.8,\n",334 " linewidth=1,\n",335 " edgecolor=\"k\",\n",336 ")\n",337 "\n",338 "autolabel(p1, ax1)\n",339 "autolabel(p2, ax1)\n",340 "autolabel(p3, ax1)\n",341 "autolabel(p4, ax2)\n",342 "autolabel(p5, ax2)\n",343 "autolabel(p6, ax2)\n",344 "\n",345 "ax1.set_xlabel(f\"(a) Job Count\")\n",346 "ax1.set_ylabel(f\"Fraction (%)\")\n",347 "ax1.set_xticks(x)\n",348 "ax1.set_xticklabels([\"Seren\", \"Kalos\"])\n",349 "ax1.set_xlim(0.5, 2.5)\n",350 "ax1.set_ylim(0, 100)\n",351 "ax1.grid(axis=\"y\", linestyle=\":\")\n",352 "\n",353 "ax2.set_xlabel(f\"(b) GPU Time\")\n",354 "ax2.set_ylabel(f\"Fraction (%)\")\n",355 "ax2.set_xticks(x)\n",356 "ax2.set_xticklabels([\"Seren\", \"Kalos\"])\n",357 "ax2.set_xlim(0.5, 2.5)\n",358 "ax2.set_ylim(0, 100)\n",359 "ax2.grid(axis=\"y\", linestyle=\":\")\n",360 "\n",361 "handles, labels = ax1.get_legend_handles_labels()\n",362 "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.18, 1.145), loc=2)\n",363 "\n",364 "sns.despine()\n",365 "fig.savefig(f\"{SAVEPATH}/bar_job_state.pdf\", bbox_inches=\"tight\")"366 ]367 },368 {369 "cell_type": "markdown",370 "metadata": {},371 "source": [372 "#### Treemap: Job Number Distribution"373 ]374 },375 {376 "cell_type": "code",377 "execution_count": null,378 "metadata": {},379 "outputs": [],380 "source": [381 "print(\"Processing Seren\")\n",382 "datas = data_seren[data_seren[\"gpu_num\"] > 0]\n",383 "\n",384 "job_type = [\"Eval\", \"Pretrain\", \"SFT\", \"MLLM\", \"Debug\", \"Other\"]\n",385 "df = pd.DataFrame(index=job_type, columns=[\"job_count\"]).fillna(0)\n",386 "df[\"job_count\"] = df.index.map(datas.groupby(\"type\").size()).astype(int)\n",387 "df[\"gtime\"] = df.index.map(datas.groupby(\"type\")[\"gpu_time\"].sum()).astype(int)\n",388 "\n",389 "total = df[\"job_count\"].sum()\n",390 "total_gtime = df[\"gtime\"].sum()\n",391 "\n",392 "df[\"count_percent\"] = df[\"job_count\"] / total * 100\n",393 "df[\"gtime_percent\"] = df[\"gtime\"] / total_gtime * 100\n",394 "\n",395 "# For plotting\n",396 "df[\"label\"] = [x + f\"\\n{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",397 "df[\"label_gtime\"] = [x + f\"\\n{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",398 "df[\"label_percent\"] = [f\"{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",399 "df[\"label_gtime_percent\"] = [f\"{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",400 "df_s = df.copy()\n",401 "\n",402 "print(\"Processing Kalos\")\n",403 "datak = data_kalos[data_kalos[\"gpu_num\"] > 0]\n",404 "\n",405 "job_type = [\"Eval\", \"Pretrain\", \"Debug\", \"Other\"]\n",406 "df = pd.DataFrame(index=job_type, columns=[\"job_count\"]).fillna(0)\n",407 "df[\"job_count\"] = df.index.map(datak.groupby(\"type\").size()).astype(int)\n",408 "df[\"gtime\"] = df.index.map(datak.groupby(\"type\")[\"gpu_time\"].sum()).astype(int)\n",409 "\n",410 "\n",411 "total = df[\"job_count\"].sum()\n",412 "total_gtime = df[\"gtime\"].sum()\n",413 "\n",414 "df[\"count_percent\"] = df[\"job_count\"] / total * 100\n",415 "df[\"gtime_percent\"] = df[\"gtime\"] / total_gtime * 100\n",416 "\n",417 "# For plotting\n",418 "df[\"label\"] = [x + f\"\\n{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",419 "df[\"label_gtime\"] = [x + f\"\\n{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",420 "df[\"label_percent\"] = [f\"{df.at[x, 'count_percent']:.1f}\\n%\" for x in list(df.index)]\n",421 "df[\"label_gtime_percent\"] = [f\"{df.at[x, 'gtime_percent']:.1f}\\n%\" for x in list(df.index)]\n",422 "df_k = df.copy()\n",423 "\n",424 "# For plotting\n",425 "df_k.at[\"Pretrain\", \"label_gtime_percent\"] = df_k.at[\"Pretrain\", \"label_gtime\"]\n",426 "df_k.at[\"Eval\", \"label_percent\"] = df_k.at[\"Eval\", \"label\"]\n",427 "df_k.at[\"Other\", \"label_percent\"] = \"\"\n",428 "df_k.at[\"Eval\", \"label_gtime_percent\"] = \" \"\n",429 "\n",430 "df_s.at[\"Pretrain\", \"label_gtime_percent\"] = df_s.at[\"Pretrain\", \"label_gtime\"]\n",431 "df_s.at[\"Eval\", \"label_percent\"] = df_s.at[\"Eval\", \"label\"]\n",432 "df_s.at[\"SFT\", \"label_percent\"] = df_s.at[\"SFT\", \"label\"]\n",433 "df_s.at[\"Other\", \"label_percent\"] = df_s.at[\"Other\", \"label\"]\n",434 "df_s.at[\"Pretrain\", \"label_percent\"] = \" \"\n",435 "df_s.at[\"Debug\", \"label_gtime_percent\"] = df_s.at[\"Debug\", \"label_gtime_percent\"].replace(\"%\", \"\\n%\")\n",436 "\n",437 "\n",438 "cmp_treemap = sns.color_palette(\"pastel\")\n",439 "label = df_s.index.to_list()\n",440 "df_s[\"color\"] = cmp_treemap[: len(df_s)]"441 ]442 },443 {444 "cell_type": "code",445 "execution_count": null,446 "metadata": {},447 "outputs": [],448 "source": [449 "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 4.2))\n",450 "FONT = 15\n",451 "\n",452 "###### Fig 1 ######\n",453 "df_s.sort_values(by=\"count_percent\", ascending=False, inplace=True)\n",454 "squarify.plot(\n",455 " ax=ax1,\n",456 " sizes=list(df_s[\"job_count\"].values),\n",457 " label=df_s[\"label_percent\"],\n",458 " text_kwargs={\"fontsize\": FONT},\n",459 " color=df_s[\"color\"],\n",460 " bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",461 ")\n",462 "\n",463 "\n",464 "handles, labels = ax1.get_legend_handles_labels()\n",465 "handles_new = [handles[0], handles[-1], handles[1], handles[3], handles[4], handles[2]]\n",466 "fig.legend(\n",467 " handles=handles_new, labels=label, ncols=6, bbox_to_anchor=(0.0, 1.135), loc=2, columnspacing=0.82, handletextpad=0.2\n",468 ")\n",469 "\n",470 "###### Fig 2 ######\n",471 "df_s.sort_values(by=\"gtime\", ascending=False, inplace=True)\n",472 "squarify.plot(\n",473 " ax=ax2,\n",474 " sizes=list(df_s[\"gtime\"].values),\n",475 " label=df_s[\"label_gtime_percent\"],\n",476 " text_kwargs={\"fontsize\": FONT},\n",477 " color=df_s[\"color\"],\n",478 " bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",479 ")\n",480 "\n",481 "plt.tick_params(axis=\"both\", which=\"both\", bottom=False, top=False, left=False, right=False)\n",482 "\n",483 "ax1.set_xlabel(f\"(a) Job Count\", fontsize=16)\n",484 "ax2.set_xlabel(f\"(b) GPU Time\", fontsize=16)\n",485 "\n",486 "\n",487 "###### Fig 3 ######\n",488 "df_k.sort_values(by=\"count_percent\", ascending=False, inplace=True)\n",489 "df_k[\"color\"] = [df_s[\"color\"][job_name] for job_name in df_k.index]\n",490 "\n",491 "squarify.plot(\n",492 " ax=ax3,\n",493 " sizes=list(df_k[\"job_count\"].values),\n",494 " label=df_k[\"label_percent\"],\n",495 " text_kwargs={\"fontsize\": FONT},\n",496 " color=df_k[\"color\"],\n",497 " bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",498 ")\n",499 "\n",500 "###### Fig 4 ######\n",501 "df_k.sort_values(by=\"gtime\", ascending=False, inplace=True)\n",502 "squarify.plot(\n",503 " ax=ax4,\n",504 " sizes=list(df_k[\"gtime\"].values),\n",505 " label=df_k[\"label_gtime_percent\"],\n",506 " text_kwargs={\"fontsize\": FONT},\n",507 " color=df_k[\"color\"],\n",508 " bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",509 ")\n",510 "\n",511 "ax1.annotate(\n",512 " df_s.at[\"Pretrain\", \"label\"].split(\"\\n\")[1],\n",513 " xy=(97, 96),\n",514 " xytext=(90, 70),\n",515 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",516 " color=\"black\",\n",517 " fontsize=15,\n",518 ")\n",519 "\n",520 "ax3.annotate(\n",521 " df_k.at[\"Other\", \"label\"].split(\"\\n\")[1],\n",522 " xy=(98.5, 92),\n",523 " xytext=(80, 80),\n",524 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",525 " color=\"black\",\n",526 " fontsize=15,\n",527 ")\n",528 "\n",529 "ax4.annotate(\n",530 " df_k.at[\"Eval\", \"label_gtime\"].split(\"\\n\")[1],\n",531 " xy=(98.5, 93),\n",532 " xytext=(80, 80),\n",533 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",534 " color=\"black\",\n",535 " fontsize=15,\n",536 ")\n",537 "\n",538 "plt.tick_params(axis=\"both\", which=\"both\", bottom=False, top=False, left=False, right=False)\n",539 "\n",540 "ax3.set_xlabel(f\"(c) Job Count\", fontsize=16)\n",541 "ax4.set_xlabel(f\"(d) GPU Time\", fontsize=16, labelpad=8)\n",542 "\n",543 "ax1.set_xticks([])\n",544 "ax1.set_yticks([])\n",545 "ax2.set_xticks([])\n",546 "ax2.set_yticks([])\n",547 "ax3.set_xticks([])\n",548 "ax3.set_yticks([])\n",549 "ax4.set_xticks([])\n",550 "ax4.set_yticks([])\n",551 "\n",552 "ax1.text(0.015, 0.03, \"Seren\", transform=ax1.transAxes, size=18, fontweight=\"bold\")\n",553 "ax2.text(0.02, 0.03, \"Seren\", transform=ax2.transAxes, size=18, fontweight=\"bold\")\n",554 "ax3.text(0.02, 0.03, \"Kalos\", transform=ax3.transAxes, size=18, fontweight=\"bold\")\n",555 "ax4.text(0.02, 0.03, \"Kalos\", transform=ax4.transAxes, size=18, fontweight=\"bold\")\n",556 "fig.savefig(f\"{SAVEPATH}/treemap_job_dist.pdf\", bbox_inches=\"tight\")"557 ]558 },559 {560 "cell_type": "markdown",561 "metadata": {},562 "source": [563 "#### CDF: Duration and Queuing Delay of Different Type"564 ]565 },566 {567 "cell_type": "code",568 "execution_count": null,569 "metadata": {},570 "outputs": [],571 "source": [572 "\"\"\"\n",573 "(a) Seren Duration (b) Seren Queuing (c) Kalos Duration (d) Kalos Queuing\n",574 "\"\"\"\n",575 "\n",576 "# Duration part\n",577 "x = [2**i for i in range(0, 22)]\n",578 "y_gpu_seren_other = calculate_num_cdf_customized_xaxis(\n",579 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Other\")], x_axis=x, key=\"duration\"\n",580 ")\n",581 "y_gpu_seren_debug = calculate_num_cdf_customized_xaxis(\n",582 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Debug\")], x_axis=x, key=\"duration\"\n",583 ")\n",584 "y_gpu_seren_pretrain = calculate_num_cdf_customized_xaxis(\n",585 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Pretrain\")], x_axis=x, key=\"duration\"\n",586 ")\n",587 "y_gpu_seren_eval = calculate_num_cdf_customized_xaxis(\n",588 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Eval\")], x_axis=x, key=\"duration\"\n",589 ")\n",590 "y_gpu_seren_tuning = calculate_num_cdf_customized_xaxis(\n",591 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"SFT\")], x_axis=x, key=\"duration\"\n",592 ")\n",593 "y_gpu_seren_mllm = calculate_num_cdf_customized_xaxis(\n",594 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"MLLM\")], x_axis=x, key=\"duration\"\n",595 ")\n",596 "\n",597 "y_gpu_kalos_other = calculate_num_cdf_customized_xaxis(\n",598 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Other\")], x_axis=x, key=\"duration\"\n",599 ")\n",600 "y_gpu_kalos_debug = calculate_num_cdf_customized_xaxis(\n",601 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Debug\")], x_axis=x, key=\"duration\"\n",602 ")\n",603 "y_gpu_kalos_pretrain = calculate_num_cdf_customized_xaxis(\n",604 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Pretrain\")], x_axis=x, key=\"duration\"\n",605 ")\n",606 "y_gpu_kalos_eval = calculate_num_cdf_customized_xaxis(\n",607 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Eval\")], x_axis=x, key=\"duration\"\n",608 ")\n",609 "y_gpu_kalos_tuning = calculate_num_cdf_customized_xaxis(\n",610 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"SFT\")], x_axis=x, key=\"duration\"\n",611 ")\n",612 "\n",613 "# Queuing part\n",614 "x2 = [2**i for i in range(0, 16)]\n",615 "y_que_s_other = calculate_num_cdf_customized_xaxis(\n",616 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Other\")], x_axis=x2, key=\"queue\"\n",617 ")\n",618 "y_que_s_debug = calculate_num_cdf_customized_xaxis(\n",619 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Debug\")], x_axis=x2, key=\"queue\"\n",620 ")\n",621 "y_que_s_pretrain = calculate_num_cdf_customized_xaxis(\n",622 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Pretrain\")], x_axis=x2, key=\"queue\"\n",623 ")\n",624 "y_que_s_eval = calculate_num_cdf_customized_xaxis(\n",625 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Eval\")], x_axis=x2, key=\"queue\"\n",626 ")\n",627 "y_que_s_tuning = calculate_num_cdf_customized_xaxis(\n",628 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"SFT\")], x_axis=x2, key=\"queue\"\n",629 ")\n",630 "y_que_s_mllm = calculate_num_cdf_customized_xaxis(\n",631 " data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"MLLM\")], x_axis=x2, key=\"queue\"\n",632 ")\n",633 "\n",634 "y_que_ali_other = calculate_num_cdf_customized_xaxis(\n",635 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Other\")], x_axis=x2, key=\"queue\"\n",636 ")\n",637 "y_que_ali_debug = calculate_num_cdf_customized_xaxis(\n",638 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Debug\")], x_axis=x2, key=\"queue\"\n",639 ")\n",640 "y_que_ali_pretrain = calculate_num_cdf_customized_xaxis(\n",641 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Pretrain\")], x_axis=x2, key=\"queue\"\n",642 ")\n",643 "y_que_ali_eval = calculate_num_cdf_customized_xaxis(\n",644 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Eval\")], x_axis=x2, key=\"queue\"\n",645 ")\n",646 "y_que_ali_tuning = calculate_num_cdf_customized_xaxis(\n",647 " data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"SFT\")], x_axis=x2, key=\"queue\"\n",648 ")"649 ]650 },651 {652 "cell_type": "code",653 "execution_count": null,654 "metadata": {},655 "outputs": [],656 "source": [657 "linestyles = [\"--\", \"-.\", \":\", \"--\", \"-.\", \":\"]\n",658 "grid_params = dict(width_ratios=[1, 1])\n",659 "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 7))\n",660 "\n",661 "# (a) Seren Duration\n",662 "ax1.plot(x, y_gpu_seren_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",663 "ax1.plot(x, y_gpu_seren_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",664 "ax1.plot(x, y_gpu_seren_tuning, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"SFT\")\n",665 "ax1.plot(x, y_gpu_seren_mllm, linestyles[0], linewidth=3, alpha=0.9, color=cmp[3], label=\"MLLM\")\n",666 "ax1.plot(x, y_gpu_seren_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",667 "ax1.plot(x, y_gpu_seren_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",668 "\n",669 "\n",670 "# (b) Seren Queuing\n",671 "ax2.plot(x2, y_que_s_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",672 "ax2.plot(x2, y_que_s_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",673 "ax2.plot(x2, y_que_s_tuning, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"SFT\")\n",674 "ax2.plot(x2, y_que_s_mllm, linestyles[0], linewidth=3, alpha=0.9, color=cmp[3], label=\"MLLM\")\n",675 "ax2.plot(x2, y_que_s_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",676 "ax2.plot(x2, y_que_s_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",677 "\n",678 "# (c) Kalos Duration\n",679 "ax3.plot(x, y_gpu_kalos_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",680 "ax3.plot(x, y_gpu_kalos_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",681 "ax3.plot(x, y_gpu_kalos_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",682 "ax3.plot(x, y_gpu_kalos_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",683 "\n",684 "\n",685 "# (d) Kalos Queuing\n",686 "ax4.plot(x2, y_que_ali_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",687 "ax4.plot(x2, y_que_ali_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",688 "ax4.plot(x2, y_que_ali_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",689 "ax4.plot(x2, y_que_ali_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",690 "\n",691 "ax1.set_xlabel(f\"(a) Job Duration (s)\")\n",692 "ax1.set_ylabel(f\"CDF (%)\")\n",693 "ax1.set_xscale(\"log\")\n",694 "ax1.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",695 "ax1.set_xlim(1, x[-1])\n",696 "ax1.set_ylim(-0.5, 100.8)\n",697 "handles, labels = ax1.get_legend_handles_labels()\n",698 "fig.legend(handles=handles, labels=labels, ncols=6, bbox_to_anchor=(-0.01, 1.08), loc=2, columnspacing=0.9, handletextpad=0.2)\n",699 "ax1.grid(linestyle=\":\")\n",700 "\n",701 "ax2.set_xlabel(f\"(b) Job Queuing Delay (s)\")\n",702 "ax2.set_ylabel(f\"CDF (%)\")\n",703 "ax2.set_xscale(\"log\")\n",704 "ax2.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4])\n",705 "ax2.set_xlim(1, x2[-1])\n",706 "ax2.set_ylim(-0.5, 100.8)\n",707 "ax2.grid(linestyle=\":\")\n",708 "\n",709 "ax3.set_xlabel(f\"(c) Job Duration (s)\")\n",710 "ax3.set_ylabel(f\"CDF (%)\")\n",711 "ax3.set_xscale(\"log\")\n",712 "ax3.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",713 "ax3.set_xlim(1, x[-1])\n",714 "ax3.set_ylim(-0.5, 100.8)\n",715 "ax3.grid(linestyle=\":\")\n",716 "\n",717 "ax4.set_xlabel(f\"(d) Job Queuing Delay (s)\")\n",718 "ax4.set_ylabel(f\"CDF (%)\")\n",719 "ax4.set_xscale(\"log\")\n",720 "ax4.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4])\n",721 "ax4.set_xlim(1, x2[-1])\n",722 "ax4.set_ylim(-0.5, 100.8)\n",723 "ax4.grid(linestyle=\":\")\n",724 "\n",725 "# 1 hour and 1 day\n",726 "ax1.axvline(x=3600, ls=\"--\", alpha=0.6, c=\"gray\", ymax=0.94, lw=1.5)\n",727 "ax1.axvline(x=3600 * 24, ls=\"--\", alpha=0.9, c=\"gray\", ymax=0.94, lw=1.5)\n",728 "ax3.axvline(x=3600, ls=\"--\", alpha=0.6, c=\"gray\", ymax=0.94, lw=1.5)\n",729 "ax3.axvline(x=3600 * 24, ls=\"--\", alpha=0.9, c=\"gray\", ymax=0.94, lw=1.5)\n",730 "\n",731 "sns.despine()\n",732 "ax1.text(0.78, 0.03, \"Seren\", transform=ax1.transAxes, size=20, fontweight=\"bold\")\n",733 "ax2.text(0.78, 0.03, \"Seren\", transform=ax2.transAxes, size=20, fontweight=\"bold\")\n",734 "ax3.text(0.78, 0.03, \"Kalos\", transform=ax3.transAxes, size=20, fontweight=\"bold\")\n",735 "ax4.text(0.78, 0.03, \"Kalos\", transform=ax4.transAxes, size=20, fontweight=\"bold\")\n",736 "\n",737 "fig.savefig(f\"{SAVEPATH}/cdf_job_duration_queue.pdf\", bbox_inches=\"tight\")"738 ]739 },740 {741 "cell_type": "markdown",742 "metadata": {},743 "source": [744 "#### Box Plot: Request GPU number Different Type"745 ]746 },747 {748 "cell_type": "code",749 "execution_count": null,750 "metadata": {},751 "outputs": [],752 "source": [753 "cmap = sns.color_palette(\"pastel\")\n",754 "fig, (ax1, ax2) = plt.subplots(\n",755 " ncols=2,\n",756 " nrows=1,\n",757 " gridspec_kw={\"width_ratios\": [4.2, 3]},\n",758 " constrained_layout=True,\n",759 " figsize=(9, 3.75),\n",760 ")\n",761 "\n",762 "############ Fig 1 ############\n",763 "data_seren.sort_values(by=\"gpu_num\", ascending=False, inplace=True)\n",764 "data_seren[\"type\"].replace(\"SFT\", \"SFT\", inplace=True)\n",765 "\n",766 "x_ticks = [\n",767 " \"Eval\",\n",768 " \"Pretrain\",\n",769 " \"SFT\",\n",770 " \"MLLM\",\n",771 " \"Debug\",\n",772 " \"Other\",\n",773 "]\n",774 "\n",775 "flierprops = dict(marker=\".\", markerfacecolor=\"k\", markersize=2, linestyle=\"none\")\n",776 "sns.boxplot(\n",777 " x=\"type\",\n",778 " y=\"gpu_num\",\n",779 " data=data_seren,\n",780 " flierprops=flierprops,\n",781 " width=0.6,\n",782 " linewidth=2.2,\n",783 " saturation=2,\n",784 " palette=cmap,\n",785 " ax=ax1,\n",786 " order=x_ticks,\n",787 " boxprops=dict(alpha=1),\n",788 ")\n",789 "sns.color_palette(\"tab10\")\n",790 "ax1.set_xlabel(\"(a) Seren\")\n",791 "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=0)\n",792 "ax1.set_ylabel(f\"Number of GPUs\")\n",793 "ax1.set_yscale(\"log\")\n",794 "ax1.grid(axis=\"y\", linestyle=\":\")\n",795 "\n",796 "\n",797 "############ Fig 2 ############\n",798 "data_kalos.sort_values(by=\"gpu_num\", ascending=False, inplace=True)\n",799 "data_kalos = data_kalos[data_kalos[\"type\"] != \"SFT\"]\n",800 "x_ticks_k = [\n",801 " \"Eval\",\n",802 " \"Pretrain\",\n",803 " \"Debug\",\n",804 " \"Other\",\n",805 "]\n",806 "my_pal = [cmap[0], cmap[1], cmap[4], cmap[5]]\n",807 "\n",808 "flierprops = dict(marker=\".\", markerfacecolor=\"k\", markersize=3, linestyle=\"none\")\n",809 "sns.boxplot(\n",810 " x=\"type\",\n",811 " y=\"gpu_num\",\n",812 " data=data_kalos,\n",813 " flierprops=flierprops,\n",814 " width=0.6,\n",815 " linewidth=2.2,\n",816 " saturation=2,\n",817 " palette=my_pal,\n",818 " ax=ax2,\n",819 " order=x_ticks_k,\n",820 " boxprops=dict(alpha=1),\n",821 ")\n",822 "sns.color_palette(\"tab10\")\n",823 "ax2.set_xlabel(\"(b) Kalos\")\n",824 "ax2.set_ylabel(None)\n",825 "ax2.set_xticklabels(ax2.get_xticklabels(), rotation=0)\n",826 "ax2.set_yscale(\"log\")\n",827 "ax2.grid(axis=\"y\", linestyle=\":\")\n",828 "\n",829 "sns.despine()\n",830 "fig.savefig(f\"{SAVEPATH}/box_gpu_num.pdf\", bbox_inches=\"tight\")"831 ]832 },833 {834 "cell_type": "markdown",835 "metadata": {},836 "source": [837 "#### CDF: Resource Utilization"838 ]839 },840 {841 "cell_type": "code",842 "execution_count": null,843 "metadata": {},844 "outputs": [],845 "source": [846 "with open(f\"{PKLPATH}/util_gpu_seren.pkl\", \"rb\") as file:\n",847 " _, _, x2, y2, x3, y3, x4, y4, x5, y5 = pickle.load(file)\n",848 "with open(f\"{PKLPATH}/util_gpu_kalos_full.pkl\", \"rb\") as file:\n",849 " _, _, x2_k, y2_k, x3_k, y3_k, x4_k, y4_k, x5_k, y5_k = pickle.load(file)\n",850 "with open(f\"{PKLPATH}/util_cpu_mem_seren.pkl\", \"rb\") as file:\n",851 " x6, y6, x7, y7 = pickle.load(file)\n",852 "with open(f\"{PKLPATH}/util_cpu_mem_kalos.pkl\", \"rb\") as file:\n",853 " x6_k, y6_k, x7_k, y7_k = pickle.load(file)\n",854 "with open(f\"{PKLPATH}/ib_seren.pkl\", \"rb\") as file:\n",855 " x8, y8, x9, y9 = pickle.load(file)\n",856 "\n",857 "x8 = x8 / x8.max() * 100\n",858 "x9 = x9 / x9.max() * 100\n",859 "\n",860 "linestyles = [\"--\", \":\", \"--\", \"-.\", \":\"]\n",861 "grid_params = dict(width_ratios=[1, 1])\n",862 "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 7))\n",863 "\n",864 "############ Fig 1: SM, Occupancy ############\n",865 "ax1.plot(x3, y3, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren SM Activity\")\n",866 "ax1.plot(x5, y5, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren Occupancy\")\n",867 "ax1.plot(x3_k, y3_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos SM Activity\")\n",868 "ax1.plot(x5_k, y5_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos Occupancy\")\n",869 "\n",870 "############ Fig 2: CPU mem usage, GPU mem usage ############\n",871 "ax2.plot(x7, y7, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren CPU Mem\")\n",872 "ax2.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Mem\")\n",873 "ax2.plot(x7_k, y7_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos CPU Mem\")\n",874 "ax2.plot(x2_k, y2_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Mem\")\n",875 "\n",876 "############ Fig 3: CPU util ############\n",877 "ax3.plot(x6, y6, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",878 "ax3.plot(x6_k, y6_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",879 "\n",880 "############ Fig 4: IB send, receive ############\n",881 "ax4.plot(x8, y8, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"IB Send\")\n",882 "ax4.plot(x9, y9, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"IB Receive\")\n",883 "\n",884 "ax1.set_xlabel(f\"(a) GPU DCGM Metric (%)\")\n",885 "ax1.set_ylabel(f\"CDF (%)\")\n",886 "ax1.set_xlim(-0.8, 100.8)\n",887 "ax1.set_ylim(0, 100.8)\n",888 "ax1.set_xticks([0, 25, 50, 75, 100])\n",889 "ax1.grid(linestyle=\":\")\n",890 "\n",891 "ax2.set_xlabel(f\"(b) Memory Footprint (%)\")\n",892 "ax2.set_ylabel(f\"CDF (%)\")\n",893 "ax2.set_xlim(-0.8, 100.8)\n",894 "ax2.set_xticks([0, 25, 50, 75, 100])\n",895 "ax2.set_ylim(0, 100.8)\n",896 "ax2.grid(linestyle=\":\")\n",897 "\n",898 "ax3.set_xlabel(f\"(c) CPU Utilization (%)\")\n",899 "ax3.set_ylabel(f\"CDF (%)\")\n",900 "ax3.set_xlim(-0.8, 100.8)\n",901 "ax3.set_xticks([0, 25, 50, 75, 100])\n",902 "ax3.set_ylim(0, 100.8)\n",903 "ax3.legend(loc=\"lower right\")\n",904 "ax3.grid(linestyle=\":\")\n",905 "\n",906 "ax4.set_xlabel(f\"(d) Network (%)\")\n",907 "ax4.set_ylabel(f\"CDF (%)\")\n",908 "ax4.set_xlim(-0.8, 100.8)\n",909 "ax4.set_xticks([0, 25, 50, 75, 100])\n",910 "ax4.set_ylim(0, 100.8)\n",911 "ax4.legend(loc=\"lower right\")\n",912 "ax4.grid(linestyle=\":\")\n",913 "sns.despine()\n",914 "\n",915 "\n",916 "S = mpatches.Patch(facecolor=cmp[0], alpha=0.9)\n",917 "K = mpatches.Patch(facecolor=cmp[1], alpha=0.9)\n",918 "A = (Line2D([0], [0], color=\"black\", lw=3, ls=\"--\"),)\n",919 "B = (Line2D([0], [0], color=\"black\", lw=3, ls=\":\"),)\n",920 "\n",921 "legend1 = ax1.legend([S, K], [\"Seren\", \"Kalos\"], bbox_to_anchor=(0.5, 0.62), loc=2, ncol=1, fontsize=17, frameon=False)\n",922 "\n",923 "ax1.add_artist(legend1)\n",924 "\n",925 "ax1.legend([A, B], [\"SM Activity\", \"TC Activity\"], bbox_to_anchor=(0.3, 0.36), loc=2, ncol=1)\n",926 "\n",927 "ax2.legend([A, B], [\"CPU Memory\", \"GPU Memory\"], bbox_to_anchor=(0.25, 0.36), loc=2, ncol=1)\n",928 "\n",929 "fig.savefig(f\"{SAVEPATH}/cdf_resource_util.pdf\", bbox_inches=\"tight\")"930 ]931 },932 {933 "cell_type": "markdown",934 "metadata": {},935 "source": [936 "#### CDF: Temperature"937 ]938 },939 {940 "cell_type": "code",941 "execution_count": null,942 "metadata": {},943 "outputs": [],944 "source": [945 "# We use August data for GPU temperature and power\n",946 "with open(f\"{PKLPATH}/gpu_temp_seren.pkl\", \"rb\") as file:\n",947 " x, y1, x2, y2 = pickle.load(file)\n",948 "with open(f\"{PKLPATH}/gpu_temp_kalos.pkl\", \"rb\") as file:\n",949 " x1_k, y1_k, x2_k, y2_k = pickle.load(file)\n",950 "with open(f\"{PKLPATH}/gpu_power_seren.pkl\", \"rb\") as file:\n",951 " x3, y3 = pickle.load(file)\n",952 "with open(f\"{PKLPATH}/gpu_power_kalos.pkl\", \"rb\") as file:\n",953 " x3_k, y3_k = pickle.load(file)\n",954 "\n",955 "linestyles = [\"-\", \":\", \":\", \"-\"]\n",956 "fig, ax1 = plt.subplots(ncols=1, nrows=1, constrained_layout=True, figsize=(5, 3.75))\n",957 "\n",958 "############ Fig 1: Temperature ############\n",959 "ax1.plot(x, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Temp\")\n",960 "ax1.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Mem Temp\")\n",961 "ax1.plot(x1_k, y1_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Temp\")\n",962 "ax1.plot(x2_k, y2_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Mem Temp\")\n",963 "\n",964 "ax1.set_xlabel(f\"Temperature (°C)\")\n",965 "ax1.set_ylabel(f\"CDF (%)\")\n",966 "ax1.set_xlim(20, 85)\n",967 "ax1.set_ylim(0, 100.8)\n",968 "ax1.grid(linestyle=\":\")\n",969 "\n",970 "S = mpatches.Patch(facecolor=cmp[0], alpha=0.9)\n",971 "K = mpatches.Patch(facecolor=cmp[1], alpha=0.9)\n",972 "A = (Line2D([0], [0], color=\"black\", lw=3, ls=\"-\"),)\n",973 "B = (Line2D([0], [0], color=\"black\", lw=3, ls=\":\"),)\n",974 "\n",975 "legend1 = ax1.legend([S, K], [\"Seren\", \"Kalos\"], bbox_to_anchor=(0.6, 0.62), loc=2, ncol=1, fontsize=17, frameon=False)\n",976 "\n",977 "ax1.add_artist(legend1)\n",978 "\n",979 "ax1.legend(\n",980 " [A, B],\n",981 " [\"GPU Temp.\", \"GMem Temp.\"],\n",982 " bbox_to_anchor=(0.4, 0.32),\n",983 " loc=2,\n",984 " ncol=1,\n",985 " fontsize=17,\n",986 ")\n",987 "\n",988 "sns.despine()\n",989 "fig.savefig(f\"{SAVEPATH}/cdf_temperature.pdf\", bbox_inches=\"tight\")"990 ]991 },992 {993 "cell_type": "markdown",994 "metadata": {},995 "source": [996 "#### CDF: Power"997 ]998 },999 {1000 "cell_type": "code",1001 "execution_count": null,1002 "metadata": {},1003 "outputs": [],1004 "source": [1005 "with open(f\"{PKLPATH}/server_power.pkl\", \"rb\") as file:\n",1006 " x1, y1, x2, y2 = pickle.load(file)\n",1007 "with open(f\"{PKLPATH}/gpu_power_seren.pkl\", \"rb\") as file:\n",1008 " x3, y3 = pickle.load(file)\n",1009 "with open(f\"{PKLPATH}/gpu_power_kalos.pkl\", \"rb\") as file:\n",1010 " x3_k, y3_k = pickle.load(file)"1011 ]1012 },1013 {1014 "cell_type": "code",1015 "execution_count": null,1016 "metadata": {},1017 "outputs": [],1018 "source": [1019 "linestyles = [\"--\", \":\", \":\", \"-\"]\n",1020 "grid_params = dict(width_ratios=[1, 1])\n",1021 "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",1022 "\n",1023 "############ Fig 1: GPU power ############\n",1024 "ax1.plot(x3, y3, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",1025 "ax1.plot(x3_k, y3_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",1026 "ax1.axvline(x=400, ls=\"--\", alpha=0.6, c=\"gray\", ymax=100, lw=1.5)\n",1027 "ax1.annotate(\n",1028 " \"A100 TDP\",\n",1029 " xy=(400, 33),\n",1030 " xytext=(420, 20),\n",1031 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",1032 " color=\"black\",\n",1033 " fontsize=16,\n",1034 ")\n",1035 "ax1.annotate(\n",1036 " \"Max=600\",\n",1037 " xy=(600, 100),\n",1038 " xytext=(430, 85),\n",1039 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",1040 " color=\"black\",\n",1041 " fontsize=16,\n",1042 ")\n",1043 "\n",1044 "############ Fig 2: Server power ############\n",1045 "ax2.plot(x1, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"GPU Node\")\n",1046 "ax2.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"CPU Node\")\n",1047 "ax2.annotate(\n",1048 " \"Max=960\",\n",1049 " xy=(960, 100),\n",1050 " xytext=(1200, 90),\n",1051 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",1052 " color=\"black\",\n",1053 " fontsize=16,\n",1054 ")\n",1055 "ax2.annotate(\n",1056 " \"Max=6550\",\n",1057 " xy=(6550, 100),\n",1058 " xytext=(4500, 70),\n",1059 " arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",1060 " color=\"black\",\n",1061 " fontsize=16,\n",1062 ")\n",1063 "\n",1064 "ax1.set_xlabel(f\"(a) GPU Power (W)\")\n",1065 "ax1.set_ylabel(f\"CDF (%)\")\n",1066 "ax1.set_xlim(-0.8, 610)\n",1067 "ax1.set_ylim(0, 100.8)\n",1068 "ax1.legend()\n",1069 "ax1.grid(linestyle=\":\")\n",1070 "ax1.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([60]))\n",1071 "ax1.xaxis.set_minor_formatter(matplotlib.ticker.FixedFormatter([60]))\n",1072 "ax1.tick_params(axis=\"x\", which=\"minor\", labelsize=15)\n",1073 "\n",1074 "ax2.set_xlabel(f\"(b) Server Power in Seren (W)\")\n",1075 "ax2.set_ylabel(f\"CDF (%)\")\n",1076 "ax2.set_xlim(-0.8, x1.max())\n",1077 "ax2.set_ylim(0, 100.8)\n",1078 "ax2.legend(loc=\"lower right\")\n",1079 "ax2.grid(linestyle=\":\")\n",1080 "ax2.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([520]))\n",1081 "ax2.xaxis.set_minor_formatter(matplotlib.ticker.FixedFormatter([520]))\n",1082 "ax2.tick_params(axis=\"x\", which=\"minor\", labelsize=15)\n",1083 "sns.despine()\n",1084 "\n",1085 "fig.savefig(f\"{SAVEPATH}/cdf_power.pdf\", bbox_inches=\"tight\")"1086 ]1087 }1088 ],1089 "metadata": {1090 "kernelspec": {1091 "display_name": "base",1092 "language": "python",1093 "name": "python3"1094 },1095 "language_info": {1096 "codemirror_mode": {1097 "name": "ipython",1098 "version": 31099 },1100 "file_extension": ".py",1101 "mimetype": "text/x-python",1102 "name": "python",1103 "nbconvert_exporter": "python",1104 "pygments_lexer": "ipython3",1105 "version": "3.9.16"1106 },1107 "orig_nbformat": 41108 },1109 "nbformat": 4,1110 "nbformat_minor": 21111}1112