echodict/llama.cpp

1#!/usr/bin/env python32 3import argparse4import csv5import heapq6import json7import logging8import os9import sqlite310import sys11from collections.abc import Iterator, Sequence12from glob import glob13from typing import Any, Optional, Union14 15try:16    import git17    from tabulate import tabulate18except ImportError as e:19    print("the following Python libraries are required: GitPython, tabulate.") # noqa: NP10020    raise e21 22 23logger = logging.getLogger("compare-llama-bench")24 25# All llama-bench SQL fields26LLAMA_BENCH_DB_FIELDS = [27    "build_commit", "build_number", "cpu_info",       "gpu_info",   "backends",     "model_filename",28    "model_type",   "model_size",   "model_n_params", "n_batch",    "n_ubatch",     "n_threads",29    "cpu_mask",     "cpu_strict",   "poll",           "type_k",     "type_v",       "n_gpu_layers",30    "split_mode",   "main_gpu",     "no_kv_offload",  "flash_attn", "tensor_split", "tensor_buft_overrides",31    "use_mmap",     "embeddings",   "no_op_offload",  "n_prompt",   "n_gen",        "n_depth",32    "test_time",    "avg_ns",       "stddev_ns",      "avg_ts",     "stddev_ts",    "n_cpu_moe",33    "fit_target",   "fit_min_ctx"34]35 36LLAMA_BENCH_DB_TYPES = [37    "TEXT",    "INTEGER", "TEXT",    "TEXT",    "TEXT",    "TEXT",38    "TEXT",    "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",39    "TEXT",    "INTEGER", "INTEGER", "TEXT",    "TEXT",    "INTEGER",40    "TEXT",    "INTEGER", "INTEGER", "INTEGER", "TEXT",    "TEXT",41    "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",42    "TEXT",    "INTEGER", "INTEGER", "REAL",    "REAL",    "INTEGER",43    "INTEGER", "INTEGER"44]45 46# All test-backend-ops SQL fields47TEST_BACKEND_OPS_DB_FIELDS = [48    "test_time", "build_commit", "backend_name",  "op_name", "op_params", "test_mode",49    "supported", "passed",       "error_message", "time_us", "flops",     "bandwidth_gb_s",50    "memory_kb", "n_runs"51]52 53TEST_BACKEND_OPS_DB_TYPES = [54    "TEXT",    "TEXT",    "TEXT", "TEXT", "TEXT", "TEXT",55    "INTEGER", "INTEGER", "TEXT", "REAL", "REAL", "REAL",56    "INTEGER", "INTEGER"57]58 59assert len(LLAMA_BENCH_DB_FIELDS) == len(LLAMA_BENCH_DB_TYPES)60assert len(TEST_BACKEND_OPS_DB_FIELDS) == len(TEST_BACKEND_OPS_DB_TYPES)61 62# Properties by which to differentiate results per commit for llama-bench:63LLAMA_BENCH_KEY_PROPERTIES = [64    "cpu_info", "gpu_info", "backends", "n_gpu_layers", "n_cpu_moe", "tensor_buft_overrides", "model_filename", "model_type",65    "n_batch", "n_ubatch", "embeddings", "cpu_mask", "cpu_strict", "poll", "n_threads", "type_k", "type_v",66    "use_mmap", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",67    "fit_target", "fit_min_ctx"68]69 70# Properties by which to differentiate results per commit for test-backend-ops:71TEST_BACKEND_OPS_KEY_PROPERTIES = [72    "backend_name", "op_name", "op_params", "test_mode"73]74 75# Properties that are boolean and are converted to Yes/No for the table:76LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "use_mmap", "no_kv_offload", "flash_attn"]77TEST_BACKEND_OPS_BOOL_PROPERTIES = ["supported", "passed"]78 79# Header names for the table (llama-bench):80LLAMA_BENCH_PRETTY_NAMES = {81    "cpu_info": "CPU", "gpu_info": "GPU", "backends": "Backends", "n_gpu_layers": "GPU layers",82    "tensor_buft_overrides": "Tensor overrides", "model_filename": "File", "model_type": "Model", "model_size": "Model size [GiB]",83    "model_n_params": "Num. of par.", "n_batch": "Batch size", "n_ubatch": "Microbatch size", "embeddings": "Embeddings",84    "cpu_mask": "CPU mask", "cpu_strict": "CPU strict", "poll": "Poll", "n_threads": "Threads", "type_k": "K type", "type_v": "V type",85    "use_mmap": "Use mmap", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",86    "flash_attn": "FlashAttention",87}88 89# Header names for the table (test-backend-ops):90TEST_BACKEND_OPS_PRETTY_NAMES = {91    "backend_name": "Backend", "op_name": "GGML op", "op_params": "Op parameters", "test_mode": "Mode",92    "supported": "Supported", "passed": "Passed", "error_message": "Error",93    "flops": "FLOPS", "bandwidth_gb_s": "Bandwidth (GB/s)", "memory_kb": "Memory (KB)", "n_runs": "Runs"94}95 96DEFAULT_SHOW_LLAMA_BENCH = ["model_type"]  # Always show these properties by default.97DEFAULT_HIDE_LLAMA_BENCH = ["model_filename"]  # Always hide these properties by default.98 99DEFAULT_SHOW_TEST_BACKEND_OPS = ["backend_name", "op_name"]  # Always show these properties by default.100DEFAULT_HIDE_TEST_BACKEND_OPS = ["error_message"]  # Always hide these properties by default.101 102GPU_NAME_STRIP = ["NVIDIA GeForce ", "Tesla ", "AMD Radeon ", "AMD Instinct "]  # Strip prefixes for smaller tables.103MODEL_SUFFIX_REPLACE = {" - Small": "_S", " - Medium": "_M", " - Large": "_L"}104 105DESCRIPTION = """Creates tables from llama-bench or test-backend-ops data written to multiple JSON/CSV files, a single JSONL file or SQLite database. Example usage (Linux):106 107For llama-bench:108$ git checkout master109$ cmake -B ${BUILD_DIR} ${CMAKE_OPTS} && cmake --build ${BUILD_DIR} -t llama-bench -j $(nproc)110$ ./llama-bench -o sql | sqlite3 llama-bench.sqlite111$ git checkout some_branch112$ cmake -B ${BUILD_DIR} ${CMAKE_OPTS} && cmake --build ${BUILD_DIR} -t llama-bench -j $(nproc)113$ ./llama-bench -o sql | sqlite3 llama-bench.sqlite114$ ./scripts/compare-llama-bench.py115 116For test-backend-ops:117$ git checkout master118$ cmake -B ${BUILD_DIR} ${CMAKE_OPTS} && cmake --build ${BUILD_DIR} -t test-backend-ops -j $(nproc)119$ ./test-backend-ops perf --output sql | sqlite3 test-backend-ops.sqlite120$ git checkout some_branch121$ cmake -B ${BUILD_DIR} ${CMAKE_OPTS} && cmake --build ${BUILD_DIR} -t test-backend-ops -j $(nproc)122$ ./test-backend-ops perf --output sql | sqlite3 test-backend-ops.sqlite123$ ./scripts/compare-llama-bench.py --tool test-backend-ops -i test-backend-ops.sqlite124 125Performance numbers from multiple runs per commit are averaged WITHOUT being weighted by the --repetitions parameter of llama-bench.126"""127 128parser = argparse.ArgumentParser(129    description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)130help_b = (131    "The baseline commit to compare performance to. "132    "Accepts either a branch name, tag name, or commit hash. "133    "Defaults to latest master commit with data."134)135parser.add_argument("-b", "--baseline", help=help_b)136help_c = (137    "The commit whose performance is to be compared to the baseline. "138    "Accepts either a branch name, tag name, or commit hash. "139    "Defaults to the non-master commit for which llama-bench was run most recently."140)141parser.add_argument("-c", "--compare", help=help_c)142help_t = (143    "The tool whose data is being compared. "144    "Either 'llama-bench' or 'test-backend-ops'. "145    "This determines the database schema and comparison logic used. "146    "If left unspecified, try to determine from the input file."147)148parser.add_argument("-t", "--tool", help=help_t, default=None, choices=[None, "llama-bench", "test-backend-ops"])149help_i = (150    "JSON/JSONL/SQLite/CSV files for comparing commits. "151    "Specify multiple times to use multiple input files (JSON/CSV only). "152    "Defaults to 'llama-bench.sqlite' in the current working directory. "153    "If no such file is found and there is exactly one .sqlite file in the current directory, "154    "that file is instead used as input."155)156parser.add_argument("-i", "--input", action="append", help=help_i)157help_o = (158    "Output format for the table. "159    "Defaults to 'pipe' (GitHub compatible). "160    "Also supports e.g. 'latex' or 'mediawiki'. "161    "See tabulate documentation for full list."162)163parser.add_argument("-o", "--output", help=help_o, default="pipe")164help_s = (165    "Columns to add to the table. "166    "Accepts a comma-separated list of values. "167    f"Legal values for test-backend-ops: {', '.join(TEST_BACKEND_OPS_KEY_PROPERTIES)}. "168    f"Legal values for llama-bench: {', '.join(LLAMA_BENCH_KEY_PROPERTIES[:-3])}. "169    "Defaults to model name (model_type) and CPU and/or GPU name (cpu_info, gpu_info) "170    "plus any column where not all data points are the same. "171    "If the columns are manually specified, then the results for each unique combination of the "172    "specified values are averaged WITHOUT weighing by the --repetitions parameter of llama-bench."173)174parser.add_argument("--check", action="store_true", help="check if all required Python libraries are installed")175parser.add_argument("-s", "--show", help=help_s)176parser.add_argument("--verbose", action="store_true", help="increase output verbosity")177parser.add_argument("--plot", help="generate a performance comparison plot and save to specified file (e.g., plot.png)")178parser.add_argument("--plot_x", help="parameter to use as x axis for plotting (default: n_depth)", default="n_depth")179parser.add_argument("--plot_log_scale", action="store_true", help="use log scale for x axis in plots (off by default)")180 181known_args, unknown_args = parser.parse_known_args()182 183logging.basicConfig(level=logging.DEBUG if known_args.verbose else logging.INFO)184 185 186if known_args.check:187    # Check if all required Python libraries are installed. Would have failed earlier if not.188    sys.exit(0)189 190if unknown_args:191    logger.error(f"Received unknown args: {unknown_args}.\n")192    parser.print_help()193    sys.exit(1)194 195input_file = known_args.input196tool = known_args.tool197 198if not input_file:199    if tool == "llama-bench" and os.path.exists("./llama-bench.sqlite"):200        input_file = ["llama-bench.sqlite"]201    elif tool == "test-backend-ops" and os.path.exists("./test-backend-ops.sqlite"):202        input_file = ["test-backend-ops.sqlite"]203 204if not input_file:205    sqlite_files = glob("*.sqlite")206    if len(sqlite_files) == 1:207        input_file = sqlite_files208 209if not input_file:210    logger.error("Cannot find a suitable input file, please provide one.\n")211    parser.print_help()212    sys.exit(1)213 214 215class LlamaBenchData:216    repo: Optional[git.Repo]217    build_len_min: int218    build_len_max: int219    build_len: int = 8220    builds: list[str] = []221    tool: str = "llama-bench"  # Tool type: "llama-bench" or "test-backend-ops"222 223    def __init__(self, tool: str = "llama-bench"):224        self.tool = tool225        try:226            self.repo = git.Repo(".", search_parent_directories=True)227        except git.InvalidGitRepositoryError:228            self.repo = None229 230        # Set schema-specific properties based on tool231        if self.tool == "llama-bench":232            self.check_keys = set(LLAMA_BENCH_KEY_PROPERTIES + ["build_commit", "test_time", "avg_ts"])233        elif self.tool == "test-backend-ops":234            self.check_keys = set(TEST_BACKEND_OPS_KEY_PROPERTIES + ["build_commit", "test_time"])235        else:236            assert False237 238    def _builds_init(self):239        self.build_len = self.build_len_min240 241    def _check_keys(self, keys: set) -> Optional[set]:242        """Private helper method that checks against required data keys and returns missing ones."""243        if not keys >= self.check_keys:244            return self.check_keys - keys245        return None246 247    def find_parent_in_data(self, commit: git.Commit) -> Optional[str]:248        """Helper method to find the most recent parent measured in number of commits for which there is data."""249        heap: list[tuple[int, git.Commit]] = [(0, commit)]250        seen_hexsha8 = set()251        while heap:252            depth, current_commit = heapq.heappop(heap)253            current_hexsha8 = commit.hexsha[:self.build_len]254            if current_hexsha8 in self.builds:255                return current_hexsha8256            for parent in commit.parents:257                parent_hexsha8 = parent.hexsha[:self.build_len]258                if parent_hexsha8 not in seen_hexsha8:259                    seen_hexsha8.add(parent_hexsha8)260                    heapq.heappush(heap, (depth + 1, parent))261        return None262 263    def get_all_parent_hexsha8s(self, commit: git.Commit) -> Sequence[str]:264        """Helper method to recursively get hexsha8 values for all parents of a commit."""265        unvisited = [commit]266        visited   = []267 268        while unvisited:269            current_commit = unvisited.pop(0)270            visited.append(current_commit.hexsha[:self.build_len])271            for parent in current_commit.parents:272                if parent.hexsha[:self.build_len] not in visited:273                    unvisited.append(parent)274 275        return visited276 277    def get_commit_name(self, hexsha8: str) -> str:278        """Helper method to find a human-readable name for a commit if possible."""279        if self.repo is None:280            return hexsha8281        for h in self.repo.heads:282            if h.commit.hexsha[:self.build_len] == hexsha8:283                return h.name284        for t in self.repo.tags:285            if t.commit.hexsha[:self.build_len] == hexsha8:286                return t.name287        return hexsha8288 289    def get_commit_hexsha8(self, name: str) -> Optional[str]:290        """Helper method to search for a commit given a human-readable name."""291        if self.repo is None:292            return None293        for h in self.repo.heads:294            if h.name == name:295                return h.commit.hexsha[:self.build_len]296        for t in self.repo.tags:297            if t.name == name:298                return t.commit.hexsha[:self.build_len]299        for remote in self.repo.remotes:300            for ref in remote.refs:301                if ref.name == name or ref.remote_head == name:302                    return ref.commit.hexsha[:self.build_len]303        for c in self.repo.iter_commits("--all"):304            if c.hexsha[:self.build_len] == name[:self.build_len]:305                return c.hexsha[:self.build_len]306        return None307 308    def builds_timestamp(self, reverse: bool = False) -> Union[Iterator[tuple], Sequence[tuple]]:309        """Helper method that gets rows of (build_commit, test_time) sorted by the latter."""310        return []311 312    def get_rows(self, properties: list[str], hexsha8_baseline: str, hexsha8_compare: str) -> Sequence[tuple]:313        """314        Helper method that gets table rows for some list of properties.315        Rows are created by combining those where all provided properties are equal.316        The resulting rows are then grouped by the provided properties and the t/s values are averaged.317        The returned rows are unique in terms of property combinations.318        """319        return []320 321 322class LlamaBenchDataSQLite3(LlamaBenchData):323    connection: Optional[sqlite3.Connection] = None324    cursor: sqlite3.Cursor325    table_name: str326 327    def __init__(self, tool: str = "llama-bench"):328        super().__init__(tool)329        if self.connection is None:330            self.connection = sqlite3.connect(":memory:")331            self.cursor = self.connection.cursor()332 333            # Set table name and schema based on tool334            if self.tool == "llama-bench":335                self.table_name = "llama_bench"336                db_fields = LLAMA_BENCH_DB_FIELDS337                db_types = LLAMA_BENCH_DB_TYPES338            elif self.tool == "test-backend-ops":339                self.table_name = "test_backend_ops"340                db_fields = TEST_BACKEND_OPS_DB_FIELDS341                db_types = TEST_BACKEND_OPS_DB_TYPES342            else:343                assert False344 345            self.cursor.execute(f"CREATE TABLE {self.table_name}({', '.join(' '.join(x) for x in zip(db_fields, db_types))});")346 347    def _builds_init(self):348        if self.connection:349            self.build_len_min = self.cursor.execute(f"SELECT MIN(LENGTH(build_commit)) from {self.table_name};").fetchone()[0]350            self.build_len_max = self.cursor.execute(f"SELECT MAX(LENGTH(build_commit)) from {self.table_name};").fetchone()[0]351 352            if self.build_len_min != self.build_len_max:353                logger.warning("Data contains commit hashes of differing lengths. It's possible that the wrong commits will be compared. "354                               "Try purging the the database of old commits.")355                self.cursor.execute(f"UPDATE {self.table_name} SET build_commit = SUBSTRING(build_commit, 1, {self.build_len_min});")356 357            builds = self.cursor.execute(f"SELECT DISTINCT build_commit FROM {self.table_name};").fetchall()358            self.builds = list(map(lambda b: b[0], builds))  # list[tuple[str]] -> list[str]359        super()._builds_init()360 361    def builds_timestamp(self, reverse: bool = False) -> Union[Iterator[tuple], Sequence[tuple]]:362        data = self.cursor.execute(363            f"SELECT build_commit, test_time FROM {self.table_name} ORDER BY test_time;").fetchall()364        return reversed(data) if reverse else data365 366    def get_rows(self, properties: list[str], hexsha8_baseline: str, hexsha8_compare: str) -> Sequence[tuple]:367        if self.tool == "llama-bench":368            return self._get_rows_llama_bench(properties, hexsha8_baseline, hexsha8_compare)369        elif self.tool == "test-backend-ops":370            return self._get_rows_test_backend_ops(properties, hexsha8_baseline, hexsha8_compare)371        else:372            assert False373 374    def _get_rows_llama_bench(self, properties: list[str], hexsha8_baseline: str, hexsha8_compare: str) -> Sequence[tuple]:375        select_string = ", ".join(376            [f"tb.{p}" for p in properties] + ["tb.n_prompt", "tb.n_gen", "tb.n_depth", "AVG(tb.avg_ts)", "AVG(tc.avg_ts)"])377        equal_string = " AND ".join(378            [f"tb.{p} = tc.{p}" for p in LLAMA_BENCH_KEY_PROPERTIES] + [379                f"tb.build_commit = '{hexsha8_baseline}'", f"tc.build_commit = '{hexsha8_compare}'"]380        )381        group_order_string = ", ".join([f"tb.{p}" for p in properties] + ["tb.n_gen", "tb.n_prompt", "tb.n_depth"])382        query = (f"SELECT {select_string} FROM {self.table_name} tb JOIN {self.table_name} tc ON {equal_string} "383                 f"GROUP BY {group_order_string} ORDER BY {group_order_string};")384        return self.cursor.execute(query).fetchall()385 386    def _get_rows_test_backend_ops(self, properties: list[str], hexsha8_baseline: str, hexsha8_compare: str) -> Sequence[tuple]:387        # For test-backend-ops, we compare FLOPS and bandwidth metrics (prioritizing FLOPS over bandwidth)388        select_string = ", ".join(389            [f"tb.{p}" for p in properties] + [390                "AVG(tb.flops)", "AVG(tc.flops)",391                "AVG(tb.bandwidth_gb_s)", "AVG(tc.bandwidth_gb_s)"392            ])393        equal_string = " AND ".join(394            [f"tb.{p} = tc.{p}" for p in TEST_BACKEND_OPS_KEY_PROPERTIES] + [395                f"tb.build_commit = '{hexsha8_baseline}'", f"tc.build_commit = '{hexsha8_compare}'",396                "tb.supported = 1", "tc.supported = 1", "tb.passed = 1", "tc.passed = 1"]  # Only compare successful tests397        )398        group_order_string = ", ".join([f"tb.{p}" for p in properties])399        query = (f"SELECT {select_string} FROM {self.table_name} tb JOIN {self.table_name} tc ON {equal_string} "400                 f"GROUP BY {group_order_string} ORDER BY {group_order_string};")401        return self.cursor.execute(query).fetchall()402 403 404class LlamaBenchDataSQLite3File(LlamaBenchDataSQLite3):405    def __init__(self, data_file: str, tool: Any):406        self.connection = sqlite3.connect(data_file)407        self.cursor = self.connection.cursor()408 409        # Check which table exists in the database410        tables = self.cursor.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()411        table_names = [table[0] for table in tables]412 413        # Tool selection logic414        if tool is None:415            if "llama_bench" in table_names:416                self.table_name = "llama_bench"417                tool = "llama-bench"418            elif "test_backend_ops" in table_names:419                self.table_name = "test_backend_ops"420                tool = "test-backend-ops"421            else:422                raise RuntimeError(f"No suitable table found in database. Available tables: {table_names}")423        elif tool == "llama-bench":424            if "llama_bench" in table_names:425                self.table_name = "llama_bench"426                tool = "llama-bench"427            else:428                raise RuntimeError(f"Table 'test' not found for tool 'llama-bench'. Available tables: {table_names}")429        elif tool == "test-backend-ops":430            if "test_backend_ops" in table_names:431                self.table_name = "test_backend_ops"432                tool = "test-backend-ops"433            else:434                raise RuntimeError(f"Table 'test_backend_ops' not found for tool 'test-backend-ops'. Available tables: {table_names}")435        else:436            raise RuntimeError(f"Unknown tool: {tool}")437 438        super().__init__(tool)439        self._builds_init()440 441    @staticmethod442    def valid_format(data_file: str) -> bool:443        connection = sqlite3.connect(data_file)444        cursor = connection.cursor()445 446        try:447            if cursor.execute("PRAGMA schema_version;").fetchone()[0] == 0:448                raise sqlite3.DatabaseError("The provided input file does not exist or is empty.")449        except sqlite3.DatabaseError as e:450            logger.debug(f'"{data_file}" is not a valid SQLite3 file.', exc_info=e)451            cursor = None452 453        connection.close()454        return True if cursor else False455 456 457class LlamaBenchDataJSONL(LlamaBenchDataSQLite3):458    def __init__(self, data_file: str, tool: str = "llama-bench"):459        super().__init__(tool)460 461        # Get the appropriate field list based on tool462        db_fields = LLAMA_BENCH_DB_FIELDS if tool == "llama-bench" else TEST_BACKEND_OPS_DB_FIELDS463 464        with open(data_file, "r", encoding="utf-8") as fp:465            for i, line in enumerate(fp):466                parsed = json.loads(line)467 468                for k in parsed.keys() - set(db_fields):469                    del parsed[k]470 471                if (missing_keys := self._check_keys(parsed.keys())):472                    raise RuntimeError(f"Missing required data key(s) at line {i + 1}: {', '.join(missing_keys)}")473 474                self.cursor.execute(f"INSERT INTO {self.table_name}({', '.join(parsed.keys())}) VALUES({', '.join('?' * len(parsed))});", tuple(parsed.values()))475 476        self._builds_init()477 478    @staticmethod479    def valid_format(data_file: str) -> bool:480        try:481            with open(data_file, "r", encoding="utf-8") as fp:482                for line in fp:483                    json.loads(line)484                    break485        except Exception as e:486            logger.debug(f'"{data_file}" is not a valid JSONL file.', exc_info=e)487            return False488 489        return True490 491 492class LlamaBenchDataJSON(LlamaBenchDataSQLite3):493    def __init__(self, data_files: list[str], tool: str = "llama-bench"):494        super().__init__(tool)495 496        # Get the appropriate field list based on tool497        db_fields = LLAMA_BENCH_DB_FIELDS if tool == "llama-bench" else TEST_BACKEND_OPS_DB_FIELDS498 499        for data_file in data_files:500            with open(data_file, "r", encoding="utf-8") as fp:501                parsed = json.load(fp)502 503                for i, entry in enumerate(parsed):504                    for k in entry.keys() - set(db_fields):505                        del entry[k]506 507                    if (missing_keys := self._check_keys(entry.keys())):508                        raise RuntimeError(f"Missing required data key(s) at entry {i + 1}: {', '.join(missing_keys)}")509 510                    self.cursor.execute(f"INSERT INTO {self.table_name}({', '.join(entry.keys())}) VALUES({', '.join('?' * len(entry))});", tuple(entry.values()))511 512        self._builds_init()513 514    @staticmethod515    def valid_format(data_files: list[str]) -> bool:516        if not data_files:517            return False518 519        for data_file in data_files:520            try:521                with open(data_file, "r", encoding="utf-8") as fp:522                    json.load(fp)523            except Exception as e:524                logger.debug(f'"{data_file}" is not a valid JSON file.', exc_info=e)525                return False526 527        return True528 529 530class LlamaBenchDataCSV(LlamaBenchDataSQLite3):531    def __init__(self, data_files: list[str], tool: str = "llama-bench"):532        super().__init__(tool)533 534        # Get the appropriate field list based on tool535        db_fields = LLAMA_BENCH_DB_FIELDS if tool == "llama-bench" else TEST_BACKEND_OPS_DB_FIELDS536 537        for data_file in data_files:538            with open(data_file, "r", encoding="utf-8") as fp:539                for i, parsed in enumerate(csv.DictReader(fp)):540                    keys = set(parsed.keys())541 542                    for k in keys - set(db_fields):543                        del parsed[k]544 545                    if (missing_keys := self._check_keys(keys)):546                        raise RuntimeError(f"Missing required data key(s) at line {i + 1}: {', '.join(missing_keys)}")547 548                    self.cursor.execute(f"INSERT INTO {self.table_name}({', '.join(parsed.keys())}) VALUES({', '.join('?' * len(parsed))});", tuple(parsed.values()))549 550        self._builds_init()551 552    @staticmethod553    def valid_format(data_files: list[str]) -> bool:554        if not data_files:555            return False556 557        for data_file in data_files:558            try:559                with open(data_file, "r", encoding="utf-8") as fp:560                    for parsed in csv.DictReader(fp):561                        break562            except Exception as e:563                logger.debug(f'"{data_file}" is not a valid CSV file.', exc_info=e)564                return False565 566        return True567 568 569def format_flops(flops_value: float) -> str:570    """Format FLOPS values with appropriate units for better readability."""571    if flops_value == 0:572        return "0.00"573 574    # Define unit thresholds and names575    units = [576        (1e12, "T"),   # TeraFLOPS577        (1e9, "G"),    # GigaFLOPS578        (1e6, "M"),    # MegaFLOPS579        (1e3, "k"),    # kiloFLOPS580        (1, "")        # FLOPS581    ]582 583    for threshold, unit in units:584        if abs(flops_value) >= threshold:585            formatted_value = flops_value / threshold586            if formatted_value >= 100:587                return f"{formatted_value:.1f}{unit}"588            else:589                return f"{formatted_value:.2f}{unit}"590 591    # Fallback for very small values592    return f"{flops_value:.2f}"593 594 595def format_flops_for_table(flops_value: float, target_unit: str) -> str:596    """Format FLOPS values for table display without unit suffix (since unit is in header)."""597    if flops_value == 0:598        return "0.00"599 600    # Define unit thresholds based on target unit601    unit_divisors = {602        "TFLOPS": 1e12,603        "GFLOPS": 1e9,604        "MFLOPS": 1e6,605        "kFLOPS": 1e3,606        "FLOPS": 1607    }608 609    divisor = unit_divisors.get(target_unit, 1)610    formatted_value = flops_value / divisor611 612    if formatted_value >= 100:613        return f"{formatted_value:.1f}"614    else:615        return f"{formatted_value:.2f}"616 617 618def get_flops_unit_name(flops_values: list) -> str:619    """Determine the best FLOPS unit name based on the magnitude of values."""620    if not flops_values or all(v == 0 for v in flops_values):621        return "FLOPS"622 623    # Find the maximum absolute value to determine appropriate unit624    max_flops = max(abs(v) for v in flops_values if v != 0)625 626    if max_flops >= 1e12:627        return "TFLOPS"628    elif max_flops >= 1e9:629        return "GFLOPS"630    elif max_flops >= 1e6:631        return "MFLOPS"632    elif max_flops >= 1e3:633        return "kFLOPS"634    else:635        return "FLOPS"636 637 638bench_data = None639if len(input_file) == 1:640    if LlamaBenchDataSQLite3File.valid_format(input_file[0]):641        bench_data = LlamaBenchDataSQLite3File(input_file[0], tool)642    elif LlamaBenchDataJSON.valid_format(input_file):643        bench_data = LlamaBenchDataJSON(input_file, tool)644    elif LlamaBenchDataJSONL.valid_format(input_file[0]):645        bench_data = LlamaBenchDataJSONL(input_file[0], tool)646    elif LlamaBenchDataCSV.valid_format(input_file):647        bench_data = LlamaBenchDataCSV(input_file, tool)648else:649    if LlamaBenchDataJSON.valid_format(input_file):650        bench_data = LlamaBenchDataJSON(input_file, tool)651    elif LlamaBenchDataCSV.valid_format(input_file):652        bench_data = LlamaBenchDataCSV(input_file, tool)653 654if not bench_data:655    raise RuntimeError("No valid (or some invalid) input files found.")656 657if not bench_data.builds:658    raise RuntimeError(f"{input_file} does not contain any builds.")659 660tool = bench_data.tool  # May have chosen a default if tool was None.661 662 663hexsha8_baseline = name_baseline = None664 665# If the user specified a baseline, try to find a commit for it:666if known_args.baseline is not None:667    if known_args.baseline in bench_data.builds:668        hexsha8_baseline = known_args.baseline669    if hexsha8_baseline is None:670        hexsha8_baseline = bench_data.get_commit_hexsha8(known_args.baseline)671        name_baseline = known_args.baseline672    if hexsha8_baseline is None:673        logger.error(f"cannot find data for baseline={known_args.baseline}.")674        sys.exit(1)675# Otherwise, search for the most recent parent of master for which there is data:676elif bench_data.repo is not None:677    hexsha8_baseline = bench_data.find_parent_in_data(bench_data.repo.heads.master.commit)678 679    if hexsha8_baseline is None:680        logger.error("No baseline was provided and did not find data for any master branch commits.\n")681        parser.print_help()682        sys.exit(1)683else:684    logger.error("No baseline was provided and the current working directory "685                 "is not part of a git repository from which a baseline could be inferred.\n")686    parser.print_help()687    sys.exit(1)688 689 690assert isinstance(hexsha8_baseline, str)691name_baseline = bench_data.get_commit_name(hexsha8_baseline)692 693hexsha8_compare = name_compare = None694 695# If the user has specified a compare value, try to find a corresponding commit:696if known_args.compare is not None:697    if known_args.compare in bench_data.builds:698        hexsha8_compare = known_args.compare699    if hexsha8_compare is None:700        hexsha8_compare = bench_data.get_commit_hexsha8(known_args.compare)701        name_compare = known_args.compare702    if hexsha8_compare is None:703        logger.error(f"cannot find data for compare={known_args.compare}.")704        sys.exit(1)705# Otherwise, search for the commit for llama-bench was most recently run706# and that is not a parent of master:707elif bench_data.repo is not None:708    hexsha8s_master = bench_data.get_all_parent_hexsha8s(bench_data.repo.heads.master.commit)709    for (hexsha8, _) in bench_data.builds_timestamp(reverse=True):710        if hexsha8 not in hexsha8s_master:711            hexsha8_compare = hexsha8712            break713 714    if hexsha8_compare is None:715        logger.error("No compare target was provided and did not find data for any non-master commits.\n")716        parser.print_help()717        sys.exit(1)718else:719    logger.error("No compare target was provided and the current working directory "720                 "is not part of a git repository from which a compare target could be inferred.\n")721    parser.print_help()722    sys.exit(1)723 724assert isinstance(hexsha8_compare, str)725name_compare = bench_data.get_commit_name(hexsha8_compare)726 727# Get tool-specific configuration728if tool == "llama-bench":729    key_properties = LLAMA_BENCH_KEY_PROPERTIES730    bool_properties = LLAMA_BENCH_BOOL_PROPERTIES731    pretty_names = LLAMA_BENCH_PRETTY_NAMES732    default_show = DEFAULT_SHOW_LLAMA_BENCH733    default_hide = DEFAULT_HIDE_LLAMA_BENCH734elif tool == "test-backend-ops":735    key_properties = TEST_BACKEND_OPS_KEY_PROPERTIES736    bool_properties = TEST_BACKEND_OPS_BOOL_PROPERTIES737    pretty_names = TEST_BACKEND_OPS_PRETTY_NAMES738    default_show = DEFAULT_SHOW_TEST_BACKEND_OPS739    default_hide = DEFAULT_HIDE_TEST_BACKEND_OPS740else:741    assert False742 743# If the user provided columns to group the results by, use them:744if known_args.show is not None:745    show = known_args.show.split(",")746    unknown_cols = []747    for prop in show:748        valid_props = key_properties if tool == "test-backend-ops" else key_properties[:-3]  # Exclude n_prompt, n_gen, n_depth for llama-bench749        if prop not in valid_props:750            unknown_cols.append(prop)751    if unknown_cols:752        logger.error(f"Unknown values for --show: {', '.join(unknown_cols)}")753        parser.print_usage()754        sys.exit(1)755    rows_show = bench_data.get_rows(show, hexsha8_baseline, hexsha8_compare)756# Otherwise, select those columns where the values are not all the same:757else:758    rows_full = bench_data.get_rows(key_properties, hexsha8_baseline, hexsha8_compare)759    properties_different = []760 761    if tool == "llama-bench":762        # For llama-bench, skip n_prompt, n_gen, n_depth from differentiation logic763        check_properties = [kp for kp in key_properties if kp not in ["n_prompt", "n_gen", "n_depth"]]764        for i, kp_i in enumerate(key_properties):765            if kp_i in default_show or kp_i in ["n_prompt", "n_gen", "n_depth"]:766                continue767            for row_full in rows_full:768                if row_full[i] != rows_full[0][i]:769                    properties_different.append(kp_i)770                    break771    elif tool == "test-backend-ops":772        # For test-backend-ops, check all key properties773        for i, kp_i in enumerate(key_properties):774            if kp_i in default_show:775                continue776            for row_full in rows_full:777                if row_full[i] != rows_full[0][i]:778                    properties_different.append(kp_i)779                    break780    else:781        assert False782 783    show = []784 785    if tool == "llama-bench":786        # Show CPU and/or GPU by default even if the hardware for all results is the same:787        if rows_full and "n_gpu_layers" not in properties_different:788            ngl = int(rows_full[0][key_properties.index("n_gpu_layers")])789 790            if ngl != 99 and "cpu_info" not in properties_different:791                show.append("cpu_info")792 793        show += properties_different794 795        index_default = 0796        for prop in ["cpu_info", "gpu_info", "n_gpu_layers", "main_gpu"]:797            if prop in show:798                index_default += 1799        show = show[:index_default] + default_show + show[index_default:]800    elif tool == "test-backend-ops":801        show = default_show + properties_different802    else:803        assert False804 805    for prop in default_hide:806        try:807            show.remove(prop)808        except ValueError:809            pass810 811    # Add plot_x parameter to parameters to show if it's not already present:812    if known_args.plot:813        for k, v in pretty_names.items():814            if v == known_args.plot_x and k not in show:815                show.append(k)816                break817 818    rows_show = bench_data.get_rows(show, hexsha8_baseline, hexsha8_compare)819 820if not rows_show:821    logger.error(f"No comparable data was found between {name_baseline} and {name_compare}.\n")822    sys.exit(1)823 824table = []825primary_metric = "FLOPS"  # Default to FLOPS for test-backend-ops826 827if tool == "llama-bench":828    # For llama-bench, create test names and compare avg_ts values829    for row in rows_show:830        n_prompt = int(row[-5])831        n_gen    = int(row[-4])832        n_depth  = int(row[-3])833        if n_prompt != 0 and n_gen == 0:834            test_name = f"pp{n_prompt}"835        elif n_prompt == 0 and n_gen != 0:836            test_name = f"tg{n_gen}"837        else:838            test_name = f"pp{n_prompt}+tg{n_gen}"839        if n_depth != 0:840            test_name = f"{test_name}@d{n_depth}"841        #           Regular columns    test name    avg t/s values              Speedup842        #            VVVVVVVVVVVVV     VVVVVVVVV    VVVVVVVVVVVVVV              VVVVVVV843        table.append(list(row[:-5]) + [test_name] + list(row[-2:]) + [float(row[-1]) / float(row[-2])])844elif tool == "test-backend-ops":845    # Determine the primary metric by checking rows until we find one with valid data846    if rows_show:847        primary_metric = "FLOPS"  # Default to FLOPS848        flops_values = []849 850        # Collect all FLOPS values to determine the best unit851        for sample_row in rows_show:852            baseline_flops = float(sample_row[-4])853            compare_flops = float(sample_row[-3])854            baseline_bandwidth = float(sample_row[-2])855 856            if baseline_flops > 0:857                flops_values.extend([baseline_flops, compare_flops])858            elif baseline_bandwidth > 0 and not flops_values:859                primary_metric = "Bandwidth (GB/s)"860 861        # If we have FLOPS data, determine the appropriate unit862        if flops_values:863            primary_metric = get_flops_unit_name(flops_values)864 865    # For test-backend-ops, prioritize FLOPS > bandwidth for comparison866    for row in rows_show:867        # Extract metrics: flops, bandwidth_gb_s (baseline and compare)868        baseline_flops = float(row[-4])869        compare_flops = float(row[-3])870        baseline_bandwidth = float(row[-2])871        compare_bandwidth = float(row[-1])872 873        # Determine which metric to use for comparison (prioritize FLOPS > bandwidth)874        if baseline_flops > 0 and compare_flops > 0:875            # Use FLOPS comparison (higher is better)876            speedup = compare_flops / baseline_flops877            baseline_str = format_flops_for_table(baseline_flops, primary_metric)878            compare_str = format_flops_for_table(compare_flops, primary_metric)879        elif baseline_bandwidth > 0 and compare_bandwidth > 0:880            # Use bandwidth comparison (higher is better)881            speedup = compare_bandwidth / baseline_bandwidth882            baseline_str = f"{baseline_bandwidth:.2f}"883            compare_str = f"{compare_bandwidth:.2f}"884        else:885            # Fallback if no valid data is available886            baseline_str = "N/A"887            compare_str = "N/A"888            from math import nan889            speedup = nan890 891        table.append(list(row[:-4]) + [baseline_str, compare_str, speedup])892else:893    assert False894 895# Some a-posteriori fixes to make the table contents prettier:896for bool_property in bool_properties:897    if bool_property in show:898        ip = show.index(bool_property)899        for row_table in table:900            row_table[ip] = "Yes" if int(row_table[ip]) == 1 else "No"901 902if tool == "llama-bench":903    if "model_type" in show:904        ip = show.index("model_type")905        for (old, new) in MODEL_SUFFIX_REPLACE.items():906            for row_table in table:907                row_table[ip] = row_table[ip].replace(old, new)908 909    if "model_size" in show:910        ip = show.index("model_size")911        for row_table in table:912            row_table[ip] = float(row_table[ip]) / 1024 ** 3913 914    if "gpu_info" in show:915        ip = show.index("gpu_info")916        for row_table in table:917            for gns in GPU_NAME_STRIP:918                row_table[ip] = row_table[ip].replace(gns, "")919 920            gpu_names = row_table[ip].split(", ")921            num_gpus = len(gpu_names)922            all_names_the_same = len(set(gpu_names)) == 1923            if len(gpu_names) >= 2 and all_names_the_same:924                row_table[ip] = f"{num_gpus}x {gpu_names[0]}"925 926headers  = [pretty_names.get(p, p) for p in show]927if tool == "llama-bench":928    headers += ["Test", f"t/s {name_baseline}", f"t/s {name_compare}", "Speedup"]929elif tool == "test-backend-ops":930    headers += [f"{primary_metric} {name_baseline}", f"{primary_metric} {name_compare}", "Speedup"]931else:932    assert False933 934if known_args.plot:935    def create_performance_plot(table_data: list[list[str]], headers: list[str], baseline_name: str, compare_name: str, output_file: str, plot_x_param: str, log_scale: bool = False, tool_type: str = "llama-bench", metric_name: str = "t/s"):936        try:937            import matplotlib938            import matplotlib.pyplot as plt939            matplotlib.use('Agg')940        except ImportError as e:941            logger.error("matplotlib is required for --plot.")942            raise e943 944        data_headers = headers[:-4] # Exclude the last 4 columns (Test, baseline t/s, compare t/s, Speedup)945        plot_x_index = None946        plot_x_label = plot_x_param947 948        if plot_x_param not in ["n_prompt", "n_gen", "n_depth"]:949            pretty_name = LLAMA_BENCH_PRETTY_NAMES.get(plot_x_param, plot_x_param)950            if pretty_name in data_headers:951                plot_x_index = data_headers.index(pretty_name)952                plot_x_label = pretty_name953            elif plot_x_param in data_headers:954                plot_x_index = data_headers.index(plot_x_param)955                plot_x_label = plot_x_param956            else:957                logger.error(f"Parameter '{plot_x_param}' not found in current table columns. Available columns: {', '.join(data_headers)}")958                return959 960        grouped_data = {}961 962        for i, row in enumerate(table_data):963            group_key_parts = []964            test_name = row[-4]965 966            base_test = ""967            x_value = None968 969            if plot_x_param in ["n_prompt", "n_gen", "n_depth"]:970                for j, val in enumerate(row[:-4]):971                    header_name = data_headers[j]972                    if val is not None and str(val).strip():973                        group_key_parts.append(f"{header_name}={val}")974 975                if plot_x_param == "n_prompt" and "pp" in test_name:976                    base_test = test_name.split("@")[0]977                    x_value = base_test978                elif plot_x_param == "n_gen" and "tg" in test_name:979                    x_value = test_name.split("@")[0]980                elif plot_x_param == "n_depth" and "@d" in test_name:981                    base_test = test_name.split("@d")[0]982                    x_value = int(test_name.split("@d")[1])983                else:984                    base_test = test_name985 986                if base_test.strip():987                    group_key_parts.append(f"Test={base_test}")988            else:989                for j, val in enumerate(row[:-4]):990                    if j != plot_x_index:991                        header_name = data_headers[j]992                        if val is not None and str(val).strip():993                            group_key_parts.append(f"{header_name}={val}")994                    else:995                        x_value = val996 997                group_key_parts.append(f"Test={test_name}")998 999            group_key = tuple(group_key_parts)1000 1001            if group_key not in grouped_data:1002                grouped_data[group_key] = []1003 1004            grouped_data[group_key].append({1005                'x_value': x_value,1006                'baseline': float(row[-3]),1007                'compare': float(row[-2]),1008                'speedup': float(row[-1])1009            })1010 1011        if not grouped_data:1012            logger.error("No data available for plotting")1013            return1014 1015        def make_axes(num_groups, max_cols=2, base_size=(8, 4)):1016            from math import ceil1017            cols = 1 if num_groups == 1 else min(max_cols, num_groups)1018            rows = ceil(num_groups / cols)1019 1020            # Scale figure size by grid dimensions1021            w, h = base_size1022            fig, ax_arr = plt.subplots(rows, cols,1023                                       figsize=(w * cols, h * rows),1024                                       squeeze=False)1025 1026            axes = ax_arr.flatten()[:num_groups]1027            return fig, axes1028 1029        num_groups = len(grouped_data)1030        fig, axes = make_axes(num_groups)1031 1032        plot_idx = 01033 1034        for group_key, points in grouped_data.items():1035            if plot_idx >= len(axes):1036                break1037            ax = axes[plot_idx]1038 1039            try:1040                points_sorted = sorted(points, key=lambda p: float(p['x_value']) if p['x_value'] is not None else 0)1041                x_values = [float(p['x_value']) if p['x_value'] is not None else 0 for p in points_sorted]1042            except ValueError:1043                points_sorted = sorted(points, key=lambda p: group_key)1044                x_values = [p['x_value'] for p in points_sorted]1045 1046            baseline_vals = [p['baseline'] for p in points_sorted]1047            compare_vals = [p['compare'] for p in points_sorted]1048 1049            ax.plot(x_values, baseline_vals, 'o-', color='skyblue',1050                    label=f'{baseline_name}', linewidth=2, markersize=6)1051            ax.plot(x_values, compare_vals, 's--', color='lightcoral', alpha=0.8,1052                    label=f'{compare_name}', linewidth=2, markersize=6)1053 1054            if log_scale:1055                ax.set_xscale('log', base=2)1056                unique_x = sorted(set(x_values))1057                ax.set_xticks(unique_x)1058                ax.set_xticklabels([str(int(x)) for x in unique_x])1059 1060            title_parts = []1061            for part in group_key:1062                if '=' in part:1063                    key, value = part.split('=', 1)1064                    title_parts.append(f"{key}: {value}")1065 1066            title = ', '.join(title_parts) if title_parts else "Performance comparison"1067 1068            # Determine y-axis label based on tool type1069            if tool_type == "llama-bench":1070                y_label = "Tokens per second (t/s)"1071            elif tool_type == "test-backend-ops":1072                y_label = metric_name1073            else:1074                assert False1075 1076            ax.set_xlabel(plot_x_label, fontsize=12, fontweight='bold')1077            ax.set_ylabel(y_label, fontsize=12, fontweight='bold')1078            ax.set_title(title, fontsize=12, fontweight='bold')1079            ax.legend(loc='best', fontsize=10)1080            ax.grid(True, alpha=0.3)1081 1082            plot_idx += 11083 1084        for i in range(plot_idx, len(axes)):1085            axes[i].set_visible(False)1086 1087        fig.suptitle(f'Performance comparison: {compare_name} vs. {baseline_name}',1088                     fontsize=14, fontweight='bold')1089        fig.subplots_adjust(top=1)1090 1091        plt.tight_layout()1092        plt.savefig(output_file, dpi=300, bbox_inches='tight')1093        plt.close()1094 1095    create_performance_plot(table, headers, name_baseline, name_compare, known_args.plot, known_args.plot_x, known_args.plot_log_scale, tool, primary_metric)1096 1097print(tabulate( # noqa: NP1001098    table,1099    headers=headers,1100    floatfmt=".2f",1101    tablefmt=known_args.output1102))1103