Mnttnt21/tokenizer-bench

1<?php2 3declare(strict_types=1);4 5require __DIR__ . '/../vendor/autoload.php';6 7use Bench\EngineRegistry;8 9$registry = new EngineRegistry();10$defaults = require __DIR__ . '/../fixtures/defaults.php';11$languages = $registry->languages();12 13$SEP = '·';14 15foreach ($languages as $code => $label) {16    $engines = $registry->forLang($code);17    $lines = array_values(array_filter(array_map('trim', explode("\n", $defaults[$code] ?? '')), fn ($l) => $l !== ''));18 19    echo "\n==================== $label ($code) ====================\n";20    echo 'engines: ' . implode(', ', array_map(fn ($e) => $e->name(), $engines)) . "\n";21 22    $names = array_map(fn ($e) => $e->name(), $engines);23    // exclude jieba search from the "disagreement" comparison, like the UI does24    $diffIdx = [];25    foreach ($names as $i => $n) {26        if (stripos($n, 'search') === false && stripos($n, 'n-gram') === false) {27            $diffIdx[] = $i;28        }29    }30 31    $disagree = 0;32    foreach ($lines as $line) {33        $cols = [];34        foreach ($engines as $i => $e) {35            try {36                $tok = $e->tokenize($line, $code);37                $cols[$i] = implode($SEP, $tok);38            } catch (\Throwable $ex) {39                $cols[$i] = 'ERR:' . $ex->getMessage();40            }41        }42 43        // disagreement among diffable engines44        $sig = array_unique(array_map(fn ($i) => $cols[$i], $diffIdx));45        $mark = count($sig) > 1 ? ' *** DIFF' : '';46        if (count($sig) > 1) {47            $disagree++;48        }49 50        echo "\n  $line$mark\n";51        foreach ($engines as $i => $e) {52            printf("      %-20s %s\n", $e->name(), $cols[$i]);53        }54    }55    echo "\n  >>> $disagree/" . count($lines) . " keywords where real engines disagree\n";56}57