Anushhh/KannadaPromptBench
KannadaPromptBench A benchmark dataset for evaluating prompt strategy sensitivity in Kannada, a low-resource Dravidian language. Dataset Summary Language: Kannada (kn) Tasks: Sentiment Analysis (100), Question Answering (75), Summarization (50) Total: 225 culturally grounded samples Inter-annotator agreement: Cohen's κ > 0.80 Dataset Structure Each sample contains: id, task, input_text, label, difficulty, domain. Citation Please… See the full description on the dataset page: https://huggingface.co/datasets/Anushhh/KannadaPromptBench.
133
1{2 "experiment_name": "KannadaPromptBench-Enhanced",3 "version": "2.0",4 "date": "2026-04-12T09:58:03.861501",5 "models": [6 "llama3.1-8b",7 "sarvam-m",8 "mistralai/mistral-7b-instruct-v0.1"9 ],10 "strategies": [11 "zero_shot",12 "few_shot",13 "cot"14 ],15 "tasks": [16 "sentiment",17 "qa",18 "summarization"19 ],20 "total_dataset_size": 225,21 "total_results_rows": 2025,22 "session_csvs_used": [23 "cerebras_results.csv",24 "sarvam_results.csv",25 "openrouter_results.csv"26 ],27 "enhancements": {28 "inter_annotator_agreement": "Cohen kappa computed in Cell 4b",29 "statistical_significance": "McNemar test in Cell 9b",30 "error_analysis": "5-category analysis in Cell 11",31 "non_llm_baselines": "TF-IDF+LogReg in Cell 8a",32 "huggingface_export": "Cell 4c",33 "adversarial_robustness": "4 perturbation types in Cell 8b",34 "expanded_dataset": "100 sentiment + 75 QA + 50 summarization",35 "rouge_kannada_fix": "Character-level LCS fallback for Kannada ROUGE-L",36 "qa_exact_match_fix": "Substring match for verbose model responses"37 },38 "aggregate_results": [39 {40 "model": "llama3.1-8b",41 "strategy": "cot",42 "task": "qa",43 "accuracy": 0.747,44 "rouge_l": 0.189,45 "n_samples": 7546 },47 {48 "model": "llama3.1-8b",49 "strategy": "cot",50 "task": "sentiment",51 "accuracy": 0.32,52 "rouge_l": NaN,53 "n_samples": 10054 },55 {56 "model": "llama3.1-8b",57 "strategy": "cot",58 "task": "summarization",59 "accuracy": 0.12,60 "rouge_l": 0.402,61 "n_samples": 5062 },63 {64 "model": "llama3.1-8b",65 "strategy": "few_shot",66 "task": "qa",67 "accuracy": 0.747,68 "rouge_l": 0.679,69 "n_samples": 7570 },71 {72 "model": "llama3.1-8b",73 "strategy": "few_shot",74 "task": "sentiment",75 "accuracy": 0.55,76 "rouge_l": NaN,77 "n_samples": 10078 },79 {80 "model": "llama3.1-8b",81 "strategy": "few_shot",82 "task": "summarization",83 "accuracy": 0.64,84 "rouge_l": 0.61,85 "n_samples": 5086 },87 {88 "model": "llama3.1-8b",89 "strategy": "zero_shot",90 "task": "qa",91 "accuracy": 0.933,92 "rouge_l": 0.694,93 "n_samples": 7594 },95 {96 "model": "llama3.1-8b",97 "strategy": "zero_shot",98 "task": "sentiment",99 "accuracy": 0.5,100 "rouge_l": NaN,101 "n_samples": 100102 },103 {104 "model": "llama3.1-8b",105 "strategy": "zero_shot",106 "task": "summarization",107 "accuracy": 0.58,108 "rouge_l": 0.604,109 "n_samples": 50110 },111 {112 "model": "mistralai/mistral-7b-instruct-v0.1",113 "strategy": "cot",114 "task": "qa",115 "accuracy": 0.52,116 "rouge_l": 0.225,117 "n_samples": 75118 },119 {120 "model": "mistralai/mistral-7b-instruct-v0.1",121 "strategy": "cot",122 "task": "sentiment",123 "accuracy": 0.42,124 "rouge_l": NaN,125 "n_samples": 100126 },127 {128 "model": "mistralai/mistral-7b-instruct-v0.1",129 "strategy": "cot",130 "task": "summarization",131 "accuracy": 0.0,132 "rouge_l": 0.109,133 "n_samples": 50134 },135 {136 "model": "mistralai/mistral-7b-instruct-v0.1",137 "strategy": "few_shot",138 "task": "qa",139 "accuracy": 0.533,140 "rouge_l": 0.237,141 "n_samples": 75142 },143 {144 "model": "mistralai/mistral-7b-instruct-v0.1",145 "strategy": "few_shot",146 "task": "sentiment",147 "accuracy": 0.39,148 "rouge_l": NaN,149 "n_samples": 100150 },151 {152 "model": "mistralai/mistral-7b-instruct-v0.1",153 "strategy": "few_shot",154 "task": "summarization",155 "accuracy": 0.12,156 "rouge_l": 0.344,157 "n_samples": 50158 },159 {160 "model": "mistralai/mistral-7b-instruct-v0.1",161 "strategy": "zero_shot",162 "task": "qa",163 "accuracy": 0.587,164 "rouge_l": 0.185,165 "n_samples": 75166 },167 {168 "model": "mistralai/mistral-7b-instruct-v0.1",169 "strategy": "zero_shot",170 "task": "sentiment",171 "accuracy": 0.46,172 "rouge_l": NaN,173 "n_samples": 100174 },175 {176 "model": "mistralai/mistral-7b-instruct-v0.1",177 "strategy": "zero_shot",178 "task": "summarization",179 "accuracy": 0.12,180 "rouge_l": 0.394,181 "n_samples": 50182 },183 {184 "model": "sarvam-m",185 "strategy": "cot",186 "task": "qa",187 "accuracy": 0.6,188 "rouge_l": 0.439,189 "n_samples": 75190 },191 {192 "model": "sarvam-m",193 "strategy": "cot",194 "task": "sentiment",195 "accuracy": 0.34,196 "rouge_l": NaN,197 "n_samples": 100198 },199 {200 "model": "sarvam-m",201 "strategy": "cot",202 "task": "summarization",203 "accuracy": 0.02,204 "rouge_l": 0.121,205 "n_samples": 50206 },207 {208 "model": "sarvam-m",209 "strategy": "few_shot",210 "task": "qa",211 "accuracy": 0.707,212 "rouge_l": 0.137,213 "n_samples": 75214 },215 {216 "model": "sarvam-m",217 "strategy": "few_shot",218 "task": "sentiment",219 "accuracy": 0.37,220 "rouge_l": NaN,221 "n_samples": 100222 },223 {224 "model": "sarvam-m",225 "strategy": "few_shot",226 "task": "summarization",227 "accuracy": 0.0,228 "rouge_l": 0.061,229 "n_samples": 50230 },231 {232 "model": "sarvam-m",233 "strategy": "zero_shot",234 "task": "qa",235 "accuracy": 0.52,236 "rouge_l": 0.04,237 "n_samples": 75238 },239 {240 "model": "sarvam-m",241 "strategy": "zero_shot",242 "task": "sentiment",243 "accuracy": 0.4,244 "rouge_l": NaN,245 "n_samples": 100246 },247 {248 "model": "sarvam-m",249 "strategy": "zero_shot",250 "task": "summarization",251 "accuracy": 0.0,252 "rouge_l": 0.064,253 "n_samples": 50254 }255 ]256}