Dragongon/FinLFQA

from datasets import load_dataset
import json

dataset = load_dataset("Dragongon/FinLFQA")

# Access the development set
dev_set = dataset["validation"]
print(f"Development set size: {len(dev_set)}")

# Access the test set
test_set = dataset["test"]
print(f"Test set size: {len(test_set)}")

# Print the first example
example = dev_set[0]
# The `context` and `clauses` fields are JSON-encoded strings; parse them as needed:
context = json.loads(example["context"])
clauses = json.loads(example["clauses"])
print(example["question"])
print("Companies:", list(context.keys()))
print("Number of clauses:", len(clauses))

{
    "id": "[int] Unique identifier for the example",
    "question": "[string] The financial analysis question",
    "answer": "[string] Expert-written long-form answer with inline attribution markers",
    "topic": "[string] The financial analysis topic category",
    "clauses": "[string] JSON-encoded list of decomposed answer clauses with fine-grained attribution",
    "context": "[string] JSON-encoded dict of financial document paragraphs keyed by company ticker",
    "professional knowledge list": "[list] Relevant financial formulas and domain knowledge",
    "numerical_values": "[list] Key numerical values involved in the answer"
}

{
    "cid": "[int] Clause ID",
    "clause": "[string] The claim text",
    "inference": "[list] Indices of clauses this clause infers from",
    "evidence": "[dict] Mapping from company ticker to paragraph indices used as evidence",
    "professional knowledge": "[string] The financial formula or knowledge applied",
    "code": "[string] Executable Python code for numerical verification",
    "code_execution_result": "[string] Result of executing the code"
}

{
    "id": 0,
    "question": "How does EBC's net interest income sensitivity compare between March 31, 2024, and December 31, 2023, when the interest rate change is +200 basis points?",
    "answer": "EBC's net interest income sensitivity decreased by 0.2% {code: [0]} (2.9% - 3.1%) from December 31, 2023, to March 31, 2024. {evidence: EBC: [4], W: [], professional knowledge: [0]} ...",
    "topic": "Cost of Capital Optimization Using Real Options Analysis",
    "clauses": [
        {
            "cid": 0,
            "clause": "EBC's net interest income sensitivity decreased by 0.2% (2.9% - 3.1%) from December 31, 2023, to March 31, 2024.",
            "inference": [],
            "evidence": {"EBC": [4], "W": []},
            "professional knowledge": "Interest Rate Risk Analysis=Net Interest Margin (NIM) = (Interest Income - Interest Expense) / Average Earning Assets",
            "code": "def calculate_net_interest_income_sensitivity_change(): ...",
            "code_execution_result": "0.20000000000000018"
        }
    ],
    "context": {
        "EBC": ["paragraph 1", "paragraph 2", "..."],
        "W": ["paragraph 1", "paragraph 2", "..."]
    },
    "professional knowledge list": [
        "Profitability Ratios=Net Profit Margin = (Net Income / Revenue) * 100",
        "..."
    ],
    "numerical_values": [0.2, 2.9, 3.1]
}

@inproceedings{long-etal-2025-finlfqa,
    title = "{F}in{LFQA}: Evaluating Attributed Text Generation of {LLM}s in Financial Long-Form Question Answering",
    author = "Long, Yitao  and
      Hu, Tiansheng  and
      Zhao, Yilun  and
      Cohan, Arman  and
      Zhao, Chen",
    editor = "Christodoulopoulos, Christos  and
      Chakraborty, Tanmoy  and
      Rose, Carolyn  and
      Peng, Violet",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2025",
    month = nov,
    year = "2025",
    address = "Suzhou, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2025.findings-emnlp.908/",
    doi = "10.18653/v1/2025.findings-emnlp.908",
    pages = "16730--16750",
    ISBN = "979-8-89176-335-7",
    abstract = "Large Language Models (LLMs) frequently hallucinate to long-form questions, producing plausible yet factually incorrect answers. A common mitigation strategy is to provide attribution to LLM outputs. However, existing benchmarks primarily focus on simple attribution that retrieves supporting textual evidence as references. We argue that in real-world scenarios such as financial applications, attribution goes beyond reference retrieval.We introduce FinLFQA, a benchmark designed to evaluate the ability of LLMs to generate long-form answers to complex financial questions with reliable and nuanced attributions. FinLFQA evaluates three critical aspects of attribution through human annotations: (1) supporting evidence extracted from financial reports, (2) intermediate numerical reasoning steps, and (3) domain-specific financial knowledge that informs the reasoning process.We further provide an automatic evaluation framework covering both answer quality and attribution quality. Through extensive experiments on eight LLMs across multiple attribution-generation paradigms, we find that fine-grained metrics are important to distinguish model capabilities, that end-to-end generation achieves comparable performance to post-hoc approaches, and that iterative refinement only helps when guided by external feedback."
}