Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ hep_available: false

# FaithfulnessEvaluator

## Overview

The `FaithfulnessEvaluator` uses an LLM to evaluate whether a generated answer can be inferred from the provided contexts. It does not require ground truth labels. This metric is called faithfulness, sometimes also referred to as groundedness or hallucination.

<div className="key-value-table">
Expand All @@ -33,7 +35,7 @@ If you are not initializing the Evaluator with your own Chat Generator other tha

Two other optional initialization parameters are:

- `raise_on_failure`: If True, raise an exception when the chat generator call fails or its response isn't valid JSON with the expected keys. If False, log a warning and set the result for that input to `None`.
- `raise_on_failure`: If True, raise an exception when the chat generator call fails, its response isn't valid JSON with the expected keys, or the numbers of `statements` and `statement_scores` differ. If False, log a warning and return `status="error"` with a NaN score for the failed input. Failed inputs are excluded from the overall mean; if all inputs fail, the overall score is NaN.
- `progress_bar`: Whether to show a progress bar during the evaluation.

`FaithfulnessEvaluator` has an optional `examples` parameter that can be used to pass few-shot examples conforming to the expected input and output format of `FaithfulnessEvaluator`. These examples are included in the prompt that is sent to the LLM. Examples, therefore, increase the number of tokens of the prompt and make each request more costly. Adding examples is helpful if you want to improve the quality of the evaluation at the cost of more tokens.
Expand Down
12 changes: 11 additions & 1 deletion haystack/components/evaluators/faithfulness.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ def __init__(
Whether to show a progress bar during the evaluation.
:param raise_on_failure:
If True, the component raises an exception when the chat generator call fails or its response isn't valid
JSON with the expected keys. If False, it logs a warning and sets the result for that input to `None`.
JSON with the expected keys, or the numbers of statements and statement scores differ. If False, it logs
a warning, marks the result for that input as `error` with a NaN score, and excludes it from the mean.
:param chat_generator:
a ChatGenerator instance which represents the LLM.
In order for the component to work, the LLM should be configured to return a JSON object. For example,
Expand Down Expand Up @@ -214,6 +215,15 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:

# calculate average statement faithfulness score per query
for idx, res in enumerate(result["results"]):
if res is not None and len(res["statements"]) != len(res["statement_scores"]):
message = (
"FaithfulnessEvaluator expected one score per statement, but received "
f"{len(res['statements'])} statements and {len(res['statement_scores'])} scores."
)
if self.raise_on_failure:
raise ValueError(message)
logger.warning("{error} Setting the result for this input to an error.", error=message)
res = None
if res is None:
result["results"][idx] = {
"statements": [],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---
upgrade:
- |
Users whose chat generators return different numbers of ``statements`` and
``statement_scores`` to ``FaithfulnessEvaluator`` will now receive an error
instead of a score derived from incomplete or extra judgments. Ensure that
each extracted statement has exactly one score, or set
``raise_on_failure=False`` to report malformed rows as errors and continue
evaluating the remaining inputs.
fixes:
- |
``FaithfulnessEvaluator`` now rejects mismatched statement and score counts
consistently in synchronous and asynchronous evaluation. With
``raise_on_failure=False``, these rows have ``status="error"`` and a NaN score
and are excluded from the overall mean. Aligned lists, including two empty
lists, retain their existing behavior.
Comment thread
Jacqueline122 marked this conversation as resolved.
Resolved
102 changes: 102 additions & 0 deletions test/components/evaluators/test_faithfulness_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
#
# SPDX-License-Identifier: Apache-2.0

import json
import math
import os
from unittest.mock import Mock

import pytest

Expand Down Expand Up @@ -350,6 +352,106 @@ def test_live_run(self):
assert "total_tokens" in result["meta"][0]["usage"]


class TestFaithfulnessEvaluatorStatementAlignment:
Comment thread
Jacqueline122 marked this conversation as resolved.
Resolved
@pytest.fixture
def mock_judgments(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
mocked_run = Mock()

def run(self, **kwargs):
return mocked_run(**kwargs)

async def run_async(self, **kwargs):
return mocked_run(**kwargs)

monkeypatch.setattr(OpenAIChatGenerator, "run", run)
monkeypatch.setattr(OpenAIChatGenerator, "run_async", run_async)
return mocked_run

@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize("raise_on_failure", [False, True])
@pytest.mark.parametrize(
("statements", "statement_scores"),
[(["a", "b"], [1]), (["a"], [1, 0]), (["a"], []), ([], [1])],
ids=["missing-score", "extra-score", "no-scores", "no-statements"],
)
async def test_mismatched_counts_follow_failure_policy(
self, mock_judgments, use_async, raise_on_failure, statements, statement_scores
):
mock_judgments.return_value = {
"replies": [
ChatMessage.from_assistant(json.dumps({"statements": statements, "statement_scores": statement_scores}))
]
}
evaluator = FaithfulnessEvaluator(raise_on_failure=raise_on_failure, progress_bar=False)
inputs = {"questions": ["q"], "contexts": [["c"]], "predicted_answers": ["a"]}
if raise_on_failure:
message = (
f"FaithfulnessEvaluator expected one score per statement, but received "
f"{len(statements)} statements and {len(statement_scores)} scores."
)
with pytest.raises(ValueError, match=message):
if use_async:
await evaluator.run_async(**inputs)
else:
evaluator.run(**inputs)
else:
result = await evaluator.run_async(**inputs) if use_async else evaluator.run(**inputs)
assert result["results"][0]["status"] == "error"
assert result["results"][0]["statements"] == []
assert result["results"][0]["statement_scores"] == []
assert math.isnan(result["results"][0]["score"])
assert math.isnan(result["individual_scores"][0])
assert math.isnan(result["score"])

@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
async def test_mismatched_counts_are_excluded_from_batch_score(self, mock_judgments, use_async, caplog):
mock_judgments.side_effect = [
{"replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1]}')]},
{"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 0]}')]},
]
evaluator = FaithfulnessEvaluator(raise_on_failure=False, progress_bar=False)
inputs = {"questions": ["q1", "q2"], "contexts": [["c1"], ["c2"]], "predicted_answers": ["a1", "a2"]}
result = await evaluator.run_async(**inputs) if use_async else evaluator.run(**inputs)

assert result["score"] == 0.5
assert math.isnan(result["individual_scores"][0])
assert result["individual_scores"][1] == 0.5
assert result["results"][0]["status"] == "error"
assert result["results"][1] == {
"statements": ["c", "d"],
"statement_scores": [1, 0],
"score": 0.5,
"status": "evaluated",
}
assert "2 statements and 1 scores" in caplog.text
assert "1 query(s) failed and were excluded from the score." in caplog.text

@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize(
("payload", "score"),
[
({"statements": ["a", "b"], "statement_scores": [1, 0]}, 0.5),
({"statements": [], "statement_scores": []}, 0),
],
ids=["aligned", "empty"],
)
async def test_aligned_counts_remain_valid(self, mock_judgments, use_async, payload, score):
mock_judgments.return_value = {"replies": [ChatMessage.from_assistant(json.dumps(payload))]}
evaluator = FaithfulnessEvaluator(progress_bar=False)
inputs = {"questions": ["q"], "contexts": [["c"]], "predicted_answers": ["a"]}
result = await evaluator.run_async(**inputs) if use_async else evaluator.run(**inputs)
assert result == {
"score": score,
"individual_scores": [score],
"results": [{**payload, "score": score, "status": "evaluated"}],
"meta": None,
}


class TestFaithfulnessEvaluatorAsync:
@pytest.mark.asyncio
async def test_run_async_calculates_mean_score(self, monkeypatch):
Expand Down
Loading