Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ hep_available: false

# FaithfulnessEvaluator

## Overview

The `FaithfulnessEvaluator` uses an LLM to evaluate whether a generated answer can be inferred from the provided contexts. It does not require ground truth labels. This metric is called faithfulness, sometimes also referred to as groundedness or hallucination.

<div className="key-value-table">
Expand All @@ -33,7 +35,7 @@ If you are not initializing the Evaluator with your own Chat Generator other tha

Two other optional initialization parameters are:

- `raise_on_failure`: If True, raise an exception when the chat generator call fails or its response isn't valid JSON with the expected keys. If False, log a warning and set the result for that input to `None`.
- `raise_on_failure`: If True, raise an exception when the chat generator call fails, its response isn't valid JSON with the expected keys, or the numbers of `statements` and `statement_scores` differ. If False, log a warning and return `status="error"` with a NaN score for the failed input. Failed inputs are excluded from the overall mean; if all inputs fail, the overall score is NaN.
- `progress_bar`: Whether to show a progress bar during the evaluation.

`FaithfulnessEvaluator` has an optional `examples` parameter that can be used to pass few-shot examples conforming to the expected input and output format of `FaithfulnessEvaluator`. These examples are included in the prompt that is sent to the LLM. Examples, therefore, increase the number of tokens of the prompt and make each request more costly. Adding examples is helpful if you want to improve the quality of the evaluation at the cost of more tokens.
Expand Down
12 changes: 11 additions & 1 deletion haystack/components/evaluators/faithfulness.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ def __init__(
Whether to show a progress bar during the evaluation.
:param raise_on_failure:
If True, the component raises an exception when the chat generator call fails or its response isn't valid
JSON with the expected keys. If False, it logs a warning and sets the result for that input to `None`.
JSON with the expected keys, or the numbers of statements and statement scores differ. If False, it logs
a warning, marks the result for that input as `error` with a NaN score, and excludes it from the mean.
:param chat_generator:
a ChatGenerator instance which represents the LLM.
In order for the component to work, the LLM should be configured to return a JSON object. For example,
Expand Down Expand Up @@ -214,6 +215,15 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:

# calculate average statement faithfulness score per query
for idx, res in enumerate(result["results"]):
if res is not None and len(res["statements"]) != len(res["statement_scores"]):
message = (
"FaithfulnessEvaluator expected one score per statement, but received "
f"{len(res['statements'])} statements and {len(res['statement_scores'])} scores."
)
if self.raise_on_failure:
raise ValueError(message)
logger.warning("{error} Setting the result for this input to an error.", error=message)
res = None
if res is None:
result["results"][idx] = {
"statements": [],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
upgrade:
- |
``FaithfulnessEvaluator`` now raises a ``ValueError`` when the LLM returns a different number of
``statements`` and ``statement_scores``. Set ``raise_on_failure=False`` to mark these inputs as errors
and exclude them from the score instead.
26 changes: 24 additions & 2 deletions test/components/evaluators/test_faithfulness_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ def test_run_missing_parameters(self, monkeypatch):
with pytest.raises(ValueError, match="LLM evaluator expected input parameter"):
component.run()

@pytest.mark.parametrize("failure_mode", ["generation", "parsing"])
@pytest.mark.parametrize("failure_mode", ["generation", "parsing", "mismatch"])
def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog, failure_mode):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
component = FaithfulnessEvaluator(raise_on_failure=False)
Expand All @@ -265,6 +265,10 @@ def chat_generator_run(self, *args, **kwargs):
if "Python" in kwargs["messages"][0].text:
if failure_mode == "generation":
raise Exception("OpenAI API request failed.")
if failure_mode == "mismatch":
return {
"replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1]}')]
}
return {"replies": [ChatMessage.from_assistant("not valid JSON")]}
return {"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 1]}')]}

Expand Down Expand Up @@ -309,7 +313,25 @@ def chat_generator_run(self, *args, **kwargs):
assert results["results"][1]["status"] == "error"

assert "1 query(s) failed and were excluded from the score." in caplog.text
assert "FaithfulnessEvaluator failed for 1 out of 2 inputs." in caplog.text
if failure_mode == "mismatch":
assert "2 statements and 1 scores" in caplog.text
else:
assert "FaithfulnessEvaluator failed for 1 out of 2 inputs." in caplog.text

def test_run_raises_on_mismatched_statement_score_counts(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
component = FaithfulnessEvaluator()

def chat_generator_run(self, *args, **kwargs):
return {"replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1]}')]}

monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)

with pytest.raises(
ValueError,
match=r"^FaithfulnessEvaluator expected one score per statement, but received 2 statements and 1 scores\.$",
):
component.run(questions=["q"], contexts=[["c"]], predicted_answers=["a"])

def test_run_all_failures_returns_nan_and_error_statuses(self, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
Expand Down
Loading