Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion haystack/components/converters/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def _extract_frontmatter(self, file_content: str, source: str | Path | ByteStrea

frontmatter_text = match.group("frontmatter")
try:
frontmatter = json.loads(json.dumps(yaml.safe_load(frontmatter_text), default=str)) or {}
frontmatter = json.loads(json.dumps(yaml.safe_load(frontmatter_text), default=str))
except yaml.YAMLError as error:
logger.warning(
"Could not parse YAML frontmatter in {source}. Keeping it as content. Error: {error}",
Expand All @@ -185,6 +185,9 @@ def _extract_frontmatter(self, file_content: str, source: str | Path | ByteStrea
)
return file_content, {}

if frontmatter is None:
frontmatter = {}

if not isinstance(frontmatter, dict):
logger.warning(
"Ignoring YAML frontmatter in {source}: expected a mapping, got {kind}.",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
fixes:
- |
``MarkdownToDocument(extract_frontmatter=True)`` now preserves frontmatter containing
``false``, ``0``, ``[]``, or an empty string as document content and logs a warning,
just like other non-mapping values. Previously these blocks were silently removed.
Empty/null frontmatter and empty mappings continue to be removed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0

import logging

import pytest

from haystack.components.converters.markdown import MarkdownToDocument
from haystack.dataclasses import ByteStream


class TestMarkdownToDocumentFrontmatter:
@pytest.mark.parametrize("frontmatter", ["false", "0", "[]", '""', "true", "[item]"])
def test_run_keeps_non_mapping_frontmatter_as_content_and_logs_warning(self, frontmatter, caplog):
source = ByteStream.from_string(f"---\n{frontmatter}\n---\n# Body\nKeep this text.\n")
expected = MarkdownToDocument(progress_bar=False).run(sources=[source])["documents"][0]
converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)

with caplog.at_level(logging.WARNING):
documents = converter.run(sources=[source])["documents"]

assert len(documents) == 1
assert documents[0].content == expected.content
assert documents[0].meta == {}
assert "expected a mapping" in caplog.text

@pytest.mark.parametrize("frontmatter", ["", "null", "{}"])
def test_run_removes_empty_frontmatter(self, frontmatter, caplog):
source = ByteStream.from_string(f"---\n{frontmatter}\n---\n# Body\nKeep this text.\n")
expected = MarkdownToDocument(progress_bar=False).run(
sources=[ByteStream.from_string("# Body\nKeep this text.\n")]
)["documents"][0]
converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)

with caplog.at_level(logging.WARNING):
documents = converter.run(sources=[source])["documents"]

assert len(documents) == 1
assert documents[0].content == expected.content
assert documents[0].meta == {}
assert not caplog.records