diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index 3de79bdf5..45382e255 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -3,12 +3,18 @@ from collections.abc import Generator, Mapping, Sequence import csv from pathlib import Path +import sys from typing import TypeGuard from timenet.errors import TimeNetConnectorError from timenet_connectors.sources.readers.base import DataReader, RowT +_NO_FIELD_SIZE_LIMIT = min(sys.maxsize, 2**31 - 1) +"""The largest cell limit that the csv module accepts on every platform. On Windows, the limit is a +32-bit C long, and a larger value raises OverflowError.""" + + class CsvDataReader(DataReader[RowT]): """Read a UTF-8 CSV file with a header.""" @@ -31,6 +37,8 @@ def iter_rows(self) -> Generator[RowT, None, None]: Raises: TimeNetConnectorError: If the file or a row is invalid. """ + # The csv module limits each cell to 131,072 characters, for the whole process. + csv.field_size_limit(_NO_FIELD_SIZE_LIMIT) try: with self.path.open(encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle, strict=True) diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index 71aaa51e7..27caf7599 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -1,4 +1,5 @@ from contextlib import closing +import csv import json from pathlib import Path @@ -65,6 +66,26 @@ def test_csv_rejects_ambiguous_rows(tmp_path, content, message): list(CsvDataReader(path, Row).iter_rows()) +_CSV_MODULE_DEFAULT_LIMIT = 128 * 1024 +"""The default cell limit of the csv module, in characters.""" + + +@pytest.fixture +def csv_module_default_limit(): + # The limit of the csv module is process-wide, so start from the default and restore the limit after. + previous = csv.field_size_limit(_CSV_MODULE_DEFAULT_LIMIT) + yield + csv.field_size_limit(previous) + + +@pytest.mark.usefixtures("csv_module_default_limit") +def test_csv_reads_a_cell_longer_than_the_default_limit_of_the_csv_module(tmp_path): + long_name = "x" * (_CSV_MODULE_DEFAULT_LIMIT + 1) + path = tmp_path / "source.csv" + path.write_text(f"name,count\n{long_name},1\n") + assert list(CsvDataReader(path, Row).iter_rows()) == [Row(name=long_name, count=1)] + + def test_csv_checks_expected_header_even_without_rows(tmp_path): path = tmp_path / "source.csv" path.write_text("count,name\n")