From 41c4854fc9e77cf445f3b544a80b9a4eb7f9d3de Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 16 Nov 2026 17:23:55 -0800 Subject: [PATCH 1/6] feat(sources): let the CSV reader accept larger cells The csv module of Python stops at a cell of 131,072 characters by default. The M4-Caption release of OpenTSLM stores a whole series in one cell, and its largest cell has 169,126 characters. CsvDataReader now takes max_field_chars. Before the reader parses, it raises the limit of the csv module to that value. The limit applies to the whole process, so the reader never lowers it. A lower limit could stop another reader in the middle of a file. --- .../timenet_connectors/sources/readers/csv.py | 22 ++++++++++++++ .../tests/test_source_readers.py | 30 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index 3de79bdf5..0b34d468c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -18,9 +18,21 @@ def __init__( row_type: type[RowT], *, expected_columns: tuple[str, ...] | None = None, + max_field_chars: int | None = None, ) -> None: + """Set up a reader for one CSV file. + + Args: + path: The CSV file. + row_type: The schema of each row. + expected_columns: The exact header, in order, or ``None`` to accept any header. + max_field_chars: The largest cell that the file may hold, in characters, or ``None`` for the + limit of the ``csv`` module, which is 131,072 by default. Use it for a source whose cells + are known to be larger, such as long series in one cell. + """ super().__init__(path, row_type) self.expected_columns = expected_columns + self.max_field_chars = max_field_chars def iter_rows(self) -> Generator[RowT, None, None]: """Yield validated rows parsed from CSV cells. @@ -31,6 +43,7 @@ def iter_rows(self) -> Generator[RowT, None, None]: Raises: TimeNetConnectorError: If the file or a row is invalid. """ + self._allow_field_chars() try: with self.path.open(encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle, strict=True) @@ -50,6 +63,15 @@ def iter_rows(self) -> Generator[RowT, None, None]: except (OSError, UnicodeError, csv.Error) as exc: raise TimeNetConnectorError(f"{self.path}: {exc}") from exc + def _allow_field_chars(self) -> None: + """Raise the cell limit of the ``csv`` module to ``max_field_chars``, if it is lower. + + The ``csv`` module has one limit for the whole process, and it checks the limit while it parses. The + reader never lowers it, because another reader may be in the middle of a file with larger cells. + """ + if self.max_field_chars is not None and self.max_field_chars > csv.field_size_limit(): + csv.field_size_limit(self.max_field_chars) + def _has_expected_columns(self, header: Sequence[str]) -> bool: """Return whether the header has exactly the expected columns in order, or no columns are expected.""" return self.expected_columns is None or tuple(header) == self.expected_columns diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index 71aaa51e7..ff798011c 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -1,4 +1,5 @@ from contextlib import closing +import csv import json from pathlib import Path @@ -65,6 +66,35 @@ def test_csv_rejects_ambiguous_rows(tmp_path, content, message): list(CsvDataReader(path, Row).iter_rows()) +@pytest.fixture +def restore_csv_field_limit(): + # The limit of the csv module is process-wide, so restore it for the tests that run after. + previous = csv.field_size_limit() + yield + csv.field_size_limit(previous) + + +@pytest.mark.usefixtures("restore_csv_field_limit") +def test_csv_reads_a_cell_up_to_max_field_chars(tmp_path): + long_name = "x" * (csv.field_size_limit() + 1) + path = tmp_path / "source.csv" + path.write_text(f"name,count\n{long_name},1\n") + with pytest.raises(TimeNetConnectorError, match="field larger than field limit"): + list(CsvDataReader(path, Row).iter_rows()) + reader = CsvDataReader(path, Row, max_field_chars=len(long_name)) + assert list(reader.iter_rows()) == [Row(name=long_name, count=1)] + + +@pytest.mark.usefixtures("restore_csv_field_limit") +def test_csv_max_field_chars_never_lowers_the_process_limit(tmp_path): + # Another reader may be in the middle of a file with long cells, so a small maximum must not cut it off. + previous = csv.field_size_limit() + path = tmp_path / "source.csv" + path.write_text("name,count\nfirst,1\n") + list(CsvDataReader(path, Row, max_field_chars=10).iter_rows()) + assert csv.field_size_limit() == previous + + def test_csv_checks_expected_header_even_without_rows(tmp_path): path = tmp_path / "source.csv" path.write_text("count,name\n") From 96d10e7495c798d672adc4f3a96ebbe42dc3696e Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 09:19:18 -0800 Subject: [PATCH 2/6] refactor(sources): raise the CSV cell limit with a static method The cell limit of the csv module applies to the whole process. The max_field_chars option looked like a setting of one reader, but it changed the limit for every later reader too. Replace the option with CsvDataReader.raise_field_size_limit. The caller now raises the limit in plain sight before it reads. The method still never lowers the limit. --- .../timenet_connectors/sources/readers/csv.py | 35 ++++++++----------- .../tests/test_source_readers.py | 14 ++++---- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index 0b34d468c..d57efa986 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -18,21 +18,24 @@ def __init__( row_type: type[RowT], *, expected_columns: tuple[str, ...] | None = None, - max_field_chars: int | None = None, ) -> None: - """Set up a reader for one CSV file. + super().__init__(path, row_type) + self.expected_columns = expected_columns + + @staticmethod + def raise_field_size_limit(chars: int) -> None: + """Let every CSV reader in this process read cells of up to ``chars`` characters. + + The ``csv`` module has one cell limit for the whole process, 131,072 characters by default, and + it checks the limit while it parses. Call this method before reading a source whose cells are + larger, such as a long series in one cell. The method never lowers the limit, because another + reader may be in the middle of a file with larger cells. Args: - path: The CSV file. - row_type: The schema of each row. - expected_columns: The exact header, in order, or ``None`` to accept any header. - max_field_chars: The largest cell that the file may hold, in characters, or ``None`` for the - limit of the ``csv`` module, which is 131,072 by default. Use it for a source whose cells - are known to be larger, such as long series in one cell. + chars: The largest cell to accept, in characters. """ - super().__init__(path, row_type) - self.expected_columns = expected_columns - self.max_field_chars = max_field_chars + if chars > csv.field_size_limit(): + csv.field_size_limit(chars) def iter_rows(self) -> Generator[RowT, None, None]: """Yield validated rows parsed from CSV cells. @@ -43,7 +46,6 @@ def iter_rows(self) -> Generator[RowT, None, None]: Raises: TimeNetConnectorError: If the file or a row is invalid. """ - self._allow_field_chars() try: with self.path.open(encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle, strict=True) @@ -63,15 +65,6 @@ def iter_rows(self) -> Generator[RowT, None, None]: except (OSError, UnicodeError, csv.Error) as exc: raise TimeNetConnectorError(f"{self.path}: {exc}") from exc - def _allow_field_chars(self) -> None: - """Raise the cell limit of the ``csv`` module to ``max_field_chars``, if it is lower. - - The ``csv`` module has one limit for the whole process, and it checks the limit while it parses. The - reader never lowers it, because another reader may be in the middle of a file with larger cells. - """ - if self.max_field_chars is not None and self.max_field_chars > csv.field_size_limit(): - csv.field_size_limit(self.max_field_chars) - def _has_expected_columns(self, header: Sequence[str]) -> bool: """Return whether the header has exactly the expected columns in order, or no columns are expected.""" return self.expected_columns is None or tuple(header) == self.expected_columns diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index ff798011c..347ca45bc 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -75,23 +75,21 @@ def restore_csv_field_limit(): @pytest.mark.usefixtures("restore_csv_field_limit") -def test_csv_reads_a_cell_up_to_max_field_chars(tmp_path): +def test_csv_reads_a_cell_up_to_the_raised_field_size_limit(tmp_path): long_name = "x" * (csv.field_size_limit() + 1) path = tmp_path / "source.csv" path.write_text(f"name,count\n{long_name},1\n") with pytest.raises(TimeNetConnectorError, match="field larger than field limit"): list(CsvDataReader(path, Row).iter_rows()) - reader = CsvDataReader(path, Row, max_field_chars=len(long_name)) - assert list(reader.iter_rows()) == [Row(name=long_name, count=1)] + CsvDataReader.raise_field_size_limit(len(long_name)) + assert list(CsvDataReader(path, Row).iter_rows()) == [Row(name=long_name, count=1)] @pytest.mark.usefixtures("restore_csv_field_limit") -def test_csv_max_field_chars_never_lowers_the_process_limit(tmp_path): - # Another reader may be in the middle of a file with long cells, so a small maximum must not cut it off. +def test_raise_field_size_limit_never_lowers_the_process_limit(): + # Another reader may be in the middle of a file with long cells, so a small limit must not cut it off. previous = csv.field_size_limit() - path = tmp_path / "source.csv" - path.write_text("name,count\nfirst,1\n") - list(CsvDataReader(path, Row, max_field_chars=10).iter_rows()) + CsvDataReader.raise_field_size_limit(10) assert csv.field_size_limit() == previous From 15e7ac837b8f22418e7ce21c5d4cbae8d375259e Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 09:24:06 -0800 Subject: [PATCH 3/6] refactor(sources): set the CSV cell limit, not only raise it Rename CsvDataReader.raise_field_size_limit to set_field_size_limit. It now sets the limit to the given value, also when the value is lower. No caller reads two CSV files with different limits at the same time. --- .../src/timenet_connectors/sources/readers/csv.py | 8 +++----- .../timenet-connectors/tests/test_source_readers.py | 12 ++---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index d57efa986..abf9c3169 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -23,19 +23,17 @@ def __init__( self.expected_columns = expected_columns @staticmethod - def raise_field_size_limit(chars: int) -> None: + def set_field_size_limit(chars: int) -> None: """Let every CSV reader in this process read cells of up to ``chars`` characters. The ``csv`` module has one cell limit for the whole process, 131,072 characters by default, and it checks the limit while it parses. Call this method before reading a source whose cells are - larger, such as a long series in one cell. The method never lowers the limit, because another - reader may be in the middle of a file with larger cells. + larger, such as a long series in one cell. Args: chars: The largest cell to accept, in characters. """ - if chars > csv.field_size_limit(): - csv.field_size_limit(chars) + csv.field_size_limit(chars) def iter_rows(self) -> Generator[RowT, None, None]: """Yield validated rows parsed from CSV cells. diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index 347ca45bc..36b89d920 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -75,24 +75,16 @@ def restore_csv_field_limit(): @pytest.mark.usefixtures("restore_csv_field_limit") -def test_csv_reads_a_cell_up_to_the_raised_field_size_limit(tmp_path): +def test_csv_reads_a_cell_up_to_the_set_field_size_limit(tmp_path): long_name = "x" * (csv.field_size_limit() + 1) path = tmp_path / "source.csv" path.write_text(f"name,count\n{long_name},1\n") with pytest.raises(TimeNetConnectorError, match="field larger than field limit"): list(CsvDataReader(path, Row).iter_rows()) - CsvDataReader.raise_field_size_limit(len(long_name)) + CsvDataReader.set_field_size_limit(len(long_name)) assert list(CsvDataReader(path, Row).iter_rows()) == [Row(name=long_name, count=1)] -@pytest.mark.usefixtures("restore_csv_field_limit") -def test_raise_field_size_limit_never_lowers_the_process_limit(): - # Another reader may be in the middle of a file with long cells, so a small limit must not cut it off. - previous = csv.field_size_limit() - CsvDataReader.raise_field_size_limit(10) - assert csv.field_size_limit() == previous - - def test_csv_checks_expected_header_even_without_rows(tmp_path): path = tmp_path / "source.csv" path.write_text("count,name\n") From ce63363fe72ea5480cb45f008a285c1dff747c7c Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 10:40:27 -0800 Subject: [PATCH 4/6] fix(sources): reject a CSV cell limit below 1 The csv module accepts zero or a negative limit, and then rejects every nonempty cell. set_field_size_limit now raises TimeFValidationError for a limit below 1 and leaves the limit as it was. --- .../src/timenet_connectors/sources/readers/csv.py | 8 +++++++- .../timenet-connectors/tests/test_source_readers.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index abf9c3169..0b4158aa5 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import TypeGuard -from timenet.errors import TimeNetConnectorError +from timenet.errors import TimeFValidationError, TimeNetConnectorError from timenet_connectors.sources.readers.base import DataReader, RowT @@ -32,7 +32,13 @@ def set_field_size_limit(chars: int) -> None: Args: chars: The largest cell to accept, in characters. + + Raises: + TimeFValidationError: If ``chars`` is less than 1. """ + # The csv module accepts zero or a negative limit, and then rejects every nonempty cell. + if chars < 1: + raise TimeFValidationError(f"the field size limit must be at least 1 character, got {chars}") csv.field_size_limit(chars) def iter_rows(self) -> Generator[RowT, None, None]: diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index 36b89d920..15b7181bd 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -8,7 +8,7 @@ from pydantic import ConfigDict, ValidationError import pytest -from timenet.errors import TimeNetConnectorError +from timenet.errors import TimeFValidationError, TimeNetConnectorError from timenet_connectors.sources.readers import ( CsvDataReader, DataRow, @@ -85,6 +85,15 @@ def test_csv_reads_a_cell_up_to_the_set_field_size_limit(tmp_path): assert list(CsvDataReader(path, Row).iter_rows()) == [Row(name=long_name, count=1)] +@pytest.mark.usefixtures("restore_csv_field_limit") +@pytest.mark.parametrize("chars", [0, -5]) +def test_csv_field_size_limit_must_be_positive(chars): + previous = csv.field_size_limit() + with pytest.raises(TimeFValidationError, match=f"field size limit must be at least 1 character, got {chars}"): + CsvDataReader.set_field_size_limit(chars) + assert csv.field_size_limit() == previous + + def test_csv_checks_expected_header_even_without_rows(tmp_path): path = tmp_path / "source.csv" path.write_text("count,name\n") From 73b330272b9d9c53d48909fe7a58ca8a5c531f19 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 17:14:29 -0800 Subject: [PATCH 5/6] refactor(sources): lift the CSV cell limit in the reader CsvDataReader now lifts the cell limit of the csv module before it reads, so no connector has to set it. The set_field_size_limit method goes away. --- .../timenet_connectors/sources/readers/csv.py | 31 +++++++------------ .../tests/test_source_readers.py | 30 +++++++----------- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index 0b4158aa5..fa904c15f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -3,12 +3,18 @@ from collections.abc import Generator, Mapping, Sequence import csv from pathlib import Path +import sys from typing import TypeGuard -from timenet.errors import TimeFValidationError, TimeNetConnectorError +from timenet.errors import TimeNetConnectorError from timenet_connectors.sources.readers.base import DataReader, RowT +_NO_FIELD_SIZE_LIMIT = min(sys.maxsize, 2**31 - 1) +"""The largest cell limit that the csv module accepts on every platform. On Windows, the limit is a +32-bit C long, and a larger value raises OverflowError.""" + + class CsvDataReader(DataReader[RowT]): """Read a UTF-8 CSV file with a header.""" @@ -22,34 +28,19 @@ def __init__( super().__init__(path, row_type) self.expected_columns = expected_columns - @staticmethod - def set_field_size_limit(chars: int) -> None: - """Let every CSV reader in this process read cells of up to ``chars`` characters. - - The ``csv`` module has one cell limit for the whole process, 131,072 characters by default, and - it checks the limit while it parses. Call this method before reading a source whose cells are - larger, such as a long series in one cell. - - Args: - chars: The largest cell to accept, in characters. - - Raises: - TimeFValidationError: If ``chars`` is less than 1. - """ - # The csv module accepts zero or a negative limit, and then rejects every nonempty cell. - if chars < 1: - raise TimeFValidationError(f"the field size limit must be at least 1 character, got {chars}") - csv.field_size_limit(chars) - def iter_rows(self) -> Generator[RowT, None, None]: """Yield validated rows parsed from CSV cells. + A cell can have any length. A source can hold a long series in one cell. + Yields: Validated source rows. Raises: TimeNetConnectorError: If the file or a row is invalid. """ + # The csv module limits each cell to 131,072 characters, for the whole process. + csv.field_size_limit(_NO_FIELD_SIZE_LIMIT) try: with self.path.open(encoding="utf-8", newline="") as handle: reader = csv.DictReader(handle, strict=True) diff --git a/packages/timenet-connectors/tests/test_source_readers.py b/packages/timenet-connectors/tests/test_source_readers.py index 15b7181bd..27caf7599 100644 --- a/packages/timenet-connectors/tests/test_source_readers.py +++ b/packages/timenet-connectors/tests/test_source_readers.py @@ -8,7 +8,7 @@ from pydantic import ConfigDict, ValidationError import pytest -from timenet.errors import TimeFValidationError, TimeNetConnectorError +from timenet.errors import TimeNetConnectorError from timenet_connectors.sources.readers import ( CsvDataReader, DataRow, @@ -66,34 +66,26 @@ def test_csv_rejects_ambiguous_rows(tmp_path, content, message): list(CsvDataReader(path, Row).iter_rows()) +_CSV_MODULE_DEFAULT_LIMIT = 128 * 1024 +"""The default cell limit of the csv module, in characters.""" + + @pytest.fixture -def restore_csv_field_limit(): - # The limit of the csv module is process-wide, so restore it for the tests that run after. - previous = csv.field_size_limit() +def csv_module_default_limit(): + # The limit of the csv module is process-wide, so start from the default and restore the limit after. + previous = csv.field_size_limit(_CSV_MODULE_DEFAULT_LIMIT) yield csv.field_size_limit(previous) -@pytest.mark.usefixtures("restore_csv_field_limit") -def test_csv_reads_a_cell_up_to_the_set_field_size_limit(tmp_path): - long_name = "x" * (csv.field_size_limit() + 1) +@pytest.mark.usefixtures("csv_module_default_limit") +def test_csv_reads_a_cell_longer_than_the_default_limit_of_the_csv_module(tmp_path): + long_name = "x" * (_CSV_MODULE_DEFAULT_LIMIT + 1) path = tmp_path / "source.csv" path.write_text(f"name,count\n{long_name},1\n") - with pytest.raises(TimeNetConnectorError, match="field larger than field limit"): - list(CsvDataReader(path, Row).iter_rows()) - CsvDataReader.set_field_size_limit(len(long_name)) assert list(CsvDataReader(path, Row).iter_rows()) == [Row(name=long_name, count=1)] -@pytest.mark.usefixtures("restore_csv_field_limit") -@pytest.mark.parametrize("chars", [0, -5]) -def test_csv_field_size_limit_must_be_positive(chars): - previous = csv.field_size_limit() - with pytest.raises(TimeFValidationError, match=f"field size limit must be at least 1 character, got {chars}"): - CsvDataReader.set_field_size_limit(chars) - assert csv.field_size_limit() == previous - - def test_csv_checks_expected_header_even_without_rows(tmp_path): path = tmp_path / "source.csv" path.write_text("count,name\n") From b382edee6e56ce74f8fc7a57a5fbb2515cd6d740 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 17 Nov 2026 17:20:44 -0800 Subject: [PATCH 6/6] docs(sources): drop the cell-length note from the CSV reader --- .../src/timenet_connectors/sources/readers/csv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py index fa904c15f..45382e255 100644 --- a/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py +++ b/packages/timenet-connectors/src/timenet_connectors/sources/readers/csv.py @@ -31,8 +31,6 @@ def __init__( def iter_rows(self) -> Generator[RowT, None, None]: """Yield validated rows parsed from CSV cells. - A cell can have any length. A source can hold a long series in one cell. - Yields: Validated source rows.