From 48044d584b43901900fbb813b2b2b0bb842feb13 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 Oct 2026 20:01:05 +1300 Subject: [PATCH 1/4] fix(adapters): bind and read every double exactly so find, count, cursors and deletes agree find() and bulk deletes bound a double as %.17F, which rounds 5e-324 to 0, while every other statement let PDO write it at the precision ini, 14 significant digits, which stores 0.30000000000000004 as 0.3. A filter then matched different rows in find() and count(), a cursor along a double order repeated or skipped rows, and a write lost digits. Every SQL bind now writes a double as the shortest locale-independent decimal that reads back as exactly that double, and every document fetch runs at the round-trip precision, because PDO stringifies a fetched double from pgsql and sqlite at the precision ini too. A filter or having value that is infinite or NaN is refused as an invalid query, as writes already were. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/SQL.php | 101 ++++--- src/Database/Adapter/SQLite.php | 4 +- src/Database/Validator/Finite.php | 27 ++ src/Database/Validator/Query/Filter.php | 4 +- src/Database/Validator/Query/Having.php | 4 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 253 +++++++++++++++--- tests/unit/Adapter/BuildsAggregates.php | 2 +- tests/unit/Adapter/FloatBindingTest.php | 165 +++++++++++- .../Documents/DeleteDocumentsRaceTest.php | 2 +- tests/unit/Validator/FiniteTest.php | 89 ++++++ tests/unit/Validator/QueryShapeHavingTest.php | 12 + 11 files changed, 566 insertions(+), 97 deletions(-) create mode 100644 src/Database/Validator/Finite.php create mode 100644 tests/unit/Validator/FiniteTest.php diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 36ef14a21..d76ce1db8 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -124,12 +124,11 @@ abstract class SQL extends Adapter implements Feature\Connection, Feature\RawQue private const int UNPAIRED_JOINED_ROWS = 4; - protected object $pdo; + private const array SHORT_FLOAT_FORMATS = ['%.15H', '%.16H']; - /** - * Controls how many fractional digits are used when binding float parameters. - */ - protected int $floatPrecision = 17; + private const string ROUND_TRIP_PRECISION = '-1'; + + protected object $pdo; /** * Lazily constructed AttributeMap shared by every builder's scope. @@ -209,11 +208,57 @@ public function getDriver(): object } /** - * Helper to format a float value according to configured precision for binding/logging. + * The shortest decimal that reads back as exactly this double. `%H` ignores the locale, and a double needs at + * most 17 significant digits to round-trip. */ - protected function getFloatPrecision(float $value): string + protected function formatFloat(float $value): string { - return sprintf('%.'.$this->floatPrecision.'F', $value); + foreach (self::SHORT_FLOAT_FORMATS as $format) { + $formatted = \sprintf($format, $value); + if ((float) $formatted === $value) { + return $formatted; + } + } + + return \sprintf('%.17H', $value); + } + + /** + * Binds a value as its PDO type, and a double as its exact decimal: PDO would write it at the `precision` ini. + * + * @param PDOStatement|DatabasePDOStatement|PDOStatementProxy $statement + */ + protected function bind(object $statement, int|string $parameter, mixed $value): void + { + if (\is_float($value)) { + $statement->bindValue($parameter, $this->formatFloat($value), PDO::PARAM_STR); + + return; + } + + $statement->bindValue($parameter, $value, $this->getPdoType($value)); + } + + /** + * Runs a fetch at the precision that round-trips every double. PDO stringifies a double it fetches at the + * `precision` ini, 14 significant digits by default. + * + * @template T + * + * @param callable(): T $fetch + * @return T + */ + protected function fetchExactly(callable $fetch): mixed + { + $precision = \ini_set('precision', self::ROUND_TRIP_PRECISION); + + try { + return $fetch(); + } finally { + if ($precision !== false) { + \ini_set('precision', $precision); + } + } } #[\Override] @@ -621,12 +666,12 @@ public function getDocument(Document $collection, string $id, array $queries = [ try { $statement = $this->prepareStatement($sql, Event::DocumentRead); foreach ($bindings as $parameter => $value) { - $statement->bindValue($parameter, $value, $this->getPdoType($value)); + $this->bind($statement, $parameter, $value); } $this->describeStatement($statement, $bindings, $name); $this->execute($statement); /** @var array|false $row */ - $row = $statement->fetch(PDO::FETCH_ASSOC); + $row = $this->fetchExactly(static fn (): mixed => $statement->fetch(PDO::FETCH_ASSOC)); } catch (PDOException $e) { $exception = $e; } finally { @@ -984,7 +1029,7 @@ private function readRows(string $name, array $documents, array $columns): array $statement = $this->executeResult($builder->build(), Event::DocumentRead); $this->execute($statement); /** @var list> $rows */ - $rows = $statement->fetchAll(PDO::FETCH_NUM); + $rows = $this->fetchExactly(static fn (): array => $statement->fetchAll(PDO::FETCH_NUM)); $statement->closeCursor(); return $rows; @@ -1536,7 +1581,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $this->describeStatement($statement, [], $name); $this->execute($statement); /** @var array> $rows */ - $rows = $statement->fetchAll(); + $rows = $this->fetchExactly(static fn (): array => $statement->fetchAll()); } catch (PDOException $e) { $exception = $e; } finally { @@ -1837,14 +1882,14 @@ public function rawQuery(string $query, array $bindings = []): array try { $statement = $this->prepareStatement($query); foreach ($bindings as $i => $value) { - $statement->bindValue($i + 1, $value, $this->getPdoType($value)); + $this->bind($statement, $i + 1, $value); } $this->execute($statement); } catch (PDOException $e) { throw $this->processException($e); } - $results = $statement->fetchAll(); + $results = $this->fetchExactly(static fn (): array => $statement->fetchAll()); $statement->closeCursor(); $documents = []; @@ -3552,7 +3597,7 @@ public function rawMutation(string $query, array $bindings = []): int try { $statement = $this->prepareStatement($query); foreach ($bindings as $i => $value) { - $statement->bindValue($i + 1, $value, $this->getPdoType($value)); + $this->bind($statement, $i + 1, $value); } $this->execute($statement); } catch (PDOException $e) { @@ -3661,10 +3706,6 @@ protected function writeContext(array $skipPermissions = []): WriteContext * Prepares the SQL statement and binds positional parameters from the Statement. * Does NOT call execute() - the caller is responsible for that. * - * Floats are bound as 7.x bound them: a find() binds them in fixed-point notation, every other statement as - * PHP writes them, so a write keeps a magnitude fixed point would round to zero. A bulk delete binds them as a - * find() does, so its filters match the rows the find() of the documents it deletes matched. - * * @param string $collection The collection the statement reads or writes, for the profiler * @return PDOStatement|DatabasePDOStatement|PDOStatementProxy */ @@ -3676,13 +3717,7 @@ protected function executeResult(Statement $result, ?Event $event = null, string if (\is_bool($value) && $this->supports(Capability::IntegerBooleans)) { $value = (int) $value; } - if (\is_float($value) && ($event === Event::DocumentFind || $event === Event::DocumentsDelete)) { - $prepared->bindValue($i + 1, $this->getFloatPrecision($value), PDO::PARAM_STR); - } elseif (\is_float($value)) { - $prepared->bindValue($i + 1, $value, PDO::PARAM_STR); - } else { - $prepared->bindValue($i + 1, $value, $this->getPdoType($value)); - } + $this->bind($prepared, $i + 1, $value); } return $prepared; @@ -6266,7 +6301,7 @@ private function runSelect(Statement $result, Event $event, string $collection): $statement = $this->executeResult($result, $event, $collection); $this->execute($statement); /** @var array> $results */ - $results = $statement->fetchAll(PDO::FETCH_ASSOC); + $results = $this->fetchExactly(static fn (): array => $statement->fetchAll(PDO::FETCH_ASSOC)); } catch (PDOException $e) { $exception = $e; } finally { @@ -6520,13 +6555,13 @@ protected function bindOperatorParameters(object $statement, Operator $operator, case OperatorType::Divide: $value = $values[0] ?? 1; $bindKey = "op_{$bindIndex}"; - $statement->bindValue(':'.$bindKey, $value, $this->getPdoType($value)); + $this->bind($statement, ':'.$bindKey, $value); $bindIndex++; if (isset($values[1])) { $limitKey = "op_{$bindIndex}"; $limit = self::exactLimit($values[1]); - $statement->bindValue(':'.$limitKey, $limit, $this->getPdoType($limit)); + $this->bind($statement, ':'.$limitKey, $limit); $bindIndex++; } break; @@ -6534,20 +6569,20 @@ protected function bindOperatorParameters(object $statement, Operator $operator, case OperatorType::Modulo: $value = $values[0] ?? 1; $bindKey = "op_{$bindIndex}"; - $statement->bindValue(':'.$bindKey, $value, $this->getPdoType($value)); + $this->bind($statement, ':'.$bindKey, $value); $bindIndex++; break; case OperatorType::Power: $value = $values[0] ?? 1; $bindKey = "op_{$bindIndex}"; - $statement->bindValue(':'.$bindKey, $value, $this->getPdoType($value)); + $this->bind($statement, ':'.$bindKey, $value); $bindIndex++; if (isset($values[1])) { $maxKey = "op_{$bindIndex}"; $limit = self::exactLimit($values[1]); - $statement->bindValue(':'.$maxKey, $limit, $this->getPdoType($limit)); + $this->bind($statement, ':'.$maxKey, $limit); $bindIndex++; } break; @@ -6596,7 +6631,7 @@ protected function bindOperatorParameters(object $statement, Operator $operator, if (is_array($value)) { $value = json_encode($value); } - $statement->bindValue(':'.$bindKey, $value, $this->getPdoType($value)); + $this->bind($statement, ':'.$bindKey, $value); $bindIndex++; break; diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 1922b4b75..fd6d7883b 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1805,7 +1805,7 @@ protected function bindOperatorParameters(object $statement, Operator $operator, if (in_array($filterType, $comparisonTypes)) { $bindKey = "op_{$bindIndex}"; $value = (is_bool($filterValue)) ? (int) $filterValue : $filterValue; - $statement->bindValue(":{$bindKey}", $value, $this->getPdoType($value)); + $this->bind($statement, ":{$bindKey}", $value); $bindIndex++; } } @@ -2408,7 +2408,7 @@ protected function executeUpsertBatch( ); foreach ($bindValues as $key => $binding) { - $statement->bindValue($key, $binding, $this->getPdoType($binding)); + $this->bind($statement, $key, $binding); } $opIndexForBinding = 0; diff --git a/src/Database/Validator/Finite.php b/src/Database/Validator/Finite.php new file mode 100644 index 000000000..10392c8b4 --- /dev/null +++ b/src/Database/Validator/Finite.php @@ -0,0 +1,27 @@ +isValid($value)) { $this->message = 'Query value is invalid for aggregate alias "'.$alias.'"'; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 6a18359f9..82c376ac0 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -9,9 +9,9 @@ use Utopia\Cache\Adapter\None as NoneCacheAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Feature; -use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Adapter\Memory; use Utopia\Database\Adapter\Mongo; +use Utopia\Database\Adapter\MySQL; use Utopia\Database\Adapter\Postgres; use Utopia\Database\Adapter\Redis; use Utopia\Database\Adapter\SQL; @@ -39,6 +39,7 @@ use Utopia\Database\Id; use Utopia\Database\Index; use Utopia\Database\IntegerWidth; +use Utopia\Database\Operator; use Utopia\Database\PDO as DatabasePDO; use Utopia\Database\Permission; use Utopia\Database\PermissionType; @@ -2240,18 +2241,6 @@ private function createPhraseSearchCollection(Database $database, string $collec return $collection; } - private function engineFamily(Database $database): ?string - { - $adapter = $database->getAdapter(); - - return match (true) { - $adapter instanceof MariaDB => 'mariadb', - $adapter instanceof Postgres => 'postgres', - $adapter instanceof Mongo => 'mongo', - default => null, - }; - } - /** * @return list */ @@ -2863,52 +2852,230 @@ public function testConcurrentIncrementsOnMongoDBAllLand(): void } } - public function testFloatsAreStoredAs7xBoundThem(): void + public function testAFloatFilterMatchesTheSameRowsInFindCountSumAndDelete(): void { /** @var Database $database */ $database = $this->getDatabase(); - $family = $this->engineFamily($database); - if ($family === null) { - $this->expectNotToPerformAssertions(); - return; + $collection = $this->createFloatCollection($database, 'float_filters'); + $documents = []; + foreach (['sum' => [0.1 + 0.2, 20], 'zero' => [0.0, 20], 'denormal' => [5e-324, 10]] as $name => [$value, $rows]) { + for ($i = 0; $i < $rows; $i++) { + $documents[] = new Document(['$id' => $name.$i, 'f' => $value]); + } } + $database->createDocuments($collection, $documents); - $collection = 'float_binding'; - $database->createCollection(Collection::create(id: $collection, permissions: [ - Permission::create(Role::any()), - Permission::read(Role::any()), - Permission::update(Role::any()), - ], documentSecurity: false)); - $database->createAttribute($collection, Attribute::double(key: 'f')); + $filters = [ + 'equal sum' => [[Query::equal('f', [0.1 + 0.2])], 20], + 'equal denormal' => [[Query::equal('f', [5e-324])], 10], + 'equal zero' => [[Query::equal('f', [0.0])], 20], + 'not equal denormal' => [[Query::notEqual('f', 5e-324)], 40], + 'above the short decimal' => [[Query::greaterThan('f', 0.3)], 20], + 'below the long decimal' => [[Query::lessThan('f', 0.1 + 0.2)], 30], + 'between zero and a denormal' => [[Query::greaterThan('f', 0.0), Query::lessThanEqual('f', 5e-324)], 10], + 'between the decimals' => [[Query::between('f', 0.3, 0.1 + 0.2)], 20], + ]; + foreach ($filters as $name => [$queries, $rows]) { + $this->assertCount($rows, $database->find($collection, [...$queries, Query::limit(100)]), 'find '.$name); + $this->assertSame($rows, $database->count($collection, $queries), 'count '.$name); + } + + $this->assertEqualsWithDelta(20 * (0.1 + 0.2), $database->sum($collection, 'f', [Query::equal('f', [0.1 + 0.2])]), 1e-12); + $this->assertSame(5e-324 * 10, $database->sum($collection, 'f', [Query::equal('f', [5e-324])])); + + $this->assertSame(10, $database->deleteDocuments($collection, [Query::equal('f', [5e-324])])); + $this->assertSame(20, $database->count($collection, [Query::equal('f', [0.0])])); + $this->assertSame(20, $database->count($collection, [Query::equal('f', [0.1 + 0.2])])); + $this->assertSame(40, $database->count($collection)); + } + + public function testACursorAlongAFloatOrderVisitsEveryRowOnce(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + $collection = $this->createFloatCollection($database, 'float_cursor'); + $values = [-5e-324, 0.0, 5e-324, 1e-300, 0.3, 0.1 + 0.2, M_PI]; + $documents = []; + foreach ($values as $rank => $value) { + for ($copy = 0; $copy < 3; $copy++) { + $documents[] = new Document(['$id' => 'r'.$rank.'c'.$copy, 'f' => $value]); + } + } + $database->createDocuments($collection, $documents); + + foreach ([1, 2, 4] as $pageSize) { + foreach ([OrderDirection::Asc, OrderDirection::Desc] as $direction) { + $order = $direction === OrderDirection::Asc ? Query::orderAsc('f') : Query::orderDesc('f'); + $visited = []; + $cursor = null; + for ($page = 0; $page <= \count($documents); $page++) { + $queries = [$order, Query::limit($pageSize)]; + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + $rows = $database->find($collection, $queries); + if ($rows === []) { + break; + } + foreach ($rows as $row) { + $visited[] = $row->getId(); + } + $cursor = \end($rows); + } + + $label = $direction->value.' by '.$pageSize; + $this->assertSame(\count($documents), \count($visited), $label.' visits every row once'); + $this->assertSame(\count($visited), \count(\array_unique($visited)), $label.' repeats no row'); + $ranks = \array_map(static fn (string $id): int => (int) \substr($id, 1, \strpos($id, 'c') - 1), $visited); + $sorted = $ranks; + $direction === OrderDirection::Asc ? \sort($sorted) : \rsort($sorted); + $this->assertSame($sorted, $ranks, $label.' follows the order'); + } + } + } + + public function testFloatsKeepEveryDigit(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + $collection = $this->createFloatCollection($database, 'float_digits'); $database->createAttribute($collection, Attribute::double(key: 'fa', array: true)); $values = [ - 'tiny' => [1e-300, 1e-300], - 'negative' => [-2.5e-20, -2.5e-20], - 'digits' => [1.2345678901234567e-10, $family === 'mongo' ? 1.2345678901234567e-10 : 1.2345678901235e-10], - 'avogadro' => [6.02214076e23, 6.02214076e23], - 'sum' => [0.1 + 0.2, $family === 'mongo' ? 0.1 + 0.2 : 0.3], - 'max' => [1.7976931348623157e308, $family === 'mongo' ? 1.7976931348623157e308 : 1.7976931348623e308], + 'sum' => 0.1 + 0.2, + 'pi' => M_PI, + 'seventeen' => 123456789.12345679, + 'small' => 1.2345678901234567e-10, + 'tiny' => 1e-300, + 'normal' => 2.2250738585072014e-308, + 'denormal' => 5e-324, + 'negative denormal' => -5e-324, + 'negative' => -2.5e-20, + 'avogadro' => 6.02214076e23, + 'max' => 1.7976931348623157e308, + 'min' => -1.7976931348623157e308, ]; - foreach ($values as $id => [$written]) { - $database->createDocument($collection, new Document(['$id' => $id, 'f' => $written, 'fa' => [$written]])); + foreach ($values as $id => $value) { + $database->createDocument($collection, new Document(['$id' => \str_replace(' ', '_', $id), 'f' => $value, 'fa' => [$value, -$value]])); } - foreach ($values as $id => [$written, $stored]) { + foreach ($values as $id => $value) { + $id = \str_replace(' ', '_', $id); $document = $database->getDocument($collection, $id); - $this->assertSame($stored, $document->getAttribute('f'), $id); - $this->assertSame([$written], $document->getAttribute('fa'), $id); + $this->assertSame($value, $document->getAttribute('f'), $id.' read'); + if ($database->getAdapter() instanceof MySQL) { + // MySQL parses a JSON number only to within one unit in the last place. + $this->assertEqualsWithDelta([$value, -$value], $document->getAttribute('fa'), \abs($value) * PHP_FLOAT_EPSILON, $id.' array read'); + } else { + $this->assertSame([$value, -$value], $document->getAttribute('fa'), $id.' array read'); + } + $this->assertSame([$id], $this->floatIds($database, $collection, [Query::equal('f', [$value])]), $id.' equal'); } - $database->updateDocument($collection, 'tiny', new Document(['f' => -1e-300])); - $this->assertSame(-1e-300, $database->getDocument($collection, 'tiny')->getAttribute('f')); + $database->updateDocument($collection, 'tiny', new Document(['f' => 0.1 + 0.2])); + $this->assertStoredFloat($database, $collection, 'tiny', 0.1 + 0.2); - $small = \array_map( - static fn (Document $document): string => $document->getId(), - $database->find($collection, [Query::lessThan('f', 1e-5), Query::greaterThan('f', -1), Query::orderAsc('$id')]), - ); - $this->assertSame(['digits', 'negative', 'tiny'], $small); + $database->updateDocuments($collection, new Document(['f' => M_PI]), [Query::equal('f', [5e-324])]); + $this->assertStoredFloat($database, $collection, 'denormal', M_PI); + + $database->createDocuments($collection, [new Document(['$id' => 'bulk', 'f' => 123456789.12345679])]); + $this->assertStoredFloat($database, $collection, 'bulk', 123456789.12345679); + + $database->increaseDocumentAttribute($collection, 'avogadro', 'f', 0.1); + $this->assertStoredFloat($database, $collection, 'avogadro', 6.02214076e23 + 0.1); + + $database->createDocument($collection, new Document(['$id' => 'counter', 'f' => 0.0])); + $database->increaseDocumentAttribute($collection, 'counter', 'f', 0.1 + 0.2); + $this->assertStoredFloat($database, $collection, 'counter', 0.1 + 0.2); + $database->decreaseDocumentAttribute($collection, 'counter', 'f', 5e-324); + $this->assertStoredFloat($database, $collection, 'counter', (0.1 + 0.2) - 5e-324); + + if ($database->getAdapter()->hasFeature(Feature\Upserts::class)) { + $database->upsertDocuments($collection, [new Document(['$id' => 'bulk', 'f' => 5e-324])]); + $this->assertStoredFloat($database, $collection, 'bulk', 5e-324); + } + + if ($database->getAdapter()->supports(Capability::Operators)) { + $database->updateDocument($collection, 'pi', new Document(['f' => Operator::increment(5e-324)])); + $this->assertStoredFloat($database, $collection, 'pi', M_PI + 5e-324); + $database->updateDocument($collection, 'small', new Document(['f' => Operator::multiply(0.1 + 0.2)])); + $this->assertStoredFloat($database, $collection, 'small', 1.2345678901234567e-10 * (0.1 + 0.2)); + } + } + + public function testANonFiniteFloatIsRefused(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + $collection = $this->createFloatCollection($database, 'float_non_finite'); + $database->createDocument($collection, new Document(['$id' => 'zero', 'f' => 0.0])); + + $nonFinite = ['infinity' => INF, 'negative infinity' => -INF, 'overflow' => \json_decode('1e400'), 'not a number' => NAN]; + foreach ($nonFinite as $name => $value) { + $this->assertIsFloat($value); + foreach ([ + 'find equal' => fn () => $database->find($collection, [Query::equal('f', [$value])]), + 'find less than' => fn () => $database->find($collection, [Query::lessThan('f', $value)]), + 'count between' => fn () => $database->count($collection, [Query::between('f', 0, $value)]), + 'sum greater than' => fn () => $database->sum($collection, 'f', [Query::greaterThan('f', $value)]), + 'delete not equal' => fn () => $database->deleteDocuments($collection, [Query::notEqual('f', $value)]), + ] as $read => $call) { + try { + $call(); + $this->fail($read.' on '.$name.' was accepted'); + } catch (QueryException) { + } + } + + foreach ([ + 'create' => fn () => $database->createDocument($collection, new Document(['f' => $value])), + 'update' => fn () => $database->updateDocument($collection, 'zero', new Document(['f' => $value])), + ] as $write => $call) { + try { + $call(); + $this->fail($write.' of '.$name.' was accepted'); + } catch (StructureException) { + } + } + } + + $this->assertSame(1, $database->count($collection)); + $this->assertSame(0.0, $database->getDocument($collection, 'zero')->getAttribute('f')); + } + + private function createFloatCollection(Database $database, string $collection): string + { + $database->createCollection(Collection::create(id: $collection, permissions: [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], documentSecurity: false)); + $database->createAttribute($collection, Attribute::double(key: 'f')); + + return $collection; + } + + private function assertStoredFloat(Database $database, string $collection, string $id, float $value): void + { + $this->assertSame($value, $database->getDocument($collection, $id)->getAttribute('f'), $id.' read'); + $this->assertContains($id, $this->floatIds($database, $collection, [Query::equal('f', [$value])]), $id.' equal'); + } + + /** + * @param list $queries + * @return list + */ + private function floatIds(Database $database, string $collection, array $queries): array + { + $ids = \array_map(static fn (Document $document): string => $document->getId(), $database->find($collection, [...$queries, Query::limit(100)])); + \sort($ids); + + return $ids; } public function testFindOrderRandomIgnoresACursorAs7xDid(): void diff --git a/tests/unit/Adapter/BuildsAggregates.php b/tests/unit/Adapter/BuildsAggregates.php index 8b6337068..3cf0730a8 100644 --- a/tests/unit/Adapter/BuildsAggregates.php +++ b/tests/unit/Adapter/BuildsAggregates.php @@ -48,6 +48,6 @@ public function builtAggregate(string $operation, Document $collection, array $q */ public function boundValues(array $bindings): array { - return $bindings; + return \array_map(fn (mixed $value): mixed => \is_float($value) ? $this->formatFloat($value) : $value, $bindings); } } diff --git a/tests/unit/Adapter/FloatBindingTest.php b/tests/unit/Adapter/FloatBindingTest.php index e1c86b55b..49ed53fac 100644 --- a/tests/unit/Adapter/FloatBindingTest.php +++ b/tests/unit/Adapter/FloatBindingTest.php @@ -4,11 +4,17 @@ use PDO; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Adapter\Postgres; +use Utopia\Database\Adapter\SQL; +use Utopia\Database\Adapter\SQLite; use Utopia\Database\Attribute; use Utopia\Database\Collection; use Utopia\Database\Document; +use Utopia\Database\Operator; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -19,28 +25,159 @@ final class FloatBindingTest extends TestCase */ private array $bindings = []; - public function testAWriteBindsAFloatAsPhpWritesIt(): void + /** + * @return array + */ + public static function floats(): array + { + return [ + 'sum' => [0.1 + 0.2, '0.30000000000000004'], + 'pi' => [M_PI, '3.141592653589793'], + 'seventeen digits' => [123456789.12345679, '123456789.12345679'], + 'denormal' => [5e-324, '4.94065645841247E-324'], + 'tiny' => [1e-300, '1.0E-300'], + 'negative' => [-2.5e-20, '-2.5E-20'], + 'max' => [1.7976931348623157e308, '1.7976931348623157E+308'], + 'short' => [0.1, '0.1'], + ]; + } + + /** + * @return array}> + */ + public static function adapters(): array { - $adapter = $this->adapter(); - $collection = Collection::create(id: 'numbers', attributes: [Attribute::double(key: 'f')]); + return [ + 'mariadb' => [MariaDB::class], + 'mysql' => [MySQL::class], + 'postgres' => [Postgres::class], + 'sqlite' => [SQLite::class], + ]; + } - $adapter->createDocument($collection, new Document(['$id' => 'tiny', '$permissions' => [], 'f' => 1e-300])); + /** + * @return array, float, string}> + */ + public static function adaptersAndFloats(): array + { + $cases = []; + foreach (self::adapters() as $adapter => [$class]) { + foreach (self::floats() as $name => [$value, $bound]) { + $cases[$adapter.' '.$name] = [$class, $value, $bound]; + } + } - $this->assertContains(1e-300, $this->bindings); - $this->assertNotContains('0.00000000000000000', $this->bindings); + return $cases; } - public function testAFindBindsAFloatInFixedPoint(): void + /** + * @param class-string $class + */ + #[DataProvider('adaptersAndFloats')] + public function testACreateBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void { - $adapter = $this->adapter(); - $collection = Collection::create(id: 'numbers', attributes: [Attribute::double(key: 'f')]); + $this->adapter($class)->createDocument($this->collection(), new Document(['$id' => 'one', '$permissions' => [], 'f' => $value])); - $adapter->find($collection, [Query::lessThan('f', 0.5)]); + $this->assertBound($value, $bound); + } - $this->assertContains('0.50000000000000000', $this->bindings); + /** + * @param class-string $class + */ + #[DataProvider('adaptersAndFloats')] + public function testAnUpdateBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + { + $collection = $this->collection(); + $this->adapter($class)->updateDocuments($collection, new Document(['f' => $value]), [new Document(['$id' => 'one', '$sequence' => '1', '$permissions' => []])]); + + $this->assertBound($value, $bound); } - private function adapter(): MariaDB + /** + * @param class-string $class + */ + #[DataProvider('adaptersAndFloats')] + public function testAnIncrementBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + { + $this->adapter($class)->increaseDocumentAttribute($this->collection(), 'one', 'f', $value, '2026-01-01 00:00:00.000'); + + $this->assertBound($value, $bound); + } + + /** + * @param class-string $class + */ + #[DataProvider('adaptersAndFloats')] + public function testAnIncrementOperatorBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + { + $collection = $this->collection(); + $this->adapter($class)->updateDocuments($collection, new Document(['f' => Operator::increment($value)]), [new Document(['$id' => 'one', '$sequence' => '1', '$permissions' => []])]); + + $this->assertBound($value, $bound); + } + + /** + * @param class-string $class + */ + #[DataProvider('adaptersAndFloats')] + public function testFindCountAndSumBindAFilterAlike(string $class, float $value, string $bound): void + { + $adapter = $this->adapter($class); + $collection = $this->collection(); + $queries = [Query::equal('f', [$value])]; + + $adapter->find($collection, $queries); + $this->assertBound($value, $bound); + + $this->bindings = []; + $adapter->count($collection, $queries); + $this->assertBound($value, $bound); + + $this->bindings = []; + $adapter->sum($collection, 'f', $queries); + $this->assertBound($value, $bound); + } + + /** + * @param class-string $class + */ + #[DataProvider('adapters')] + public function testAFetchStringifiesADoubleAtTheRoundTripPrecision(string $class): void + { + $adapter = $this->adapter($class); + $fetchExactly = (new \ReflectionMethod($adapter, 'fetchExactly'))->getClosure($adapter); + $precision = \ini_get('precision'); + + $this->assertSame('0.30000000000000004', $fetchExactly(static fn (): string => (string) (0.1 + 0.2))); + $this->assertSame('5.0E-324', $fetchExactly(static fn (): string => (string) 5e-324)); + $this->assertSame($precision, \ini_get('precision')); + + try { + $fetchExactly(static fn () => throw new \RuntimeException('fetch failed')); + $this->fail('The fetch failure was swallowed'); + } catch (\RuntimeException) { + } + $this->assertSame($precision, \ini_get('precision')); + } + + private function assertBound(float $value, string $bound): void + { + $this->assertContains($bound, $this->bindings); + $this->assertSame($value, (float) $bound); + foreach ($this->bindings as $binding) { + $this->assertIsNotFloat($binding); + } + } + + private function collection(): Collection + { + return Collection::create(id: 'numbers', attributes: [Attribute::double(key: 'f')]); + } + + /** + * @param class-string $class + */ + private function adapter(string $class): SQL { $pdo = $this->createStub(PDO::class); $pdo->method('prepare')->willReturnCallback(function (): PDOStatement { @@ -53,14 +190,16 @@ private function adapter(): MariaDB }); $statement->method('fetchAll')->willReturn([]); $statement->method('fetch')->willReturn(false); + $statement->method('fetchColumn')->willReturn(false); $statement->method('rowCount')->willReturn(1); $statement->method('closeCursor')->willReturn(true); return $statement; }); $pdo->method('lastInsertId')->willReturn('1'); + $pdo->method('quote')->willReturnCallback(static fn (string $value): string => "'".$value."'"); - $adapter = new MariaDB($pdo); + $adapter = new $class($pdo); $adapter->setDatabase('database'); $adapter->setNamespace('namespace'); $authorization = new Authorization(); diff --git a/tests/unit/Documents/DeleteDocumentsRaceTest.php b/tests/unit/Documents/DeleteDocumentsRaceTest.php index cdb79be50..f45a512be 100644 --- a/tests/unit/Documents/DeleteDocumentsRaceTest.php +++ b/tests/unit/Documents/DeleteDocumentsRaceTest.php @@ -240,7 +240,7 @@ public function testWithoutATransactionADocumentChangedRightBeforeTheDeleteIsKep public static function floatFinds(): iterable { yield 'memory compares exactly' => [static fn (): Memory => new Memory(), ['large']]; - yield 'sqlite binds fixed point' => [static fn (): SQLite => SQLite::inMemory(), ['large', 'tiny']]; + yield 'sqlite compares exactly' => [static fn (): SQLite => SQLite::inMemory(), ['large']]; } /** diff --git a/tests/unit/Validator/FiniteTest.php b/tests/unit/Validator/FiniteTest.php new file mode 100644 index 000000000..b43bb9a57 --- /dev/null +++ b/tests/unit/Validator/FiniteTest.php @@ -0,0 +1,89 @@ + + */ + public static function finite(): array + { + return [ + 'zero' => [0.0], + 'integer' => [5], + 'fraction' => [0.1 + 0.2], + 'denormal' => [5e-324], + 'negative denormal' => [-5e-324], + 'maximum' => [PHP_FLOAT_MAX], + 'minimum' => [-PHP_FLOAT_MAX], + ]; + } + + /** + * @return array + */ + public static function nonFinite(): array + { + return [ + 'infinity' => [INF], + 'negative infinity' => [-INF], + 'not a number' => [NAN], + 'overflowing literal' => [\floatval(\json_decode('1e400'))], + 'negative overflowing literal' => [\floatval(\json_decode('-1e400'))], + ]; + } + + #[DataProvider('finite')] + public function testAcceptsAFiniteNumber(int|float $value): void + { + $this->assertTrue((new Finite())->isValid($value)); + $this->assertTrue($this->filter()->isValid(Query::equal('f', [$value]))); + $this->assertTrue($this->filter()->isValid(Query::between('f', $value, $value))); + } + + #[DataProvider('nonFinite')] + public function testRejectsANonFiniteNumber(float $value): void + { + $this->assertFalse((new Finite())->isValid($value)); + + foreach ([ + Query::equal('f', [$value]), + Query::notEqual('f', $value), + Query::lessThan('f', $value), + Query::greaterThanEqual('f', $value), + Query::between('f', 0, $value), + Query::equal('fa', [$value]), + ] as $query) { + $filter = $this->filter(); + $this->assertFalse($filter->isValid($query), $query->getMethod()->value); + $this->assertStringContainsString('Query value is invalid for attribute', $filter->getDescription()); + } + } + + public function testLooseRejectsAnOverflowingString(): void + { + $this->assertTrue((new Finite(loose: true))->isValid('1e300')); + $this->assertFalse((new Finite(loose: true))->isValid('1e400')); + $this->assertFalse((new Finite(loose: true))->isValid('-1e400')); + } + + private function filter(): Filter + { + return new Filter( + attributes: [ + new Document(['$id' => 'f', 'key' => 'f', 'type' => ColumnType::Double->value, 'array' => false]), + new Document(['$id' => 'fa', 'key' => 'fa', 'type' => ColumnType::Double->value, 'array' => true]), + ], + idAttributeType: ColumnType::Integer->value, + ); + } +} diff --git a/tests/unit/Validator/QueryShapeHavingTest.php b/tests/unit/Validator/QueryShapeHavingTest.php index aa4895d1b..7ce1a1544 100644 --- a/tests/unit/Validator/QueryShapeHavingTest.php +++ b/tests/unit/Validator/QueryShapeHavingTest.php @@ -80,6 +80,18 @@ public static function invalidHavingProvider(): array [Query::sum('price', 'total'), Query::groupBy(['name']), Query::having([Query::greaterThan('total', 'abc')])], 'Query value is invalid for aggregate alias "total"', ], + 'numeric alias compared with infinity' => [ + [Query::sum('price', 'total'), Query::groupBy(['name']), Query::having([Query::greaterThan('total', INF)])], + 'Query value is invalid for aggregate alias "total"', + ], + 'numeric alias compared with not a number' => [ + [Query::avg('rating', 'mean'), Query::groupBy(['name']), Query::having([Query::equal('mean', [NAN])])], + 'Query value is invalid for aggregate alias "mean"', + ], + 'grouped double compared with infinity' => [ + [Query::count('*', 'rows'), Query::groupBy(['rating']), Query::having([Query::lessThan('rating', -INF)])], + 'Query value is invalid for attribute "rating"', + ], 'numeric alias compared with a boolean' => [ [Query::count('*', 'rows'), Query::groupBy(['name']), Query::having([Query::equal('rows', [true])])], 'Query value is invalid for aggregate alias "rows"', From 59f5a067ffb6cb62ea55a1cb0b15a528ddd3451d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 Oct 2026 21:39:23 +1300 Subject: [PATCH 2/4] perf(adapters): bind a short double as PHP writes it and drop an always-true assertion A double that PHP's own cast already writes exactly binds as that string, so a short value costs what PDO's conversion cost before; longer ones fall back to 16 and then 17 significant digits. Co-Authored-By: Claude Opus 5.5 --- src/Database/Adapter/SQL.php | 18 ++++++++---------- tests/e2e/Adapter/Scopes/DocumentTests.php | 1 - tests/unit/Adapter/FloatBindingTest.php | 10 +++++----- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index d76ce1db8..529a206c2 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -124,8 +124,6 @@ abstract class SQL extends Adapter implements Feature\Connection, Feature\RawQue private const int UNPAIRED_JOINED_ROWS = 4; - private const array SHORT_FLOAT_FORMATS = ['%.15H', '%.16H']; - private const string ROUND_TRIP_PRECISION = '-1'; protected object $pdo; @@ -208,19 +206,19 @@ public function getDriver(): object } /** - * The shortest decimal that reads back as exactly this double. `%H` ignores the locale, and a double needs at - * most 17 significant digits to round-trip. + * A decimal that reads back as exactly this double, in as few digits as PHP writes it when they suffice. Casts + * and `%H` ignore the locale, and a double needs at most 17 significant digits to round-trip. */ protected function formatFloat(float $value): string { - foreach (self::SHORT_FLOAT_FORMATS as $format) { - $formatted = \sprintf($format, $value); - if ((float) $formatted === $value) { - return $formatted; - } + $formatted = (string) $value; + if ((float) $formatted === $value) { + return $formatted; } - return \sprintf('%.17H', $value); + $formatted = \sprintf('%.16H', $value); + + return (float) $formatted === $value ? $formatted : \sprintf('%.17H', $value); } /** diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 82c376ac0..ffca884e1 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3016,7 +3016,6 @@ public function testANonFiniteFloatIsRefused(): void $nonFinite = ['infinity' => INF, 'negative infinity' => -INF, 'overflow' => \json_decode('1e400'), 'not a number' => NAN]; foreach ($nonFinite as $name => $value) { - $this->assertIsFloat($value); foreach ([ 'find equal' => fn () => $database->find($collection, [Query::equal('f', [$value])]), 'find less than' => fn () => $database->find($collection, [Query::lessThan('f', $value)]), diff --git a/tests/unit/Adapter/FloatBindingTest.php b/tests/unit/Adapter/FloatBindingTest.php index 49ed53fac..3534dfe1b 100644 --- a/tests/unit/Adapter/FloatBindingTest.php +++ b/tests/unit/Adapter/FloatBindingTest.php @@ -34,7 +34,7 @@ public static function floats(): array 'sum' => [0.1 + 0.2, '0.30000000000000004'], 'pi' => [M_PI, '3.141592653589793'], 'seventeen digits' => [123456789.12345679, '123456789.12345679'], - 'denormal' => [5e-324, '4.94065645841247E-324'], + 'denormal' => [5e-324, '4.9406564584125E-324'], 'tiny' => [1e-300, '1.0E-300'], 'negative' => [-2.5e-20, '-2.5E-20'], 'max' => [1.7976931348623157e308, '1.7976931348623157E+308'], @@ -74,7 +74,7 @@ public static function adaptersAndFloats(): array * @param class-string $class */ #[DataProvider('adaptersAndFloats')] - public function testACreateBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + public function testACreateBindsADecimalThatRoundTrips(string $class, float $value, string $bound): void { $this->adapter($class)->createDocument($this->collection(), new Document(['$id' => 'one', '$permissions' => [], 'f' => $value])); @@ -85,7 +85,7 @@ public function testACreateBindsTheShortestStringThatRoundTrips(string $class, f * @param class-string $class */ #[DataProvider('adaptersAndFloats')] - public function testAnUpdateBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + public function testAnUpdateBindsADecimalThatRoundTrips(string $class, float $value, string $bound): void { $collection = $this->collection(); $this->adapter($class)->updateDocuments($collection, new Document(['f' => $value]), [new Document(['$id' => 'one', '$sequence' => '1', '$permissions' => []])]); @@ -97,7 +97,7 @@ public function testAnUpdateBindsTheShortestStringThatRoundTrips(string $class, * @param class-string $class */ #[DataProvider('adaptersAndFloats')] - public function testAnIncrementBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + public function testAnIncrementBindsADecimalThatRoundTrips(string $class, float $value, string $bound): void { $this->adapter($class)->increaseDocumentAttribute($this->collection(), 'one', 'f', $value, '2026-01-01 00:00:00.000'); @@ -108,7 +108,7 @@ public function testAnIncrementBindsTheShortestStringThatRoundTrips(string $clas * @param class-string $class */ #[DataProvider('adaptersAndFloats')] - public function testAnIncrementOperatorBindsTheShortestStringThatRoundTrips(string $class, float $value, string $bound): void + public function testAnIncrementOperatorBindsADecimalThatRoundTrips(string $class, float $value, string $bound): void { $collection = $this->collection(); $this->adapter($class)->updateDocuments($collection, new Document(['f' => Operator::increment($value)]), [new Document(['$id' => 'one', '$sequence' => '1', '$permissions' => []])]); From a13331f60c56fa0c97ef50384268c217389a0e84 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 Oct 2026 21:51:36 +1300 Subject: [PATCH 3/4] test(find): give each distinct row its own creation time Two documents created in the same millisecond share $createdAt, so a distinct read of k and $createdAt returned three rows instead of four on fast runs. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Scopes/DocumentTests.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index ffca884e1..bb7556fbb 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -11573,9 +11573,16 @@ public function testDistinctRowsCarryOnlyWhatTheySelectAndRebuildIntoADocument() ], permissions: [Permission::create(Role::any()), Permission::read(Role::any())], )); - foreach ([['a', 1], ['a', 1], ['b', 2], [null, null]] as [$k, $n]) { - $database->createDocument($collection, new Document(['k' => $k, 'n' => $n, '$permissions' => [Permission::read(Role::any())]])); - } + $database->withPreserveDates(true, function () use ($database, $collection): void { + foreach ([['a', 1], ['a', 1], ['b', 2], [null, null]] as $second => [$k, $n]) { + $database->createDocument($collection, new Document([ + 'k' => $k, + 'n' => $n, + '$createdAt' => '2026-01-01T00:00:0'.$second.'.000+00:00', + '$permissions' => [Permission::read(Role::any())], + ])); + } + }); $roundTrip = function (array $rows, string $case): void { foreach ($rows as $row) { From 0043660b8113be1ca6845b7c1517f1f131339d0f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 16 Oct 2026 22:06:00 +1300 Subject: [PATCH 4/4] test(adapters): allow MySQL's one-ulp JSON parse behind a pool too Co-Authored-By: Claude Opus 5.5 --- tests/e2e/Adapter/Scopes/DocumentTests.php | 26 ++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index bb7556fbb..35f150d4b 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -12,6 +12,7 @@ use Utopia\Database\Adapter\Memory; use Utopia\Database\Adapter\Mongo; use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Adapter\Pool; use Utopia\Database\Adapter\Postgres; use Utopia\Database\Adapter\Redis; use Utopia\Database\Adapter\SQL; @@ -2966,8 +2967,7 @@ public function testFloatsKeepEveryDigit(): void $id = \str_replace(' ', '_', $id); $document = $database->getDocument($collection, $id); $this->assertSame($value, $document->getAttribute('f'), $id.' read'); - if ($database->getAdapter() instanceof MySQL) { - // MySQL parses a JSON number only to within one unit in the last place. + if ($this->parsesJsonNumbersToWithinAnUlp($database)) { $this->assertEqualsWithDelta([$value, -$value], $document->getAttribute('fa'), \abs($value) * PHP_FLOAT_EPSILON, $id.' array read'); } else { $this->assertSame([$value, -$value], $document->getAttribute('fa'), $id.' array read'); @@ -3059,6 +3059,28 @@ private function createFloatCollection(Database $database, string $collection): return $collection; } + /** + * MySQL, pooled or not, parses a number in a JSON column only to within one unit in the last place. + */ + private function parsesJsonNumbersToWithinAnUlp(Database $database): bool + { + $adapter = $database->getAdapter(); + if ($adapter instanceof MySQL) { + return true; + } + if (! $adapter instanceof Pool) { + return false; + } + + $driver = $adapter->getDriver(); + if (! $driver instanceof PDO && ! $driver instanceof DatabasePDO) { + return false; + } + $version = $driver->getAttribute(PDO::ATTR_SERVER_VERSION); + + return $driver->getAttribute(PDO::ATTR_DRIVER_NAME) === 'mysql' && \is_string($version) && ! \str_contains($version, 'MariaDB'); + } + private function assertStoredFloat(Database $database, string $collection, string $id, float $value): void { $this->assertSame($value, $database->getDocument($collection, $id)->getAttribute('f'), $id.' read');