diff --git a/docs/testing.md b/docs/testing.md index 31caeaa..430237c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -13,7 +13,7 @@ uv run --no-sync python scripts/run_tests.py focused --feature native-descriptor # The same suites and shard assignments used in CI. uv run --no-sync python scripts/run_tests.py interpreter --shard 1 --shards 4 uv run --no-sync python scripts/run_tests.py prepare-browser -uv run --no-sync python scripts/run_tests.py browser-conformance --shard 1 --shards 4 +uv run --no-sync python scripts/run_tests.py browser-conformance --shard 1 --shards 6 uv run --no-sync python scripts/run_tests.py browser-integration --shard 1 --shards 3 uv run --no-sync python scripts/run_tests.py python ``` @@ -68,6 +68,25 @@ Representative form submissions, all source-error/diagnostic checks, offline reloads, service-worker/profile updates and every Stop/Restart cycle retain their UI paths. The console report records UI/worker counts and elapsed time. +Shard weights prefer the checked-in `scripts/test_costs.json` inventory of +successful command timings. New checks retain the manifest's fallback estimate; +costs affect assignments only, never selection or preparation/packaging order. +To refresh from a passing Linux CI run, download its `interpreter-*`, +`browser-conformance-*` and `browser-integration-*` artifacts under one directory, +then run: + +```sh +uv run --no-sync python scripts/update_test_costs.py build/ci-timings \ + --source https://github.com/outside-labs/pocket-python/actions/runs/RUN_ID +``` + +The importer records report hashes/revisions, ignores removed IDs, deduplicates +reports and takes the median for repeated samples. Failed, incomplete or invalid +timings are rejected. Review the generated inventory before committing it; do +not mix different platforms or worker configurations. Runner reports now record +the host and worker configuration. The inventory is a scheduling estimate, +never a performance assertion or a substitute for running the complete gate. + ## Fast development and merge policy Run the smallest meaningful local checks for the behavior being changed. A diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 7295bd0..12ec6f6 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path +import platform import shlex import subprocess import sys @@ -53,6 +54,9 @@ def run(selected, report_path, metadata): report = { 'schema_version': 1, 'revision': git_output('rev-parse', 'HEAD'), 'working_tree_modified': bool(git_output('status', '--porcelain')), + 'host': {'os': platform.system(), 'machine': platform.machine(), + 'python': platform.python_version(), + 'browser_workers': env.get('POCKET_PYTHON_BROWSER_WORKERS', '2')}, 'started_at': datetime.now(UTC).isoformat(), **metadata, 'checks': [], } try: diff --git a/scripts/test_costs.json b/scripts/test_costs.json new file mode 100644 index 0000000..0d29cc5 --- /dev/null +++ b/scripts/test_costs.json @@ -0,0 +1,878 @@ +{ + "schema_version": 1, + "source": "Optimized Linux CI: https://github.com/outside-labs/pocket-python/actions/runs/41423024431 (interpreter/conformance); https://github.com/outside-labs/pocket-python/actions/runs/41424015512 (integration)", + "reports": [ + { + "sha256": "cbbf03f727700cb44ecc1cfd9cd3c764b3a678fe1cff91b9afde6d4077e840df", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 1, + "shards": 6 + }, + { + "sha256": "0dc1c27d05fd0a5f0f5e5d86026b01838080a4033b3532d8fe096915690f1383", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "f1a07a947b79b9efac7c1f96f8450e2e46b2a375080c4329f81f47d0595d02a2", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 2, + "shards": 6 + }, + { + "sha256": "13e9013fdc3a70a47a4cf960e14a76ddf9b52f157da086417b2a6dedbbe6e716", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "5112a9b6b9730da9ea2efd0841b3f921376b798fb17cfe3a65741684732754c0", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 3, + "shards": 6 + }, + { + "sha256": "99dbc47880a224a07102d54f0e49cd3bd1261deac81f0ab125195c12961c753c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "1893135d935e8cc692e777126af6a2e0eb7f6d913de636a0658cf6b444981d5c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 4, + "shards": 6 + }, + { + "sha256": "fa3034a607d064679f837444b276b057c1a7fb88eeaa965655ae81ed49cb6a35", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "89980e5b934c7dc343c1a885b78a769bf35783c4111762cad99116dbc647dca0", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 5, + "shards": 6 + }, + { + "sha256": "e948d09ec207018dbac7d643dd4612db6501da90221f605853a3191ab3fae14c", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "39d6aaa0369f3530535bc78f0228badec0434b2b829e65dacf1e102f63a68de1", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "browser-conformance" + ], + "shard": 6, + "shards": 6 + }, + { + "sha256": "3d0a112daa3d36784739bb95fe0385b524b2d05562cf05b6c19b87de9f03037e", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "dabed0579d1e72d2a181fe7eb7103428c3043023c8e7284761ec8023ec9ab5a2", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 1, + "shards": 4 + }, + { + "sha256": "c5fbce181ca3e6181050fe3ca302b1be5c7f7126f169de6031dabb1d99116c76", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 2, + "shards": 4 + }, + { + "sha256": "4f5609d3425b49693f498a196952072a1ff76a16b477a25eddad23500db3aadf", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 3, + "shards": 4 + }, + { + "sha256": "38f43ec0aae2c780d08c8c025c78567e95f8772164f23cf3ac2df3a5b11124ca", + "revision": "004e9859481d77fd30b7fa3800df440c48104ca5", + "suites": [ + "interpreter" + ], + "shard": 4, + "shards": 4 + }, + { + "sha256": "e8a60a0321d67436cc308eef8de144e48da20b81119af1fe8f84e6d54f215d7c", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 1, + "shards": 3 + }, + { + "sha256": "04bda8dbc8a9f45a57fcda2f206d4611409e907e0041df02faa947a955195e6b", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "75c74ac8510ba8b8b43b23654b6bc0617dcc6d175e34db844b3b8fb66ab27b79", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 2, + "shards": 3 + }, + { + "sha256": "9d4ec95ba7cd3f817a24d6fb95586e0dc09b2c628c017e211b96cc5b28d79d37", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + }, + { + "sha256": "7276463aacf44d0ef8f7a958010d7b244aec39bfe39b758c70b316e0e88f88d8", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "browser-integration" + ], + "shard": 3, + "shards": 3 + }, + { + "sha256": "c7b9ff619a502cb94825e38bfdd78c2228c9156437b7f2bc23952180068a99a9", + "revision": "421341c7bad3bc829c925ed93b5b19b1fa1ffdd3", + "suites": [ + "prepare-browser" + ], + "shard": 1, + "shards": 1 + } + ], + "seconds": { + "abstract-flags-classes-drain-browser": 11.777, + "abstract-flags-classes-drain-node": 3.525, + "abstract-flags-classes-parked-browser": 12.125, + "abstract-flags-classes-parked-node": 0.925, + "abstract-flags-classes-stepped-browser": 11.501, + "abstract-flags-classes-stepped-node": 5.411, + "advertised-directories-classes-drain-browser": 7.934, + "advertised-directories-classes-drain-node": 3.916, + "advertised-directories-classes-parked-browser": 8.385, + "advertised-directories-classes-parked-node": 2.12, + "advertised-directories-classes-stepped-browser": 5.409, + "advertised-directories-classes-stepped-node": 4.403, + "arithmetic-dispatch-classes-drain-browser": 6.544, + "arithmetic-dispatch-classes-drain-node": 2.78, + "arithmetic-dispatch-classes-parked-browser": 4.233, + "arithmetic-dispatch-classes-parked-node": 1.949, + "arithmetic-dispatch-classes-stepped-browser": 4.741, + "arithmetic-dispatch-classes-stepped-node": 3.051, + "attribute-hooks-classes-drain-browser": 10.195, + "attribute-hooks-classes-drain-node": 4.412, + "attribute-hooks-classes-stepped-browser": 10.477, + "attribute-hooks-classes-stepped-node": 4.736, + "browser-worker-startup": 1.09, + "buffer-directories-classes-drain-browser": 12.834, + "buffer-directories-classes-drain-node": 6.411, + "buffer-directories-classes-parked-browser": 12.7, + "buffer-directories-classes-parked-node": 4.287, + "buffer-directories-classes-stepped-browser": 12.995, + "buffer-directories-classes-stepped-node": 6.564, + "callable-type-directories-classes-drain-browser": 6.857, + "callable-type-directories-classes-drain-node": 4.959, + "callable-type-directories-classes-parked-browser": 10.624, + "callable-type-directories-classes-parked-node": 2.234, + "callable-type-directories-classes-stepped-browser": 10.551, + "callable-type-directories-classes-stepped-node": 5.107, + "callbacks-browser": 1.772, + "callbacks-node": 0.841, + "class-body-namespaces-classes-drain-browser": 7.125, + "class-body-namespaces-classes-drain-node": 3.187, + "class-body-namespaces-classes-parked-browser": 7.448, + "class-body-namespaces-classes-parked-node": 3.268, + "class-body-namespaces-classes-stepped-browser": 7.693, + "class-body-namespaces-classes-stepped-node": 2.169, + "class-directories-classes-drain-browser": 5.718, + "class-directories-classes-drain-node": 4.555, + "class-directories-classes-parked-browser": 9.44, + "class-directories-classes-parked-node": 4.789, + "class-directories-classes-stepped-browser": 9.301, + "class-directories-classes-stepped-node": 4.787, + "class-mapping-proxies-classes-drain-browser": 21.869, + "class-mapping-proxies-classes-drain-node": 20.472, + "class-mapping-proxies-classes-parked-browser": 18.744, + "class-mapping-proxies-classes-parked-node": 16.8, + "class-mapping-proxies-classes-stepped-browser": 38.926, + "class-mapping-proxies-classes-stepped-node": 23.449, + "class-namespaces-classes-drain-browser": 3.5, + "class-namespaces-classes-drain-node": 2.443, + "class-namespaces-classes-parked-browser": 5.278, + "class-namespaces-classes-parked-node": 1.719, + "class-namespaces-classes-stepped-browser": 5.334, + "class-namespaces-classes-stepped-node": 2.447, + "classes-classes-drain-browser": 3.377, + "classes-classes-drain-node": 2.049, + "classes-classes-stepped-browser": 5.129, + "classes-classes-stepped-node": 2.122, + "classes-console-browser": 312.385, + "code-cells-classes-drain-browser": 3.058, + "code-cells-classes-drain-node": 1.354, + "code-cells-classes-parked-browser": 3.628, + "code-cells-classes-parked-node": 0.919, + "code-cells-classes-stepped-browser": 3.091, + "code-cells-classes-stepped-node": 1.371, + "compatibility-browser": 82.418, + "compatibility-node": 38.686, + "compiled-browser": 4.683, + "construction-classes-drain-browser": 12.434, + "construction-classes-drain-node": 3.9, + "construction-classes-stepped-browser": 12.739, + "construction-classes-stepped-node": 5.65, + "container-directories-classes-drain-browser": 9.991, + "container-directories-classes-drain-node": 3.262, + "container-directories-classes-parked-browser": 6.433, + "container-directories-classes-parked-node": 5.015, + "container-directories-classes-stepped-browser": 10.204, + "container-directories-classes-stepped-node": 4.808, + "container-type-directories-classes-drain-browser": 9.486, + "container-type-directories-classes-drain-node": 8.15, + "container-type-directories-classes-parked-browser": 15.663, + "container-type-directories-classes-parked-node": 8.468, + "container-type-directories-classes-stepped-browser": 9.516, + "container-type-directories-classes-stepped-node": 8.116, + "containers-browser": 4.447, + "custom-mro-classes-drain-browser": 4.996, + "custom-mro-classes-drain-node": 2.053, + "custom-mro-classes-parked-browser": 3.296, + "custom-mro-classes-parked-node": 1.03, + "custom-mro-classes-stepped-browser": 4.856, + "custom-mro-classes-stepped-node": 2.246, + "descriptor-binding-classes-drain-browser": 5.214, + "descriptor-binding-classes-drain-node": 3.499, + "descriptor-binding-classes-parked-browser": 7.931, + "descriptor-binding-classes-parked-node": 3.568, + "descriptor-binding-classes-stepped-browser": 7.91, + "descriptor-binding-classes-stepped-node": 2.428, + "descriptors-classes-drain-browser": 8.215, + "descriptors-classes-drain-node": 2.399, + "descriptors-classes-stepped-browser": 8.32, + "descriptors-classes-stepped-node": 3.653, + "documentation-classes-drain-browser": 11.338, + "documentation-classes-drain-node": 5.266, + "documentation-classes-parked-browser": 7.715, + "documentation-classes-parked-node": 5.197, + "documentation-classes-stepped-browser": 11.998, + "documentation-classes-stepped-node": 3.671, + "dynamic-compile-classes-drain-browser": 3.543, + "dynamic-compile-classes-drain-node": 2.004, + "dynamic-compile-classes-parked-browser": 5.361, + "dynamic-compile-classes-parked-node": 2.079, + "dynamic-compile-classes-stepped-browser": 3.521, + "dynamic-compile-classes-stepped-node": 2.147, + "dynamic-eval-classes-drain-browser": 9.151, + "dynamic-eval-classes-drain-node": 5.419, + "dynamic-eval-classes-parked-browser": 9.542, + "dynamic-eval-classes-parked-node": 5.45, + "dynamic-eval-classes-stepped-browser": 9.479, + "dynamic-eval-classes-stepped-node": 3.409, + "dynamic-exec-classes-drain-browser": 6.844, + "dynamic-exec-classes-drain-node": 4.117, + "dynamic-exec-classes-parked-browser": 5.499, + "dynamic-exec-classes-parked-node": 6.642, + "dynamic-exec-classes-stepped-browser": 10.629, + "dynamic-exec-classes-stepped-node": 6.541, + "dynamic-imports-classes-drain-browser": 5.148, + "dynamic-imports-classes-drain-node": 3.819, + "dynamic-imports-classes-parked-browser": 8.145, + "dynamic-imports-classes-parked-node": 3.937, + "dynamic-imports-classes-stepped-browser": 4.985, + "dynamic-imports-classes-stepped-node": 3.871, + "equality-classes-drain-browser": 10.936, + "equality-classes-drain-node": 5.051, + "equality-classes-stepped-browser": 10.824, + "equality-classes-stepped-node": 5.056, + "exceptions-browser": 59.96, + "exceptions-classes-drain-browser": 218.763, + "exceptions-classes-drain-node": 94.159, + "exceptions-classes-parked-browser": 225.899, + "exceptions-classes-parked-node": 102.897, + "exceptions-classes-stepped-browser": 219.495, + "exceptions-classes-stepped-node": 64.991, + "exceptions-core-drain-browser": 54.459, + "exceptions-core-drain-node": 63.221, + "exceptions-core-parked-browser": 82.899, + "exceptions-core-parked-node": 1.357, + "exceptions-core-stepped-browser": 53.236, + "exceptions-core-stepped-node": 2.144, + "execution-classes-drain-browser": 1.999, + "execution-classes-drain-node": 0.768, + "execution-classes-parked-browser": 1.145, + "execution-classes-parked-node": 0.845, + "execution-classes-stepped-browser": 1.17, + "execution-classes-stepped-node": 0.528, + "expanded-class-bases-classes-drain-browser": 4.407, + "expanded-class-bases-classes-drain-node": 1.169, + "expanded-class-bases-classes-parked-browser": 4.358, + "expanded-class-bases-classes-parked-node": 1.837, + "expanded-class-bases-classes-stepped-browser": 4.29, + "expanded-class-bases-classes-stepped-node": 1.845, + "formatting-classes-drain-browser": 8.325, + "formatting-classes-drain-node": 3.175, + "formatting-classes-stepped-browser": 7.058, + "formatting-classes-stepped-node": 3.223, + "function-binding-classes-drain-browser": 7.181, + "function-binding-classes-drain-node": 5.082, + "function-binding-classes-stepped-browser": 10.857, + "function-binding-classes-stepped-node": 5.168, + "function-binding-core-drain-browser": 8.954, + "function-binding-core-drain-node": 0.504, + "function-binding-core-stepped-browser": 8.212, + "function-binding-core-stepped-node": 3.167, + "function-dictionaries-classes-drain-browser": 9.243, + "function-dictionaries-classes-drain-node": 3.995, + "function-dictionaries-classes-parked-browser": 5.974, + "function-dictionaries-classes-parked-node": 4.009, + "function-dictionaries-classes-stepped-browser": 9.139, + "function-dictionaries-classes-stepped-node": 2.793, + "function-directories-classes-drain-browser": 3.965, + "function-directories-classes-drain-node": 2.114, + "function-directories-classes-parked-browser": 6.242, + "function-directories-classes-parked-node": 3.466, + "function-directories-classes-stepped-browser": 6.303, + "function-directories-classes-stepped-node": 3.474, + "function-metadata-classes-drain-browser": 8.811, + "function-metadata-classes-drain-node": 3.828, + "function-metadata-classes-parked-browser": 6.022, + "function-metadata-classes-parked-node": 4.049, + "function-metadata-classes-stepped-browser": 9.215, + "function-metadata-classes-stepped-node": 4.052, + "function-state-classes-drain-browser": 3.756, + "function-state-classes-drain-node": 1.549, + "function-state-classes-parked-browser": 2.643, + "function-state-classes-parked-node": 0.706, + "function-state-classes-stepped-browser": 2.605, + "function-state-classes-stepped-node": 1.629, + "functions-browser": 6.044, + "functions-node": 1.752, + "generate-identifiers": 0.212, + "generate-native-namespaces": 0.04, + "generate-printability": 0.117, + "generator-directories-classes-drain-browser": 5.228, + "generator-directories-classes-drain-node": 2.323, + "generator-directories-classes-parked-browser": 5.114, + "generator-directories-classes-parked-node": 2.293, + "generator-directories-classes-stepped-browser": 5.212, + "generator-directories-classes-stepped-node": 1.625, + "generator-frames-classes-drain-browser": 4.242, + "generator-frames-classes-drain-node": 2.028, + "generator-frames-classes-parked-browser": 4.287, + "generator-frames-classes-parked-node": 3.171, + "generator-frames-classes-stepped-browser": 6.583, + "generator-frames-classes-stepped-node": 3.116, + "generators-classes-drain-browser": 4.594, + "generators-classes-drain-node": 1.337, + "generators-classes-stepped-browser": 4.501, + "generators-classes-stepped-node": 1.924, + "generators-core-drain-browser": 2.875, + "generators-core-drain-node": 1.846, + "generators-core-stepped-browser": 2.836, + "generators-core-stepped-node": 1.86, + "immutable-subclasses-classes-drain-browser": 11.302, + "immutable-subclasses-classes-drain-node": 6.324, + "immutable-subclasses-classes-parked-browser": 7.792, + "immutable-subclasses-classes-parked-node": 4.113, + "immutable-subclasses-classes-stepped-browser": 12.019, + "immutable-subclasses-classes-stepped-node": 6.33, + "indexing-classes-drain-browser": 10.119, + "indexing-classes-drain-node": 4.59, + "indexing-classes-stepped-browser": 6.728, + "indexing-classes-stepped-node": 2.947, + "instance-dictionaries-classes-drain-browser": 8.14, + "instance-dictionaries-classes-drain-node": 2.46, + "instance-dictionaries-classes-parked-browser": 5.599, + "instance-dictionaries-classes-parked-node": 3.637, + "instance-dictionaries-classes-stepped-browser": 5.555, + "instance-dictionaries-classes-stepped-node": 3.562, + "instance-directories-classes-drain-browser": 8.502, + "instance-directories-classes-drain-node": 2.71, + "instance-directories-classes-parked-browser": 8.318, + "instance-directories-classes-parked-node": 4.244, + "instance-directories-classes-stepped-browser": 5.374, + "instance-directories-classes-stepped-node": 4.156, + "integer-identity-classes-drain-browser": 3.544, + "integer-identity-classes-drain-node": 1.533, + "integer-identity-classes-parked-browser": 2.582, + "integer-identity-classes-parked-node": 1.723, + "integer-identity-classes-stepped-browser": 3.584, + "integer-identity-classes-stepped-node": 1.133, + "interpreter-browser": 0.924, + "interpreter-console-browser": 7.914, + "interpreter-node": 0.182, + "iteration-classes-drain-browser": 10.998, + "iteration-classes-drain-node": 4.768, + "iteration-classes-stepped-browser": 10.857, + "iteration-classes-stepped-node": 3.437, + "iterator-directories-classes-drain-browser": 10.412, + "iterator-directories-classes-drain-node": 3.075, + "iterator-directories-classes-parked-browser": 6.521, + "iterator-directories-classes-parked-node": 0.972, + "iterator-directories-classes-stepped-browser": 6.571, + "iterator-directories-classes-stepped-node": 4.695, + "iterator-type-directories-classes-drain-browser": 15.078, + "iterator-type-directories-classes-drain-node": 7.039, + "iterator-type-directories-classes-parked-browser": 9.914, + "iterator-type-directories-classes-parked-node": 7.129, + "iterator-type-directories-classes-stepped-browser": 9.773, + "iterator-type-directories-classes-stepped-node": 7.238, + "length-truth-classes-drain-browser": 6.7, + "length-truth-classes-drain-node": 4.32, + "length-truth-classes-stepped-browser": 10.649, + "length-truth-classes-stepped-node": 4.561, + "library-console-browser": 4.187, + "library-profiles-browser": 2.454, + "library-profiles-node": 0.801, + "memory-browser": 1.142, + "memory-node": 0.188, + "metaclass-attributes-classes-drain-browser": 11.373, + "metaclass-attributes-classes-drain-node": 3.558, + "metaclass-attributes-classes-parked-browser": 7.376, + "metaclass-attributes-classes-parked-node": 4.957, + "metaclass-attributes-classes-stepped-browser": 7.4, + "metaclass-attributes-classes-stepped-node": 5.066, + "metaclass-factories-classes-drain-browser": 3.711, + "metaclass-factories-classes-drain-node": 0.973, + "metaclass-factories-classes-parked-browser": 2.63, + "metaclass-factories-classes-parked-node": 1.585, + "metaclass-factories-classes-stepped-browser": 3.731, + "metaclass-factories-classes-stepped-node": 1.505, + "metaclass-protocols-classes-drain-browser": 11.649, + "metaclass-protocols-classes-drain-node": 5.097, + "metaclass-protocols-classes-parked-browser": 11.311, + "metaclass-protocols-classes-parked-node": 5.036, + "metaclass-protocols-classes-stepped-browser": 12.182, + "metaclass-protocols-classes-stepped-node": 5.099, + "method-directories-classes-drain-browser": 4.597, + "method-directories-classes-drain-node": 2.268, + "method-directories-classes-parked-browser": 7.023, + "method-directories-classes-parked-node": 3.628, + "method-directories-classes-stepped-browser": 3.483, + "method-directories-classes-stepped-node": 3.517, + "micropython-browser": 1.797, + "module-namespaces-classes-drain-browser": 4.395, + "module-namespaces-classes-drain-node": 3.11, + "module-namespaces-classes-parked-browser": 6.987, + "module-namespaces-classes-parked-node": 3.149, + "module-namespaces-classes-stepped-browser": 6.862, + "module-namespaces-classes-stepped-node": 2.189, + "modules-browser": 2.204, + "modules-node": 1.25, + "mutable-subclasses-classes-drain-browser": 8.67, + "mutable-subclasses-classes-drain-node": 4.572, + "mutable-subclasses-classes-parked-browser": 8.704, + "mutable-subclasses-classes-parked-node": 4.848, + "mutable-subclasses-classes-stepped-browser": 8.505, + "mutable-subclasses-classes-stepped-node": 4.791, + "namespace-view-directories-classes-drain-browser": 4.378, + "namespace-view-directories-classes-drain-node": 1.4, + "namespace-view-directories-classes-parked-browser": 2.728, + "namespace-view-directories-classes-parked-node": 2.096, + "namespace-view-directories-classes-stepped-browser": 2.734, + "namespace-view-directories-classes-stepped-node": 2.049, + "native-attributes-classes-drain-browser": 15.017, + "native-attributes-classes-drain-node": 6.711, + "native-attributes-classes-parked-browser": 15.907, + "native-attributes-classes-parked-node": 6.98, + "native-attributes-classes-stepped-browser": 16.208, + "native-attributes-classes-stepped-node": 4.759, + "native-callable-directories-classes-drain-browser": 5.891, + "native-callable-directories-classes-drain-node": 2.923, + "native-callable-directories-classes-parked-browser": 4.158, + "native-callable-directories-classes-parked-node": 2.884, + "native-callable-directories-classes-stepped-browser": 6.528, + "native-callable-directories-classes-stepped-node": 2.109, + "native-descriptors-classes-drain-browser": 5.958, + "native-descriptors-classes-drain-node": 4.267, + "native-descriptors-classes-parked-browser": 9.931, + "native-descriptors-classes-parked-node": 3.035, + "native-descriptors-classes-stepped-browser": 9.831, + "native-descriptors-classes-stepped-node": 4.296, + "native-directory-closure-classes-drain-browser": 33.818, + "native-directory-closure-classes-drain-node": 15.748, + "native-directory-closure-classes-parked-browser": 21.112, + "native-directory-closure-classes-parked-node": 9.004, + "native-directory-closure-classes-stepped-browser": 26.868, + "native-directory-closure-classes-stepped-node": 16.598, + "native-namespaces-classes-drain-browser": 16.349, + "native-namespaces-classes-drain-node": 18.897, + "native-namespaces-classes-parked-browser": 28.74, + "native-namespaces-classes-parked-node": 12.501, + "native-namespaces-classes-stepped-browser": 18.043, + "native-namespaces-classes-stepped-node": 20.833, + "native-object-subclasses-classes-drain-browser": 7.618, + "native-object-subclasses-classes-drain-node": 2.635, + "native-object-subclasses-classes-parked-browser": 8.304, + "native-object-subclasses-classes-parked-node": 4.022, + "native-object-subclasses-classes-stepped-browser": 8.268, + "native-object-subclasses-classes-stepped-node": 4.052, + "native-protocols-classes-drain-browser": 57.429, + "native-protocols-classes-drain-node": 25.414, + "native-protocols-classes-parked-browser": 59.857, + "native-protocols-classes-parked-node": 3.599, + "native-protocols-classes-stepped-browser": 57.225, + "native-protocols-classes-stepped-node": 24.972, + "native-rebinding-classes-drain-browser": 10.214, + "native-rebinding-classes-drain-node": 4.493, + "native-rebinding-classes-parked-browser": 10.087, + "native-rebinding-classes-parked-node": 4.683, + "native-rebinding-classes-stepped-browser": 10.343, + "native-rebinding-classes-stepped-node": 3.105, + "nested-descriptors-classes-drain-browser": 5.692, + "nested-descriptors-classes-drain-node": 1.838, + "nested-descriptors-classes-parked-browser": 3.99, + "nested-descriptors-classes-parked-node": 2.737, + "nested-descriptors-classes-stepped-browser": 5.766, + "nested-descriptors-classes-stepped-node": 2.632, + "node-unit-abstract-flags": 2.418, + "node-unit-advertised-directories": 1.994, + "node-unit-arithmetic-dispatch": 1.431, + "node-unit-attribute-hooks": 2.154, + "node-unit-browser-host": 0.685, + "node-unit-browser-pool": 0.225, + "node-unit-buffer-directories": 5.529, + "node-unit-callable-type-directories": 2.953, + "node-unit-class-body-namespaces": 0.6, + "node-unit-class-directories": 2.848, + "node-unit-class-mapping-proxies": 13.601, + "node-unit-class-namespaces": 1.446, + "node-unit-class-policy": 0.083, + "node-unit-class-values": 0.171, + "node-unit-classes": 0.252, + "node-unit-classes-source": 1.082, + "node-unit-code-cells": 0.307, + "node-unit-construction": 1.696, + "node-unit-container-directories": 3.94, + "node-unit-container-type-directories": 6.578, + "node-unit-custom-mro": 0.633, + "node-unit-descriptor-binding": 1.365, + "node-unit-descriptors": 0.498, + "node-unit-documentation": 1.213, + "node-unit-dynamic-compile": 0.328, + "node-unit-dynamic-eval": 4.326, + "node-unit-dynamic-exec": 5.001, + "node-unit-dynamic-imports": 1.801, + "node-unit-equality": 1.276, + "node-unit-exceptions": 47.149, + "node-unit-execution": 0.738, + "node-unit-execution-stepping": 1.278, + "node-unit-expanded-class-bases": 0.453, + "node-unit-formatting": 2.704, + "node-unit-function-binding": 1.084, + "node-unit-function-dictionaries": 1.462, + "node-unit-function-directories": 1.944, + "node-unit-function-metadata": 1.391, + "node-unit-function-state": 0.709, + "node-unit-functions": 1.248, + "node-unit-generator-directories": 0.715, + "node-unit-generator-frames": 1.823, + "node-unit-generators": 0.499, + "node-unit-hashing": 1.555, + "node-unit-immutable-subclasses": 2.134, + "node-unit-indexing": 2.848, + "node-unit-instance-dictionaries": 1.513, + "node-unit-instance-directories": 2.693, + "node-unit-integer-identity": 0.612, + "node-unit-iteration": 3.198, + "node-unit-iterator-directories": 2.058, + "node-unit-iterator-type-directories": 3.892, + "node-unit-iterator-types": 0.209, + "node-unit-length-truth": 1.248, + "node-unit-library-profiles": 0.271, + "node-unit-memory": 0.169, + "node-unit-metaclass-attributes": 1.593, + "node-unit-metaclass-factories": 0.711, + "node-unit-metaclass-protocols": 1.856, + "node-unit-method-directories": 3.923, + "node-unit-module-namespaces": 0.721, + "node-unit-module-registry": 0.172, + "node-unit-modules": 0.673, + "node-unit-mutable-subclasses": 2.942, + "node-unit-namespace-view-directories": 0.64, + "node-unit-native-attributes": 1.757, + "node-unit-native-callable-directories": 1.857, + "node-unit-native-descriptors": 1.812, + "node-unit-native-directory-closure": 5.944, + "node-unit-native-namespaces": 20.195, + "node-unit-native-object-subclasses": 2.113, + "node-unit-native-protocols": 3.412, + "node-unit-native-rebinding": 1.411, + "node-unit-nested-descriptors": 2.851, + "node-unit-numeric": 0.335, + "node-unit-numeric-callbacks": 3.373, + "node-unit-numeric-directories": 2.898, + "node-unit-numeric-type-directories": 2.617, + "node-unit-object-hashing": 0.922, + "node-unit-object-integration": 5.712, + "node-unit-object-protocols": 0.556, + "node-unit-operations": 0.111, + "node-unit-ordering": 2.14, + "node-unit-parked-executions": 0.611, + "node-unit-persistent-callbacks": 0.671, + "node-unit-practical-containers": 3.035, + "node-unit-prepared-class-namespaces": 1.373, + "node-unit-private-names": 0.261, + "node-unit-property-diagnostics": 1.512, + "node-unit-property-directories": 1.081, + "node-unit-property-names": 1.156, + "node-unit-prototype": 0.093, + "node-unit-reassignment": 4.179, + "node-unit-reflection": 5.175, + "node-unit-repetition": 2.072, + "node-unit-sequence-directories": 1.59, + "node-unit-sequences": 3.038, + "node-unit-slots": 1.66, + "node-unit-source": 0.566, + "node-unit-source-tokens": 0.17, + "node-unit-special-calls": 0.628, + "node-unit-str-identity": 0.681, + "node-unit-subclass-initialization": 2.0, + "node-unit-syntax": 2.532, + "node-unit-type-checks": 1.111, + "node-unit-type-reflection": 1.578, + "node-unit-value-families": 1.706, + "node-unit-values": 0.122, + "node-unit-vm": 0.071, + "node-unit-wrapper-metadata": 0.727, + "node-unit-wrapper-super-directories": 2.009, + "numeric-classes-drain-browser": 10.781, + "numeric-classes-drain-node": 7.339, + "numeric-classes-stepped-browser": 16.681, + "numeric-classes-stepped-node": 7.57, + "numeric-directories-classes-drain-browser": 12.246, + "numeric-directories-classes-drain-node": 5.63, + "numeric-directories-classes-parked-browser": 7.787, + "numeric-directories-classes-parked-node": 2.321, + "numeric-directories-classes-stepped-browser": 7.808, + "numeric-directories-classes-stepped-node": 5.829, + "numeric-type-directories-classes-drain-browser": 5.965, + "numeric-type-directories-classes-drain-node": 3.062, + "numeric-type-directories-classes-parked-browser": 9.054, + "numeric-type-directories-classes-parked-node": 4.934, + "numeric-type-directories-classes-stepped-browser": 9.585, + "numeric-type-directories-classes-stepped-node": 4.743, + "object-console-browser": 8.97, + "object-hashing-classes-drain-browser": 5.206, + "object-hashing-classes-drain-node": 2.203, + "object-hashing-classes-parked-browser": 3.524, + "object-hashing-classes-parked-node": 2.235, + "object-hashing-classes-stepped-browser": 5.258, + "object-hashing-classes-stepped-node": 2.294, + "object-integration-classes-drain-browser": 6.5, + "object-integration-classes-drain-node": 4.759, + "object-integration-classes-parked-browser": 3.943, + "object-integration-classes-parked-node": 4.192, + "object-integration-classes-stepped-browser": 3.944, + "object-integration-classes-stepped-node": 4.899, + "object-protocols-classes-drain-browser": 5.385, + "object-protocols-classes-drain-node": 2.354, + "object-protocols-classes-stepped-browser": 4.244, + "object-protocols-classes-stepped-node": 2.452, + "object-scope-inventory": 0.186, + "ordering-classes-drain-browser": 12.269, + "ordering-classes-drain-node": 5.185, + "ordering-classes-stepped-browser": 8.198, + "ordering-classes-stepped-node": 5.532, + "ordering-core-drain-browser": 3.467, + "ordering-core-drain-node": 0.267, + "ordering-core-stepped-browser": 5.252, + "ordering-core-stepped-node": 4.43, + "ordinary-syntax-classes-drain-browser": 9.964, + "ordinary-syntax-classes-drain-node": 4.558, + "ordinary-syntax-classes-stepped-browser": 6.51, + "ordinary-syntax-classes-stepped-node": 4.672, + "ordinary-syntax-core-drain-browser": 5.451, + "ordinary-syntax-core-drain-node": 4.174, + "ordinary-syntax-core-stepped-browser": 9.188, + "ordinary-syntax-core-stepped-node": 2.914, + "practical-containers-classes-drain-browser": 12.463, + "practical-containers-classes-drain-node": 8.676, + "practical-containers-classes-parked-browser": 19.519, + "practical-containers-classes-parked-node": 9.051, + "practical-containers-classes-stepped-browser": 20.149, + "practical-containers-classes-stepped-node": 6.358, + "practical-containers-core-drain-browser": 14.541, + "practical-containers-core-drain-node": 7.8, + "practical-containers-core-parked-browser": 4.499, + "practical-containers-core-parked-node": 0.524, + "practical-containers-core-stepped-browser": 14.489, + "practical-containers-core-stepped-node": 0.75, + "prepare-browser": 9.662, + "prepared-class-namespaces-classes-drain-browser": 5.337, + "prepared-class-namespaces-classes-drain-node": 2.366, + "prepared-class-namespaces-classes-parked-browser": 5.279, + "prepared-class-namespaces-classes-parked-node": 2.524, + "prepared-class-namespaces-classes-stepped-browser": 5.251, + "prepared-class-namespaces-classes-stepped-node": 1.527, + "private-names-classes-drain-browser": 4.449, + "private-names-classes-drain-node": 1.206, + "private-names-classes-stepped-browser": 4.49, + "private-names-classes-stepped-node": 1.833, + "property-diagnostics-classes-drain-browser": 12.407, + "property-diagnostics-classes-drain-node": 5.559, + "property-diagnostics-classes-parked-browser": 8.495, + "property-diagnostics-classes-parked-node": 3.898, + "property-diagnostics-classes-stepped-browser": 7.905, + "property-diagnostics-classes-stepped-node": 5.545, + "property-directories-classes-drain-browser": 3.355, + "property-directories-classes-drain-node": 1.543, + "property-directories-classes-parked-browser": 4.633, + "property-directories-classes-parked-node": 2.293, + "property-directories-classes-stepped-browser": 4.526, + "property-directories-classes-stepped-node": 2.206, + "property-names-classes-drain-browser": 5.108, + "property-names-classes-drain-node": 3.308, + "property-names-classes-parked-browser": 7.708, + "property-names-classes-parked-node": 3.295, + "property-names-classes-stepped-browser": 5.039, + "property-names-classes-stepped-node": 2.347, + "reassignment-classes-drain-browser": 7.587, + "reassignment-classes-drain-node": 3.911, + "reassignment-classes-parked-browser": 11.957, + "reassignment-classes-parked-node": 5.891, + "reassignment-classes-stepped-browser": 11.798, + "reassignment-classes-stepped-node": 5.98, + "reflection-classes-drain-browser": 18.559, + "reflection-classes-drain-node": 12.753, + "reflection-classes-stepped-browser": 18.961, + "reflection-classes-stepped-node": 3.764, + "repetition-classes-drain-browser": 8.698, + "repetition-classes-drain-node": 5.755, + "repetition-classes-stepped-browser": 13.608, + "repetition-classes-stepped-node": 6.076, + "repetition-core-drain-browser": 8.818, + "repetition-core-drain-node": 5.394, + "repetition-core-stepped-browser": 8.441, + "repetition-core-stepped-node": 3.914, + "sequence-directories-classes-drain-browser": 5.828, + "sequence-directories-classes-drain-node": 2.544, + "sequence-directories-classes-parked-browser": 4.263, + "sequence-directories-classes-parked-node": 1.741, + "sequence-directories-classes-stepped-browser": 3.506, + "sequence-directories-classes-stepped-node": 2.664, + "sequences-classes-drain-browser": 3.75, + "sequences-classes-drain-node": 2.611, + "sequences-classes-stepped-browser": 5.655, + "sequences-classes-stepped-node": 1.762, + "sequences-core-drain-browser": 3.259, + "sequences-core-drain-node": 2.275, + "sequences-core-stepped-browser": 4.59, + "sequences-core-stepped-node": 2.408, + "shared-application-browser": 3.308, + "slots-classes-drain-browser": 6.727, + "slots-classes-drain-node": 2.819, + "slots-classes-parked-browser": 4.312, + "slots-classes-parked-node": 2.793, + "slots-classes-stepped-browser": 6.74, + "slots-classes-stepped-node": 1.874, + "source-browser": 24.263, + "source-node": 8.656, + "source-tokens-browser": 2.7, + "source-tokens-node": 0.175, + "special-calls-classes-drain-browser": 7.436, + "special-calls-classes-drain-node": 3.605, + "special-calls-classes-stepped-browser": 7.493, + "special-calls-classes-stepped-node": 2.397, + "subclass-initialization-classes-drain-browser": 7.527, + "subclass-initialization-classes-drain-node": 3.678, + "subclass-initialization-classes-stepped-browser": 5.054, + "subclass-initialization-classes-stepped-node": 3.746, + "type-checks-classes-drain-browser": 9.414, + "type-checks-classes-drain-node": 2.902, + "type-checks-classes-stepped-browser": 5.972, + "type-checks-classes-stepped-node": 4.232, + "type-reflection-classes-drain-browser": 8.894, + "type-reflection-classes-drain-node": 3.919, + "type-reflection-classes-parked-browser": 9.26, + "type-reflection-classes-parked-node": 4.13, + "type-reflection-classes-stepped-browser": 9.455, + "type-reflection-classes-stepped-node": 4.183, + "value-families-classes-drain-browser": 12.84, + "value-families-classes-drain-node": 5.655, + "value-families-classes-stepped-browser": 13.275, + "value-families-classes-stepped-node": 5.973, + "value-families-core-drain-browser": 11.143, + "value-families-core-drain-node": 5.673, + "value-families-core-stepped-browser": 7.25, + "value-families-core-stepped-node": 3.894, + "vm1-browser": 2.883, + "vm1-node": 0.782, + "vm2-browser": 8.822, + "vm2-node": 2.273, + "vm3-browser": 10.579, + "vm3-node": 3.23, + "wrapper-metadata-classes-drain-browser": 5.385, + "wrapper-metadata-classes-drain-node": 2.186, + "wrapper-metadata-classes-parked-browser": 3.627, + "wrapper-metadata-classes-parked-node": 2.354, + "wrapper-metadata-classes-stepped-browser": 5.216, + "wrapper-metadata-classes-stepped-node": 1.543, + "wrapper-super-directories-classes-drain-browser": 4.073, + "wrapper-super-directories-classes-drain-node": 2.831, + "wrapper-super-directories-classes-parked-browser": 6.287, + "wrapper-super-directories-classes-parked-node": 2.871, + "wrapper-super-directories-classes-stepped-browser": 6.411, + "wrapper-super-directories-classes-stepped-node": 1.981 + } +} diff --git a/scripts/test_manifest.py b/scripts/test_manifest.py index 06cb5de..9eccf39 100644 --- a/scripts/test_manifest.py +++ b/scripts/test_manifest.py @@ -1,7 +1,8 @@ """Shared validation inventory; commands run from the repository root.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace import json +import math from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -124,7 +125,19 @@ class Check: 'sequences': ('Q1', 'Q2', 'Q3', 'Hints')} -def checks(): +def measured_costs(path): + if not path.exists(): + return {} + inventory = json.loads(path.read_bytes()) + costs = inventory['seconds'] + if (inventory['schema_version'] != 1 or not isinstance(costs, dict) or + any(isinstance(value, bool) or not isinstance(value, (int, float)) or + not math.isfinite(value) or value <= 0 for value in costs.values())): + raise ValueError('Invalid measured validation costs') + return costs + + +def checks(costs_path=None): """Expand profile/driver matrices once for local and CI callers.""" result = [] @@ -216,4 +229,5 @@ def add(id, suite, *command, seconds=1): add('smoke-build-wheel', 'smoke', 'uv', 'build', '--wheel') add('smoke-installed-cli', 'smoke', 'python', 'scripts/smoke_install.py') add('smoke-interpreter-startup', 'smoke', 'node', 'scripts/startup_smoke.mjs') - return result + costs = measured_costs(costs_path or ROOT / 'scripts/test_costs.json') + return [replace(check, seconds=costs.get(check.id, check.seconds)) for check in result] diff --git a/scripts/update_test_costs.py b/scripts/update_test_costs.py new file mode 100644 index 0000000..24aead6 --- /dev/null +++ b/scripts/update_test_costs.py @@ -0,0 +1,68 @@ +"""Build deterministic shard costs from successful validation runner reports.""" + +import argparse +from collections import defaultdict +import hashlib +import json +import math +from pathlib import Path +import statistics +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.test_manifest import checks + + +def build_inventory(paths, source, known_ids=None): + known_ids = {check.id for check in checks()} if known_ids is None else set(known_ids) + files = sorted({file for path in paths for file in + (path.rglob('*-of-*.json') if path.is_dir() else [path])}) + samples, sources, seen = defaultdict(list), [], set() + for path in files: + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest in seen: + continue + seen.add(digest) + report = json.loads(raw) + rows, summary = report['checks'], report['summary'] + if (report['schema_version'] != 1 or summary['fail'] != 0 or + summary['pass'] != len(rows) or summary['selected'] != len(rows) or not rows): + raise ValueError(f'{path}: requires a complete successful runner report') + ids = set() + for row in rows: + seconds = row['seconds'] + if (row['returncode'] != 0 or row['id'] in ids or + isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or + not math.isfinite(seconds) or seconds < 0): + raise ValueError(f'{path}: invalid or failed timing sample') + ids.add(row['id']) + if row['id'] in known_ids: + samples[row['id']].append(max(0.001, seconds)) + sources.append({'sha256': digest, 'revision': report['revision'], + **{key: report[key] for key in ('suites', 'shard', 'shards', 'host') if key in report}}) + if not samples: + raise ValueError('No timings match the current validation inventory') + return {'schema_version': 1, 'source': source, 'reports': sources, + 'seconds': {id: round(statistics.median(values), 3) for id, values in sorted(samples.items())}} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('reports', nargs='+', type=Path, help='Runner JSON files or artifact directories') + parser.add_argument('--source', required=True, help='Workflow run URL or measurement provenance') + parser.add_argument('--output', type=Path, default=ROOT / 'scripts/test_costs.json') + args = parser.parse_args(argv) + try: + inventory = build_inventory(args.reports, args.source) + except (OSError, ValueError, KeyError, TypeError) as error: + parser.error(str(error)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(inventory, indent=2) + '\n', encoding='utf-8') + print(f'{len(inventory["seconds"])} measured costs from {len(inventory["reports"])} reports: {args.output}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_test_costs.py b/tests/test_test_costs.py new file mode 100644 index 0000000..e201d98 --- /dev/null +++ b/tests/test_test_costs.py @@ -0,0 +1,85 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.update_test_costs import build_inventory +from scripts.test_manifest import checks, measured_costs +from scripts.run_tests import shards + + +class TestCostInventoryTests(unittest.TestCase): + def test_measured_costs_control_balancing_without_changing_coverage_or_order(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'costs.json' + fallback = checks(path) + selected = [check for check in fallback if check.suite == 'browser-conformance'] + expensive = selected[0].id + path.write_text(json.dumps({'schema_version': 1, 'seconds': {expensive: 10000}}), encoding='utf-8') + measured = checks(path) + self.assertEqual([check.id for check in fallback], [check.id for check in measured]) + self.assertEqual([check.command for check in fallback], [check.command for check in measured]) + self.assertEqual(measured[0].seconds, fallback[0].seconds) + selected = [check for check in measured if check.suite == 'browser-conformance'] + self.assertEqual(selected[0].seconds, 10000) + groups = shards(selected, 6) + self.assertEqual([check.id for check in groups[0]], [expensive]) + self.assertCountEqual([check.id for group in groups for check in group], [check.id for check in selected]) + self.assertEqual(groups, shards(selected, 6)) + for invalid in (0, -1, float('nan'), float('inf'), True, 'slow'): + path.write_text(json.dumps({'schema_version': 1, 'seconds': {'known': invalid}}), encoding='utf-8') + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + measured_costs(path) + + def report(self, seconds=10): + return {'schema_version': 1, 'revision': 'a' * 40, 'suites': ['browser-conformance'], + 'shard': 1, 'shards': 6, + 'checks': [{'id': 'known', 'seconds': seconds, 'returncode': 0}], + 'summary': {'pass': 1, 'fail': 0, 'selected': 1}} + + def test_medians_provenance_deduplication_and_stale_ids(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for name, seconds in [('one', 10), ('two', 20), ('duplicate', 20), ('three', 30)]: + (root / f'{name}-1-of-1.json').write_text(json.dumps(self.report(seconds)), encoding='utf-8') + stale = self.report(1000) + stale['checks'][0]['id'] = 'removed' + (root / 'stale-1-of-1.json').write_text(json.dumps(stale), encoding='utf-8') + observed = build_inventory([root], 'workflow-url', ['known']) + self.assertEqual(observed['seconds'], {'known': 20}) + self.assertEqual(len(observed['reports']), 4) + self.assertEqual(observed['source'], 'workflow-url') + self.assertEqual(observed, build_inventory([root, root], 'workflow-url', ['known'])) + self.assertTrue(all(row['revision'] == 'a' * 40 for row in observed['reports'])) + + def test_incomplete_failed_duplicate_and_invalid_samples_are_rejected(self): + reports = [] + for seconds in (-1, float('nan'), float('inf'), True, '10'): + reports.append(self.report(seconds)) + failed = self.report() + failed['checks'][0]['returncode'] = 1 + reports.append(failed) + partial = self.report() + partial['summary']['selected'] = 2 + reports.append(partial) + failed_summary = self.report() + failed_summary['summary']['fail'] = 1 + reports.append(failed_summary) + duplicate = self.report() + duplicate['checks'] *= 2 + duplicate['summary']['pass'] = duplicate['summary']['selected'] = 2 + reports.append(duplicate) + for report in reports: + with self.subTest(report=report), tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'report.json' + path.write_text(json.dumps(report), encoding='utf-8') + with self.assertRaises(ValueError): + build_inventory([path], 'workflow-url', ['known']) + + def test_zero_duration_is_positive_and_unknown_inventory_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / 'report.json' + path.write_text(json.dumps(self.report(0)), encoding='utf-8') + self.assertEqual(build_inventory([path], 'local', ['known'])['seconds'], {'known': 0.001}) + with self.assertRaisesRegex(ValueError, 'No timings'): + build_inventory([path], 'local', ['new'])