diff --git a/.ci_support/environment-old.yml b/.ci_support/environment-old.yml index 0b046d762..2f618a93c 100644 --- a/.ci_support/environment-old.yml +++ b/.ci_support/environment-old.yml @@ -4,13 +4,13 @@ dependencies: - python - numpy =1.23.5 - openmpi =4.1.4 -- cloudpickle =2.0.0 +- cloudpickle =2.2.1 - mpi4py =3.1.4 - pyzmq =25.0.0 -- h5py =3.6.0 +- h5py =3.7.0 - networkx =2.8.8 -- ipython =7.33.0 +- ipython =8.4.0 - pydantic =2.5.3 - pygraphviz =1.10 -- hatchling =1.27.0 -- hatch-vcs =0.4.0 +- hatchling =1.33.0 +- hatch-vcs =0.5.0 diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 3dfa61348..ab3f80aaa 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -429,7 +429,7 @@ jobs: - uses: actions/checkout@v7 - uses: conda-incubator/setup-miniconda@v4 with: - python-version: '3.10' + python-version: '3.11' miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" diff --git a/pyproject.toml b/pyproject.toml index 0004a0d97..4586463f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,14 +16,13 @@ authors = [ readme = "README.md" license = { file = "LICENSE" } keywords = ["high performance computing", "hpc", "task scheduler", "slurm", "flux-framework", "executor", "python", "parallel computing", "distributed computing", "scientific computing", "concurrent.futures", "mpi4py", "gpu"] -requires-python = ">3.9, <3.16" +requires-python = ">3.10, <3.16" classifiers = [ "Development Status :: 5 - Production/Stable", "Topic :: Scientific/Engineering :: Physics", "License :: OSI Approved :: BSD License", "Intended Audience :: Science/Research", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/src/executorlib/__init__.py b/src/executorlib/__init__.py index 2021320bb..8099eba92 100644 --- a/src/executorlib/__init__.py +++ b/src/executorlib/__init__.py @@ -13,7 +13,6 @@ """ from concurrent.futures import Future -from typing import Optional import executorlib._version from executorlib.executor.base import BaseExecutor @@ -46,8 +45,8 @@ def get_cache_data(cache_directory: str) -> list[dict]: def get_cache_data_queue( cache_directory: str, - queue_type: Optional[str] = None, - config_directory: Optional[str] = None, + queue_type: str | None = None, + config_directory: str | None = None, ) -> list[dict]: """ Collect all HDF5 files in the cache directory and check their status in the queue system @@ -101,8 +100,8 @@ def get_future_from_cache( def terminate_tasks_in_cache( cache_directory: str, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, ): """ Delete all jobs stored in the cache directory from the queuing system @@ -124,8 +123,8 @@ def terminate_tasks_in_cache( def terminate_task_in_cache( cache_directory: str, cache_key: str, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, ): """ Delete a specific job stored in the cache directory from the queuing system diff --git a/src/executorlib/backend/interactive_parallel.py b/src/executorlib/backend/interactive_parallel.py index 7f968391d..4273f6d35 100644 --- a/src/executorlib/backend/interactive_parallel.py +++ b/src/executorlib/backend/interactive_parallel.py @@ -1,7 +1,6 @@ import pickle import sys from os.path import abspath -from typing import Optional import cloudpickle import zmq @@ -36,8 +35,8 @@ def main() -> None: mpi_size_larger_one = MPI.COMM_WORLD.Get_size() > 1 argument_dict = parse_arguments(argument_lst=sys.argv) - context: Optional[zmq.Context] = None - socket: Optional[zmq.Socket] = None + context: zmq.Context | None = None + socket: zmq.Socket | None = None if mpi_rank_zero: context, socket = interface_connect( host=argument_dict["host"], port=argument_dict["zmqport"] diff --git a/src/executorlib/backend/interactive_serial.py b/src/executorlib/backend/interactive_serial.py index dc3971b0d..542664239 100644 --- a/src/executorlib/backend/interactive_serial.py +++ b/src/executorlib/backend/interactive_serial.py @@ -1,6 +1,5 @@ import sys from os.path import abspath -from typing import Optional from executorlib.standalone.error import backend_write_error_file from executorlib.standalone.interactive.backend import call_funct, parse_arguments @@ -12,7 +11,7 @@ ) -def evaluate_cmd(argument_lst: Optional[list[str]] = None): +def evaluate_cmd(argument_lst: list[str] | None = None): """ The main function of the program. diff --git a/src/executorlib/executor/base.py b/src/executorlib/executor/base.py index 552f182f4..a1c0f384a 100644 --- a/src/executorlib/executor/base.py +++ b/src/executorlib/executor/base.py @@ -1,13 +1,13 @@ import queue from abc import ABC -from collections.abc import Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator from concurrent.futures import ( Executor as FutureExecutor, ) from concurrent.futures import ( Future, ) -from typing import Any, Callable, Optional, TypeVar +from typing import Any, TypeVar from executorlib.task_scheduler.base import TaskSchedulerBase @@ -27,7 +27,7 @@ def __init__(self, executor: TaskSchedulerBase): self._is_active = True @property - def max_workers(self) -> Optional[int]: + def max_workers(self) -> int | None: """ Return the number of parallel workers configured for this executor. @@ -47,7 +47,7 @@ def max_workers(self, max_workers: int): self._task_scheduler.max_workers = max_workers @property - def info(self) -> Optional[dict]: + def info(self) -> dict | None: """ Get the information about the executor. @@ -57,7 +57,7 @@ def info(self) -> Optional[dict]: return self._task_scheduler.info @property - def future_queue(self) -> Optional[queue.Queue]: + def future_queue(self) -> queue.Queue | None: """ Get the future queue. @@ -88,7 +88,7 @@ def submit( # type: ignore fn: Callable, /, *args, - resource_dict: Optional[dict] = None, + resource_dict: dict | None = None, **kwargs, ) -> Future: """ @@ -126,9 +126,9 @@ def map( self, fn: Callable[..., _T], *iterables: Iterable[Any], - timeout: Optional[float] = None, + timeout: float | None = None, chunksize: int = 1, - buffersize: Optional[int] = None, + buffersize: int | None = None, ) -> Iterator[_T]: """Returns an iterator equivalent to map(fn, iter). diff --git a/src/executorlib/executor/flux.py b/src/executorlib/executor/flux.py index 8ae1b80fb..6d1797aa5 100644 --- a/src/executorlib/executor/flux.py +++ b/src/executorlib/executor/flux.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Union +from collections.abc import Callable from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( @@ -107,22 +107,22 @@ class FluxJobExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pmi_mode: Optional[str] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + pmi_mode: str | None = None, flux_executor=None, flux_executor_nesting: bool = False, flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, restart_limit: int = 0, @@ -336,20 +336,20 @@ class FluxClusterExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - pmi_mode: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + pysqa_config_directory: str | None = None, + pmi_mode: str | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, openmpi_oversubscribe: bool = False, @@ -503,22 +503,22 @@ def __init__( def create_flux_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - executor_kwargs: Optional[dict] = None, - pmi_mode: Optional[str] = None, + max_workers: int | None = None, + max_cores: int | None = None, + cache_directory: str | None = None, + executor_kwargs: dict | None = None, + pmi_mode: str | None = None, flux_executor=None, flux_executor_nesting: bool = False, flux_log_files: bool = False, - hostname_localhost: Optional[bool] = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, log_obj_size: bool = False, wait: bool = True, validator: Callable = validate_resource_dict, restart_limit: int = 0, -) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: +) -> OneProcessTaskScheduler | BlockAllocationTaskScheduler: """ Create a flux executor diff --git a/src/executorlib/executor/single.py b/src/executorlib/executor/single.py index f2c8e534d..aa3cbed77 100644 --- a/src/executorlib/executor/single.py +++ b/src/executorlib/executor/single.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Union +from collections.abc import Callable from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( @@ -102,18 +102,18 @@ class SingleNodeExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, restart_limit: int = 0, @@ -310,18 +310,18 @@ class TestClusterExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, openmpi_oversubscribe: bool = False, @@ -446,18 +446,18 @@ def __init__( def create_single_node_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - executor_kwargs: Optional[dict] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + max_cores: int | None = None, + cache_directory: str | None = None, + executor_kwargs: dict | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, log_obj_size: bool = False, wait: bool = True, validator: Callable = validate_resource_dict, restart_limit: int = 0, -) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: +) -> OneProcessTaskScheduler | BlockAllocationTaskScheduler: """ Create a single node executor diff --git a/src/executorlib/executor/slurm.py b/src/executorlib/executor/slurm.py index dae584202..95bf941c4 100644 --- a/src/executorlib/executor/slurm.py +++ b/src/executorlib/executor/slurm.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Union +from collections.abc import Callable from executorlib.executor.base import BaseExecutor from executorlib.standalone.inputcheck import ( @@ -105,20 +105,20 @@ class SlurmClusterExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - pmi_mode: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + pysqa_config_directory: str | None = None, + pmi_mode: str | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, openmpi_oversubscribe: bool = False, @@ -346,19 +346,19 @@ class SlurmJobExecutor(BaseExecutor): def __init__( self, - max_workers: Optional[int] = None, - cache_directory: Optional[str] = None, - max_cores: Optional[int] = None, - resource_dict: Optional[dict] = None, - pmi_mode: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + cache_directory: str | None = None, + max_cores: int | None = None, + resource_dict: dict | None = None, + pmi_mode: str | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, log_obj_size: bool = False, wait: bool = True, restart_limit: int = 0, @@ -484,19 +484,19 @@ def __init__( def create_slurm_executor( - max_workers: Optional[int] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - executor_kwargs: Optional[dict] = None, - pmi_mode: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + max_workers: int | None = None, + max_cores: int | None = None, + cache_directory: str | None = None, + executor_kwargs: dict | None = None, + pmi_mode: str | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, log_obj_size: bool = False, wait: bool = True, validator: Callable = validate_resource_dict, restart_limit: int = 0, -) -> Union[OneProcessTaskScheduler, BlockAllocationTaskScheduler]: +) -> OneProcessTaskScheduler | BlockAllocationTaskScheduler: """ Create a SLURM executor diff --git a/src/executorlib/standalone/command.py b/src/executorlib/standalone/command.py index 2826f3e23..fe8946f4b 100644 --- a/src/executorlib/standalone/command.py +++ b/src/executorlib/standalone/command.py @@ -1,7 +1,6 @@ import importlib.util import os import sys -from typing import Optional SLURM_COMMAND = "srun" @@ -22,10 +21,10 @@ def get_command_path(executable: str) -> str: def get_cache_execute_command( file_name: str, cores: int = 1, - backend: Optional[str] = None, + backend: str | None = None, exclusive: bool = False, openmpi_oversubscribe: bool = False, - pmi_mode: Optional[str] = None, + pmi_mode: str | None = None, ) -> list: """ Get command to call backend as a list of two strings @@ -118,15 +117,15 @@ def get_interactive_execute_command( def generate_slurm_command( cores: int, - cwd: Optional[str], + cwd: str | None, threads_per_core: int = 1, gpus_per_core: int = 0, - num_nodes: Optional[int] = None, + num_nodes: int | None = None, exclusive: bool = False, openmpi_oversubscribe: bool = False, - slurm_cmd_args: Optional[list[str]] = None, - pmi_mode: Optional[str] = None, - run_time_max: Optional[int] = None, + slurm_cmd_args: list[str] | None = None, + pmi_mode: str | None = None, + run_time_max: int | None = None, ) -> list[str]: """ Generate the command list for the SLURM interface. diff --git a/src/executorlib/standalone/command_pysqa.py b/src/executorlib/standalone/command_pysqa.py index 6267d4889..b6846f29f 100644 --- a/src/executorlib/standalone/command_pysqa.py +++ b/src/executorlib/standalone/command_pysqa.py @@ -2,7 +2,6 @@ import os import subprocess from time import monotonic -from typing import Optional, Union from pysqa import QueueAdapter @@ -17,9 +16,9 @@ def pysqa_job_output_validation( task_key: str, file_name: str, queue_id: int, - status_check_dict: Optional[dict], - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + status_check_dict: dict | None, + pysqa_config_directory: str | None = None, + backend: str | None = None, job_status_check_interval: float = _JOB_STATUS_CHECK_INTERVAL, ) -> bool: """ @@ -65,8 +64,8 @@ def pysqa_job_output_validation( def pysqa_terminate( queue_id: int, - config_directory: Optional[str] = None, - backend: Optional[str] = None, + config_directory: str | None = None, + backend: str | None = None, ): """ Delete job from queuing system @@ -89,9 +88,9 @@ def pysqa_terminate( def pysqa_get_status_of_job( queue_id: int, - config_directory: Optional[str] = None, - backend: Optional[str] = None, -) -> Optional[str]: + config_directory: str | None = None, + backend: str | None = None, +) -> str | None: """ Query the status of a job from the queuing system. @@ -114,11 +113,11 @@ def pysqa_get_status_of_job( def pysqa_execute_command( commands: str, - working_directory: Optional[str] = None, + working_directory: str | None = None, split_output: bool = True, shell: bool = False, error_filename: str = "pysqa.err", -) -> Union[str, list[str]]: +) -> str | list[str]: """ A wrapper around the subprocess.check_output function. Modified from pysqa to raise an exception if the subprocess fails to submit the job to the queue. diff --git a/src/executorlib/standalone/hdf.py b/src/executorlib/standalone/hdf.py index 8c61b6591..c61a9dbc3 100644 --- a/src/executorlib/standalone/hdf.py +++ b/src/executorlib/standalone/hdf.py @@ -2,7 +2,7 @@ import os from concurrent.futures import Future from time import sleep -from typing import Any, Optional +from typing import Any import cloudpickle import h5py @@ -21,7 +21,7 @@ } -def dump(file_name: Optional[str], data_dict: dict) -> None: +def dump(file_name: str | None, data_dict: dict) -> None: """ Dump data dictionary into HDF5 file @@ -134,7 +134,7 @@ def get_runtime(file_name: str) -> float: return 0.0 -def get_queue_id(file_name: Optional[str]) -> Optional[int]: +def get_queue_id(file_name: str | None) -> int | None: """ Get queuing system id from HDF5 file diff --git a/src/executorlib/standalone/inputcheck.py b/src/executorlib/standalone/inputcheck.py index 0431acbb5..5c845739b 100644 --- a/src/executorlib/standalone/inputcheck.py +++ b/src/executorlib/standalone/inputcheck.py @@ -1,8 +1,8 @@ import inspect import multiprocessing import os.path +from collections.abc import Callable from concurrent.futures import Executor -from typing import Callable, Optional from warnings import warn @@ -113,7 +113,7 @@ def check_plot_dependency_graph(plot_dependency_graph: bool) -> None: ) -def check_pmi(backend: Optional[str], pmi: Optional[str]) -> None: +def check_pmi(backend: str | None, pmi: str | None) -> None: """ Check if pmi is valid for the selected backend and raise a ValueError if it is not. """ @@ -128,9 +128,7 @@ def check_pmi(backend: Optional[str], pmi: Optional[str]) -> None: ) -def check_init_function( - block_allocation: bool, init_function: Optional[Callable] -) -> None: +def check_init_function(block_allocation: bool, init_function: Callable | None) -> None: """ Check if block_allocation is False and init_function is not None, and raise a ValueError if it is. """ @@ -139,7 +137,7 @@ def check_init_function( def check_init_function_with_cache( - init_function: Optional[Callable], cache_directory: Optional[str] + init_function: Callable | None, cache_directory: str | None ) -> None: """ Check if init_function and cache_directory are both set, and raise a warning if they are. The cache key is @@ -155,9 +153,7 @@ def check_init_function_with_cache( ) -def check_max_workers_and_cores( - max_workers: Optional[int], max_cores: Optional[int] -) -> None: +def check_max_workers_and_cores(max_workers: int | None, max_cores: int | None) -> None: """ Check that neither max_workers nor max_cores is set when using a pysqa-based backend. @@ -178,7 +174,7 @@ def check_max_workers_and_cores( ) -def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None: +def check_hostname_localhost(hostname_localhost: bool | None) -> None: """ Check that hostname_localhost is not set when using a pysqa-based backend. @@ -211,7 +207,7 @@ def check_restart_limit(restart_limit: int, block_allocation: bool = True) -> No ) -def check_pmi_mode(pmi_mode: Optional[str]) -> None: +def check_pmi_mode(pmi_mode: str | None) -> None: """ Check that pmi_mode is not set on a local workstation without SLURM or flux. @@ -227,7 +223,7 @@ def check_pmi_mode(pmi_mode: Optional[str]) -> None: ) -def check_flux_log_files(flux_log_files: Optional[bool]) -> None: +def check_flux_log_files(flux_log_files: bool | None) -> None: """ Check if flux_log_files is True and raise a ValueError if it is. """ @@ -237,7 +233,7 @@ def check_flux_log_files(flux_log_files: Optional[bool]) -> None: ) -def check_pysqa_config_directory(pysqa_config_directory: Optional[str]) -> None: +def check_pysqa_config_directory(pysqa_config_directory: str | None) -> None: """ Check if pysqa_config_directory is None and raise a ValueError if it is not. """ @@ -248,9 +244,9 @@ def check_pysqa_config_directory(pysqa_config_directory: Optional[str]) -> None: def validate_number_of_cores( - max_cores: Optional[int] = None, - max_workers: Optional[int] = None, - cores_per_worker: Optional[int] = 1, + max_cores: int | None = None, + max_workers: int | None = None, + cores_per_worker: int | None = 1, set_local_cores: bool = False, ) -> int: """ @@ -289,7 +285,7 @@ def validate_number_of_cores( return max_workers -def check_file_exists(file_name: Optional[str]): +def check_file_exists(file_name: str | None): """ Check if file exists and raise a ValueError if it does not or file_name is None. """ diff --git a/src/executorlib/standalone/interactive/arguments.py b/src/executorlib/standalone/interactive/arguments.py index 7f0f7d893..4b41f74c4 100644 --- a/src/executorlib/standalone/interactive/arguments.py +++ b/src/executorlib/standalone/interactive/arguments.py @@ -1,6 +1,6 @@ from asyncio.exceptions import CancelledError from concurrent.futures import Future, TimeoutError -from typing import Any, Union +from typing import Any def get_future_objects_from_input(args: tuple, kwargs: dict): @@ -89,7 +89,7 @@ def update_futures_in_input(args: tuple, kwargs: dict) -> tuple[tuple, dict]: tuple, dict: arguments and keyword arguments with each future object in them being evaluated """ - def get_result(arg: Union[list[Future], Future]) -> Any: + def get_result(arg: list[Future] | Future) -> Any: if isinstance(arg, Future): return arg.result() elif isinstance(arg, list): diff --git a/src/executorlib/standalone/interactive/backend.py b/src/executorlib/standalone/interactive/backend.py index 53d014c56..133e60f31 100644 --- a/src/executorlib/standalone/interactive/backend.py +++ b/src/executorlib/standalone/interactive/backend.py @@ -1,9 +1,10 @@ import inspect -from typing import Any, Callable, Optional +from collections.abc import Callable +from typing import Any def call_funct( - input_dict: dict, funct: Optional[Callable] = None, memory: Optional[dict] = None + input_dict: dict, funct: Callable | None = None, memory: dict | None = None ) -> Any: """ Call function from dictionary diff --git a/src/executorlib/standalone/interactive/communication.py b/src/executorlib/standalone/interactive/communication.py index a9f4ea59a..6e74119ce 100644 --- a/src/executorlib/standalone/interactive/communication.py +++ b/src/executorlib/standalone/interactive/communication.py @@ -1,7 +1,8 @@ import logging import sys +from collections.abc import Callable from socket import gethostname -from typing import Any, Callable, Optional +from typing import Any import cloudpickle import zmq @@ -38,13 +39,13 @@ def __init__( self._poller.register(self._socket, zmq.POLLIN) self._process = None self._time_out_ms = time_out_ms - self._logger: Optional[logging.Logger] = None + self._logger: logging.Logger | None = None if log_obj_size: self._logger = logging.getLogger("executorlib") self._spawner = spawner self._command_lst: list[str] = [] self._booted_sucessfully: bool = False - self._stop_function: Optional[Callable] = None + self._stop_function: Callable | None = None @property def status(self) -> bool: @@ -116,8 +117,8 @@ def bind_to_random_port(self) -> int: def bootup( self, - command_lst: Optional[list[str]] = None, - stop_function: Optional[Callable] = None, + command_lst: list[str] | None = None, + stop_function: Callable | None = None, ): """ Boot up the client process to connect to the SocketInterface. @@ -182,10 +183,10 @@ def __del__(self): def interface_bootup( command_lst: list[str], connections, - hostname_localhost: Optional[bool] = None, + hostname_localhost: bool | None = None, log_obj_size: bool = False, - worker_id: Optional[int] = None, - stop_function: Optional[Callable] = None, + worker_id: int | None = None, + stop_function: Callable | None = None, ) -> SocketInterface: """ Start interface for ZMQ communication @@ -250,7 +251,7 @@ def interface_connect(host: str, port: str) -> tuple[zmq.Context, zmq.Socket]: return context, socket -def interface_send(socket: Optional[zmq.Socket], result_dict: dict): +def interface_send(socket: zmq.Socket | None, result_dict: dict): """ Send results to a SocketInterface instance. @@ -262,7 +263,7 @@ def interface_send(socket: Optional[zmq.Socket], result_dict: dict): socket.send(cloudpickle.dumps(result_dict)) -def interface_receive(socket: Optional[zmq.Socket]) -> dict: +def interface_receive(socket: zmq.Socket | None) -> dict: """ Receive instructions from a SocketInterface instance. @@ -278,7 +279,7 @@ def interface_receive(socket: Optional[zmq.Socket]) -> dict: return {} -def interface_shutdown(socket: Optional[zmq.Socket], context: Optional[zmq.Context]): +def interface_shutdown(socket: zmq.Socket | None, context: zmq.Context | None): """ Close the connection to a SocketInterface instance. diff --git a/src/executorlib/standalone/interactive/spawner.py b/src/executorlib/standalone/interactive/spawner.py index eac10a2ee..c348c3862 100644 --- a/src/executorlib/standalone/interactive/spawner.py +++ b/src/executorlib/standalone/interactive/spawner.py @@ -1,7 +1,7 @@ import os import subprocess from abc import ABC, abstractmethod -from typing import Callable, Optional +from collections.abc import Callable MPI_COMMAND = "mpiexec" @@ -9,7 +9,7 @@ class BaseSpawner(ABC): def __init__( self, - cwd: Optional[str] = None, + cwd: str | None = None, cores: int = 1, worker_id: int = 0, openmpi_oversubscribe: bool = False, @@ -32,7 +32,7 @@ def __init__( def bootup( self, command_lst: list[str], - stop_function: Optional[Callable] = None, + stop_function: Callable | None = None, ) -> bool: """ Method to start the interface. @@ -70,7 +70,7 @@ def poll(self): class SubprocessSpawner(BaseSpawner): def __init__( self, - cwd: Optional[str] = None, + cwd: str | None = None, cores: int = 1, worker_id: int = 0, openmpi_oversubscribe: bool = False, @@ -92,13 +92,13 @@ def __init__( worker_id=worker_id, openmpi_oversubscribe=openmpi_oversubscribe, ) - self._process: Optional[subprocess.Popen] = None + self._process: subprocess.Popen | None = None self._threads_per_core = threads_per_core def bootup( self, command_lst: list[str], - stop_function: Optional[Callable] = None, + stop_function: Callable | None = None, ) -> bool: """ Method to start the subprocess interface. diff --git a/src/executorlib/standalone/select.py b/src/executorlib/standalone/select.py index 2df2f19c3..8580f4397 100644 --- a/src/executorlib/standalone/select.py +++ b/src/executorlib/standalone/select.py @@ -1,5 +1,5 @@ from concurrent.futures import Future -from typing import Any, Optional +from typing import Any class FutureSelector(Future): @@ -37,7 +37,7 @@ def __setattr__(self, name: str, value: Any): else: setattr(self._future, name, value) - def result(self, timeout: Optional[float] = None) -> Any: + def result(self, timeout: float | None = None) -> Any: """ Return the selected element from the underlying future's result. diff --git a/src/executorlib/standalone/serialize.py b/src/executorlib/standalone/serialize.py index 5d5c86dff..f33c7719f 100644 --- a/src/executorlib/standalone/serialize.py +++ b/src/executorlib/standalone/serialize.py @@ -1,7 +1,7 @@ import hashlib import inspect import re -from typing import Callable, Optional +from collections.abc import Callable import cloudpickle @@ -30,11 +30,11 @@ def cloudpickle_register(ind: int = 2): def serialize_funct( fn: Callable, - fn_args: Optional[list] = None, - fn_kwargs: Optional[dict] = None, - resource_dict: Optional[dict] = None, - cache_key: Optional[str] = None, -) -> tuple[str, dict, Optional[Exception]]: + fn_args: list | None = None, + fn_kwargs: dict | None = None, + resource_dict: dict | None = None, + cache_key: str | None = None, +) -> tuple[str, dict, Exception | None]: """ Serialize a function and its arguments and keyword arguments into an HDF5 file. diff --git a/src/executorlib/standalone/validate.py b/src/executorlib/standalone/validate.py index c77d7df1d..2dd8e1cef 100644 --- a/src/executorlib/standalone/validate.py +++ b/src/executorlib/standalone/validate.py @@ -1,5 +1,4 @@ import warnings -from typing import Optional try: from pydantic import BaseModel, Extra @@ -33,19 +32,19 @@ class ResourceDictValidation(BaseModel): submission_template (str, optional): Template for queuing system job submission scripts. """ - cores: Optional[int] = None - threads_per_core: Optional[int] = None - gpus_per_core: Optional[int] = None - cwd: Optional[str] = None - cache_key: Optional[str] = None - cache_directory: Optional[str] = None - num_nodes: Optional[int] = None - exclusive: Optional[bool] = None - error_log_file: Optional[str] = None - run_time_max: Optional[int] = None - priority: Optional[int] = None - slurm_cmd_args: Optional[list[str]] = None - submission_template: Optional[str] = None + cores: int | None = None + threads_per_core: int | None = None + gpus_per_core: int | None = None + cwd: str | None = None + cache_key: str | None = None + cache_directory: str | None = None + num_nodes: int | None = None + exclusive: bool | None = None + error_log_file: str | None = None + run_time_max: int | None = None + priority: int | None = None + slurm_cmd_args: list[str] | None = None + submission_template: str | None = None if HAS_PYDANTIC: diff --git a/src/executorlib/task_scheduler/base.py b/src/executorlib/task_scheduler/base.py index 34534950c..a0db6ab76 100644 --- a/src/executorlib/task_scheduler/base.py +++ b/src/executorlib/task_scheduler/base.py @@ -1,7 +1,7 @@ import contextlib import queue import sys -from collections.abc import Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator from concurrent.futures import ( Executor as FutureExecutor, ) @@ -9,7 +9,7 @@ Future, ) from threading import Thread -from typing import Any, Callable, Optional, TypeVar, Union +from typing import Any, TypeVar from executorlib.standalone.inputcheck import check_resource_dict from executorlib.standalone.queue import cancel_items_in_queue @@ -38,7 +38,7 @@ class TaskSchedulerBase(FutureExecutor): def __init__( self, - max_cores: Optional[int] = None, + max_cores: int | None = None, validator: Callable = validate_resource_dict, ): """ @@ -53,12 +53,12 @@ def __init__( cloudpickle_register(ind=3) self._process_kwargs: dict = {} self._max_cores = max_cores - self._future_queue: Optional[queue.Queue] = queue.Queue() - self._process: Optional[Union[Thread, list[Thread]]] = None + self._future_queue: queue.Queue | None = queue.Queue() + self._process: Thread | list[Thread] | None = None self._validator = validator @property - def max_workers(self) -> Optional[int]: + def max_workers(self) -> int | None: """ Return the configured number of parallel workers, or None if unconstrained. @@ -78,7 +78,7 @@ def max_workers(self, max_workers: int): raise NotImplementedError("The max_workers setter is not implemented.") @property - def info(self) -> Optional[dict]: + def info(self) -> dict | None: """ Get the information about the executor. @@ -97,7 +97,7 @@ def info(self) -> Optional[dict]: return None @property - def future_queue(self) -> Optional[queue.Queue]: + def future_queue(self) -> queue.Queue | None: """ Get the future queue. @@ -128,7 +128,7 @@ def submit( # type: ignore fn: Callable, /, *args, - resource_dict: Optional[dict] = None, + resource_dict: dict | None = None, **kwargs, ) -> Future: """ @@ -185,9 +185,9 @@ def map( self, fn: Callable[..., _T], *iterables: Iterable[Any], - timeout: Optional[float] = None, + timeout: float | None = None, chunksize: int = 1, - buffersize: Optional[int] = None, + buffersize: int | None = None, ) -> Iterator[_T]: """Returns an iterator equivalent to map(fn, iter). diff --git a/src/executorlib/task_scheduler/file/shared.py b/src/executorlib/task_scheduler/file/shared.py index 75749dffe..5e06547dd 100644 --- a/src/executorlib/task_scheduler/file/shared.py +++ b/src/executorlib/task_scheduler/file/shared.py @@ -1,9 +1,10 @@ import contextlib import os import queue +from collections.abc import Callable from concurrent.futures import CancelledError, Future from time import sleep -from typing import Any, Callable, Optional +from typing import Any from executorlib.standalone.command import get_cache_execute_command from executorlib.standalone.hdf import get_cache_files, get_output, get_queue_id @@ -12,7 +13,7 @@ class FutureItem: - def __init__(self, file_name: str, selector: Optional[int | str] = None): + def __init__(self, file_name: str, selector: int | str | None = None): """ Initialize a FutureItem object. @@ -58,12 +59,12 @@ def execute_tasks_h5( future_queue: queue.Queue, execute_function: Callable, executor_kwargs: dict, - terminate_function: Optional[Callable] = None, - validate_function: Optional[Callable] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + terminate_function: Callable | None = None, + validate_function: Callable | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, disable_dependencies: bool = False, - pmi_mode: Optional[str] = None, + pmi_mode: str | None = None, wait: bool = True, refresh_rate: float = 0.01, ) -> None: @@ -224,12 +225,12 @@ def _check_task_output( task_key: str, future_obj: Future, cache_directory: str, - queue_id: Optional[int] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, - validate_function: Optional[Callable] = None, - status_check_dict: Optional[dict] = None, - duplicate_dict: Optional[dict] = None, + queue_id: int | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, + validate_function: Callable | None = None, + status_check_dict: dict | None = None, + duplicate_dict: dict | None = None, ) -> Future: """ Check the output of a task and set the result of the future object if available. @@ -321,7 +322,7 @@ def _convert_args_and_kwargs( task_dict: dict, memory_dict: dict, file_name_dict: dict, - duplicate_dict: Optional[dict] = None, + duplicate_dict: dict | None = None, ) -> tuple[list, dict, list]: """ Convert the arguments and keyword arguments in a task dictionary to the appropriate types. @@ -412,12 +413,12 @@ def _refresh_memory_dict( memory_dict: dict, cache_dir_dict: dict, process_dict: dict, - duplicate_dict: Optional[dict] = None, - status_check_dict: Optional[dict] = None, - terminate_function: Optional[Callable] = None, - validate_function: Optional[Callable] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + duplicate_dict: dict | None = None, + status_check_dict: dict | None = None, + terminate_function: Callable | None = None, + validate_function: Callable | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, refresh_rate: float = 0.01, ) -> dict: """ @@ -473,9 +474,9 @@ def _refresh_memory_dict( def _cancel_processes( process_dict: dict, - terminate_function: Optional[Callable] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + terminate_function: Callable | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, ): """ Cancel processes @@ -500,7 +501,7 @@ def _cancel_processes( def _get_task_input( task_resource_dict: dict, executor_kwargs: dict -) -> tuple[dict, Optional[str], str, Optional[str]]: +) -> tuple[dict, str | None, str, str | None]: """ Merge per-task resource requirements with executor defaults and extract scheduling metadata. @@ -547,12 +548,12 @@ def _shutdown_executor( memory_dict: dict, process_dict: dict, cache_dir_dict: dict, - duplicate_dict: Optional[dict] = None, - status_check_dict: Optional[dict] = None, - terminate_function: Optional[Callable] = None, - validate_function: Optional[Callable] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + duplicate_dict: dict | None = None, + status_check_dict: dict | None = None, + terminate_function: Callable | None = None, + validate_function: Callable | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, refresh_rate: float = 0.01, ): """ diff --git a/src/executorlib/task_scheduler/file/spawner_pysqa.py b/src/executorlib/task_scheduler/file/spawner_pysqa.py index b6e36d932..75da55789 100644 --- a/src/executorlib/task_scheduler/file/spawner_pysqa.py +++ b/src/executorlib/task_scheduler/file/spawner_pysqa.py @@ -1,6 +1,5 @@ import os from subprocess import CalledProcessError -from typing import Optional from pandas import DataFrame from pysqa import QueueAdapter @@ -18,11 +17,11 @@ def execute_with_pysqa( file_name: str, data_dict: dict, cache_directory: str, - task_dependent_lst: Optional[list[int]] = None, - resource_dict: Optional[dict] = None, - config_directory: Optional[str] = None, - backend: Optional[str] = None, -) -> Optional[int]: + task_dependent_lst: list[int] | None = None, + resource_dict: dict | None = None, + config_directory: str | None = None, + backend: str | None = None, +) -> int | None: """ Execute a command by submitting it to the queuing system @@ -105,8 +104,8 @@ def execute_with_pysqa( def terminate_tasks_in_cache( cache_directory: str, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, ): """ Delete all jobs stored in the cache directory from the queuing system @@ -134,8 +133,8 @@ def terminate_tasks_in_cache( def terminate_task_in_cache( cache_directory: str, cache_key: str, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, ): """ Delete a specific job stored in the cache directory from the queuing system @@ -159,8 +158,8 @@ def terminate_task_in_cache( def get_queue_system_cache_data( cache_dict: list[dict], - queue_type: Optional[str] = None, - config_directory: Optional[str] = None, + queue_type: str | None = None, + config_directory: str | None = None, ) -> list[dict]: """ Get the status of the queue system. diff --git a/src/executorlib/task_scheduler/file/spawner_subprocess.py b/src/executorlib/task_scheduler/file/spawner_subprocess.py index febeedc7b..cf202249d 100644 --- a/src/executorlib/task_scheduler/file/spawner_subprocess.py +++ b/src/executorlib/task_scheduler/file/spawner_subprocess.py @@ -1,7 +1,6 @@ import os import subprocess import time -from typing import Optional from executorlib.standalone.hdf import dump from executorlib.standalone.inputcheck import check_file_exists @@ -14,11 +13,11 @@ def subprocess_execute( command: list, file_name: str, data_dict: dict, - cache_directory: Optional[str] = None, - task_dependent_lst: Optional[list] = None, - resource_dict: Optional[dict] = None, - config_directory: Optional[str] = None, - backend: Optional[str] = None, + cache_directory: str | None = None, + task_dependent_lst: list | None = None, + resource_dict: dict | None = None, + config_directory: str | None = None, + backend: str | None = None, ) -> subprocess.Popen: """ Execute a command in a subprocess. @@ -78,7 +77,7 @@ def subprocess_terminate(task): def _get_working_directory( - cache_directory: Optional[str] = None, resource_dict: Optional[dict] = None + cache_directory: str | None = None, resource_dict: dict | None = None ): if resource_dict is None: resource_dict = {} diff --git a/src/executorlib/task_scheduler/file/task_scheduler.py b/src/executorlib/task_scheduler/file/task_scheduler.py index 95fbfce6b..a13705ddc 100644 --- a/src/executorlib/task_scheduler/file/task_scheduler.py +++ b/src/executorlib/task_scheduler/file/task_scheduler.py @@ -1,5 +1,5 @@ +from collections.abc import Callable from threading import Thread -from typing import Callable, Optional from executorlib.standalone.inputcheck import ( check_executor, @@ -32,13 +32,13 @@ class FileTaskScheduler(TaskSchedulerBase): def __init__( self, - executor_kwargs: Optional[dict] = None, + executor_kwargs: dict | None = None, execute_function: Callable = execute_with_pysqa, - terminate_function: Optional[Callable] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + terminate_function: Callable | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, disable_dependencies: bool = False, - pmi_mode: Optional[str] = None, + pmi_mode: str | None = None, wait: bool = True, refresh_rate: float = 0.01, validator: Callable = validate_resource_dict, @@ -96,18 +96,18 @@ def __init__( def create_file_executor( executor_kwargs: dict, - max_workers: Optional[int] = None, - backend: Optional[str] = None, - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - pmi_mode: Optional[str] = None, + max_workers: int | None = None, + backend: str | None = None, + max_cores: int | None = None, + cache_directory: str | None = None, + pmi_mode: str | None = None, flux_executor=None, flux_executor_nesting: bool = False, flux_log_files: bool = False, - pysqa_config_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + pysqa_config_directory: str | None = None, + hostname_localhost: bool | None = None, block_allocation: bool = False, - init_function: Optional[Callable] = None, + init_function: Callable | None = None, disable_dependencies: bool = False, execute_function: Callable = execute_with_pysqa, wait: bool = True, diff --git a/src/executorlib/task_scheduler/interactive/blockallocation.py b/src/executorlib/task_scheduler/interactive/blockallocation.py index e9ee1fd27..2b254a1c1 100644 --- a/src/executorlib/task_scheduler/interactive/blockallocation.py +++ b/src/executorlib/task_scheduler/interactive/blockallocation.py @@ -1,8 +1,9 @@ import queue import random +from _thread import LockType +from collections.abc import Callable from concurrent.futures import Future from threading import Event, Lock, Thread -from typing import Callable, Optional from executorlib.standalone.command import get_interactive_execute_command from executorlib.standalone.inputcheck import ( @@ -64,7 +65,7 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase): def __init__( self, max_workers: int = 1, - executor_kwargs: Optional[dict] = None, + executor_kwargs: dict | None = None, spawner: type[BaseSpawner] = MpiExecSpawner, validator: Callable = validate_resource_dict, restart_limit: int = 0, @@ -149,7 +150,7 @@ def max_workers(self, max_workers: int): self._max_workers = max_workers def submit( # type: ignore - self, fn: Callable, *args, resource_dict: Optional[dict] = None, **kwargs + self, fn: Callable, *args, resource_dict: dict | None = None, **kwargs ) -> Future: """ Submits a callable to be executed with the given arguments. @@ -229,20 +230,20 @@ def _execute_multiple_tasks( future_queue: queue.Queue, cores: int = 1, spawner: type[BaseSpawner] = MpiExecSpawner, - hostname_localhost: Optional[bool] = None, - init_function: Optional[Callable] = None, - cache_directory: Optional[str] = None, - cache_key: Optional[str] = None, + hostname_localhost: bool | None = None, + init_function: Callable | None = None, + cache_directory: str | None = None, + cache_key: str | None = None, queue_join_on_shutdown: bool = True, log_obj_size: bool = False, - error_log_file: Optional[str] = None, + error_log_file: str | None = None, worker_id: int = 0, - stop_function: Optional[Callable] = None, + stop_function: Callable | None = None, restart_limit: int = 0, - bootup_event: Optional[Event] = None, - next_bootup_event: Optional[Event] = None, - alive_workers: Optional[list] = None, - alive_workers_lock: Optional[Lock] = None, + bootup_event: Event | None = None, + next_bootup_event: Event | None = None, + alive_workers: list | None = None, + alive_workers_lock: LockType | None = None, **kwargs, ) -> None: """ @@ -344,8 +345,8 @@ def _execute_multiple_tasks( def _drain_dead_worker( future_queue: queue.Queue, - alive_workers: Optional[list] = None, - alive_workers_lock: Optional[Lock] = None, + alive_workers: list | None = None, + alive_workers_lock: LockType | None = None, ) -> None: """Handle a permanently dead worker by recycling or failing its tasks. @@ -385,8 +386,8 @@ def _drain_dead_worker( def _set_init_function( interface: SocketInterface, - init_function: Optional[Callable] = None, -) -> Optional[Exception]: + init_function: Callable | None = None, +) -> Exception | None: interface_initialization_exception = None if init_function is not None and interface.status: output = interface.send_and_receive_dict( diff --git a/src/executorlib/task_scheduler/interactive/dependency.py b/src/executorlib/task_scheduler/interactive/dependency.py index e781195d5..afb5c3b19 100644 --- a/src/executorlib/task_scheduler/interactive/dependency.py +++ b/src/executorlib/task_scheduler/interactive/dependency.py @@ -1,8 +1,9 @@ import queue +from collections.abc import Callable from concurrent.futures import Future from threading import Thread from time import sleep -from typing import Any, Callable, Optional +from typing import Any from executorlib.standalone.batched import batched_futures from executorlib.standalone.interactive.arguments import ( @@ -43,11 +44,11 @@ class DependencyTaskScheduler(TaskSchedulerBase): def __init__( self, executor: TaskSchedulerBase, - max_cores: Optional[int] = None, + max_cores: int | None = None, refresh_rate: float = 0.01, plot_dependency_graph: bool = False, - plot_dependency_graph_filename: Optional[str] = None, - export_workflow_filename: Optional[str] = None, + plot_dependency_graph_filename: str | None = None, + export_workflow_filename: str | None = None, validator: Callable = validate_resource_dict, ) -> None: super().__init__(max_cores=max_cores, validator=validator) @@ -73,7 +74,7 @@ def __init__( self._generate_dependency_graph = True @property - def info(self) -> Optional[dict]: + def info(self) -> dict | None: """ Get the information about the executor. @@ -90,7 +91,7 @@ def info(self) -> Optional[dict]: return None @property - def max_workers(self) -> Optional[int]: + def max_workers(self) -> int | None: if isinstance(self._future_queue, queue.Queue): f: Future = Future() self._future_queue.queue.insert( @@ -120,7 +121,7 @@ def submit( # type: ignore self, fn: Callable[..., Any], *args: Any, - resource_dict: Optional[dict[str, Any]] = None, + resource_dict: dict[str, Any] | None = None, **kwargs: Any, ) -> Future: """ diff --git a/src/executorlib/task_scheduler/interactive/dependency_plot.py b/src/executorlib/task_scheduler/interactive/dependency_plot.py index b0bac9aad..488e778f3 100644 --- a/src/executorlib/task_scheduler/interactive/dependency_plot.py +++ b/src/executorlib/task_scheduler/interactive/dependency_plot.py @@ -2,7 +2,6 @@ import json import os.path from concurrent.futures import Future -from typing import Optional import cloudpickle @@ -203,7 +202,7 @@ def convert_arg(arg, future_hash_inverse_dict): def plot_dependency_graph_function( - node_lst: list, edge_lst: list, filename: Optional[str] = None + node_lst: list, edge_lst: list, filename: str | None = None ): """ Draw the graph visualization of nodes and edges. diff --git a/src/executorlib/task_scheduler/interactive/onetoone.py b/src/executorlib/task_scheduler/interactive/onetoone.py index 52c06976e..38a675e60 100644 --- a/src/executorlib/task_scheduler/interactive/onetoone.py +++ b/src/executorlib/task_scheduler/interactive/onetoone.py @@ -1,7 +1,7 @@ import queue +from collections.abc import Callable from concurrent.futures import Future from threading import Thread -from typing import Callable, Optional from executorlib.standalone.command import get_interactive_execute_command from executorlib.standalone.interactive.communication import ( @@ -47,9 +47,9 @@ class OneProcessTaskScheduler(TaskSchedulerBase): def __init__( self, - max_cores: Optional[int] = None, - max_workers: Optional[int] = None, - executor_kwargs: Optional[dict] = None, + max_cores: int | None = None, + max_workers: int | None = None, + executor_kwargs: dict | None = None, spawner: type[BaseSpawner] = MpiExecSpawner, validator: Callable = validate_resource_dict, ): @@ -78,9 +78,9 @@ def __init__( def _execute_single_task( future_queue: queue.Queue, spawner: type[BaseSpawner] = MpiExecSpawner, - max_cores: Optional[int] = None, - max_workers: Optional[int] = None, - hostname_localhost: Optional[bool] = None, + max_cores: int | None = None, + max_workers: int | None = None, + hostname_localhost: bool | None = None, **kwargs, ): """ @@ -130,8 +130,8 @@ def _execute_single_task( def _wait_for_free_slots( active_task_dict: dict, cores_requested: int, - max_cores: Optional[int] = None, - max_workers: Optional[int] = None, + max_cores: int | None = None, + max_workers: int | None = None, ) -> dict: """ Wait for available computing resources to become available. @@ -165,9 +165,9 @@ def _wrap_execute_task_in_separate_process( active_task_dict: dict, spawner: type[BaseSpawner], executor_kwargs: dict, - max_cores: Optional[int] = None, - max_workers: Optional[int] = None, - hostname_localhost: Optional[bool] = None, + max_cores: int | None = None, + max_workers: int | None = None, + hostname_localhost: bool | None = None, ): """ Submit function to be executed in separate Python process @@ -229,12 +229,12 @@ def _execute_task_in_thread( future_obj: Future, cores: int = 1, spawner: type[BaseSpawner] = MpiExecSpawner, - hostname_localhost: Optional[bool] = None, - cache_directory: Optional[str] = None, - cache_key: Optional[str] = None, + hostname_localhost: bool | None = None, + cache_directory: str | None = None, + cache_key: str | None = None, log_obj_size: bool = False, - error_log_file: Optional[str] = None, - worker_id: Optional[int] = None, + error_log_file: str | None = None, + worker_id: int | None = None, **kwargs, ): """ diff --git a/src/executorlib/task_scheduler/interactive/shared.py b/src/executorlib/task_scheduler/interactive/shared.py index abb633ae3..3d33c400b 100644 --- a/src/executorlib/task_scheduler/interactive/shared.py +++ b/src/executorlib/task_scheduler/interactive/shared.py @@ -4,7 +4,6 @@ import time from concurrent.futures import Future from concurrent.futures._base import PENDING -from typing import Optional from executorlib.standalone.interactive.communication import ( ExecutorlibSocketError, @@ -17,9 +16,9 @@ def execute_task_dict( task_dict: dict, future_obj: Future, interface: SocketInterface, - cache_directory: Optional[str] = None, - cache_key: Optional[str] = None, - error_log_file: Optional[str] = None, + cache_directory: str | None = None, + cache_key: str | None = None, + error_log_file: str | None = None, ) -> bool: """ Execute the task in the task_dict by communicating it via the interface. @@ -120,7 +119,7 @@ def _execute_task_with_cache( task_dict: dict, future_obj: Future, cache_directory: str, - cache_key: Optional[str] = None, + cache_key: str | None = None, ) -> bool: """ Execute the task in the task_dict by communicating it via the interface using the cache in the cache directory. diff --git a/src/executorlib/task_scheduler/interactive/spawner_flux.py b/src/executorlib/task_scheduler/interactive/spawner_flux.py index 5989c624c..2a52c860d 100644 --- a/src/executorlib/task_scheduler/interactive/spawner_flux.py +++ b/src/executorlib/task_scheduler/interactive/spawner_flux.py @@ -1,6 +1,6 @@ import contextlib import os -from typing import Callable, Optional +from collections.abc import Callable import flux import flux.job @@ -49,20 +49,20 @@ class FluxPythonSpawner(BaseSpawner): def __init__( self, - cwd: Optional[str] = None, + cwd: str | None = None, cores: int = 1, threads_per_core: int = 1, gpus_per_core: int = 0, - num_nodes: Optional[int] = None, + num_nodes: int | None = None, worker_id: int = 0, exclusive: bool = False, - priority: Optional[int] = None, + priority: int | None = None, openmpi_oversubscribe: bool = False, - pmi_mode: Optional[str] = None, - flux_executor: Optional[flux.job.FluxExecutor] = None, + pmi_mode: str | None = None, + flux_executor: flux.job.FluxExecutor | None = None, flux_executor_nesting: bool = False, flux_log_files: bool = False, - run_time_max: Optional[int] = None, + run_time_max: int | None = None, ): super().__init__( cwd=cwd, @@ -85,7 +85,7 @@ def __init__( def bootup( self, command_lst: list[str], - stop_function: Optional[Callable] = None, + stop_function: Callable | None = None, ) -> bool: """ Boot up the client process to connect to the SocketInterface. diff --git a/src/executorlib/task_scheduler/interactive/spawner_pysqa.py b/src/executorlib/task_scheduler/interactive/spawner_pysqa.py index c34b6fedb..cb5984a35 100644 --- a/src/executorlib/task_scheduler/interactive/spawner_pysqa.py +++ b/src/executorlib/task_scheduler/interactive/spawner_pysqa.py @@ -1,7 +1,7 @@ import hashlib import os +from collections.abc import Callable from time import sleep -from typing import Callable, Optional from pysqa import QueueAdapter @@ -23,19 +23,19 @@ class PysqaSpawner(BaseSpawner): def __init__( self, - cwd: Optional[str] = None, + cwd: str | None = None, cores: int = 1, threads_per_core: int = 1, gpus_per_core: int = 0, - num_nodes: Optional[int] = None, + num_nodes: int | None = None, worker_id: int = 0, exclusive: bool = False, openmpi_oversubscribe: bool = False, - slurm_cmd_args: Optional[list[str]] = None, - pmi_mode: Optional[str] = None, - config_directory: Optional[str] = None, - backend: Optional[str] = None, - run_time_max: Optional[int] = None, + slurm_cmd_args: list[str] | None = None, + pmi_mode: str | None = None, + config_directory: str | None = None, + backend: str | None = None, + run_time_max: int | None = None, **kwargs, ): """ @@ -72,14 +72,14 @@ def __init__( self._config_directory = config_directory self._backend = backend self._pysqa_submission_kwargs = kwargs - self._process: Optional[int] = None - self._queue_adapter: Optional[QueueAdapter] = None + self._process: int | None = None + self._queue_adapter: QueueAdapter | None = None self._run_time_max = run_time_max def bootup( self, command_lst: list[str], - stop_function: Optional[Callable] = None, + stop_function: Callable | None = None, ): """ Method to start the subprocess interface. @@ -224,16 +224,16 @@ def __del__(self): def create_pysqa_block_allocation_scheduler( - max_cores: Optional[int] = None, - cache_directory: Optional[str] = None, - hostname_localhost: Optional[bool] = None, + max_cores: int | None = None, + cache_directory: str | None = None, + hostname_localhost: bool | None = None, log_obj_size: bool = False, - pmi_mode: Optional[str] = None, - init_function: Optional[Callable] = None, - max_workers: Optional[int] = None, - executor_kwargs: Optional[dict] = None, - pysqa_config_directory: Optional[str] = None, - backend: Optional[str] = None, + pmi_mode: str | None = None, + init_function: Callable | None = None, + max_workers: int | None = None, + executor_kwargs: dict | None = None, + pysqa_config_directory: str | None = None, + backend: str | None = None, validator: Callable = validate_resource_dict, ): check_init_function_with_cache( diff --git a/src/executorlib/task_scheduler/interactive/spawner_slurm.py b/src/executorlib/task_scheduler/interactive/spawner_slurm.py index 34262b29a..c0ec9db88 100644 --- a/src/executorlib/task_scheduler/interactive/spawner_slurm.py +++ b/src/executorlib/task_scheduler/interactive/spawner_slurm.py @@ -1,5 +1,4 @@ import os -from typing import Optional from executorlib.standalone.command import generate_slurm_command from executorlib.standalone.interactive.spawner import SubprocessSpawner @@ -22,17 +21,17 @@ def validate_max_workers(max_workers: int, cores: int, threads_per_core: int): class SrunSpawner(SubprocessSpawner): def __init__( self, - cwd: Optional[str] = None, + cwd: str | None = None, cores: int = 1, threads_per_core: int = 1, gpus_per_core: int = 0, - num_nodes: Optional[int] = None, + num_nodes: int | None = None, worker_id: int = 0, exclusive: bool = False, openmpi_oversubscribe: bool = False, - slurm_cmd_args: Optional[list[str]] = None, - pmi_mode: Optional[str] = None, - run_time_max: Optional[int] = None, + slurm_cmd_args: list[str] | None = None, + pmi_mode: str | None = None, + run_time_max: int | None = None, ): """ Srun interface implementation.