diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9d850cb9..930d09ea 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Inspects activated or project virtual environments, including with `uv tool install`. + Global installations now inspect activated environments. `rsconnect preflight` checks + the same environment's Python version. Use `--python` to override selection. (#889) - **Breaking:** Posit Connect content can be targeted only by GUID. `--app-id` with a numeric Connect content ID fails with an error explaining how to find the content's GUID, and rsconnect-python no longer calls the legacy diff --git a/docs/deploying.md b/docs/deploying.md index 1c0ef83c..9b9774e1 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -20,9 +20,14 @@ be included in the bundle. It must specify the package dependencies needed to ex the notebook. Posit Connect will reconstruct the Python environment using the specified package list. -If there is no `requirements.txt` file or the `--force-generate` option is specified, -the package dependencies will be determined from the current Python environment, or -from an alternative Python executable specified via the `--python` option: +With `--force-generate`, rsconnect-python determines package dependencies from the +selected Python environment. It checks `--python`, `VIRTUAL_ENV`, a non-base +`CONDA_PREFIX`, a project environment (`.venv` in the +content directory or any parent directory), and a base `CONDA_PREFIX`, in that order, +like `uv pip`. If none is available, it uses the Python interpreter running +rsconnect-python. + +To select a Python executable explicitly: ```bash rsconnect deploy notebook --python /path/to/python my-notebook.ipynb @@ -35,12 +40,8 @@ ensuring that you use the same Python that you use to run your Jupyter Notebook: /path/to/python -m pip list --format=freeze ``` -Generating requirements from an environment requires pip 20.3 or later in the -selected Python environment. Pip 20.3 introduced named `pip freeze --exclude` -options and supports Python 3.8, the oldest Python version supported by -rsconnect-python. Upgrade pip in older environments or supply a requirements -file. Newer Python versions may need a later pip release. This requirement -applies to inspection, not the installer used by Posit Connect. +If the environment does not have pip 20.3 or later, rsconnect uses uv to generate +requirements. To inspect it manually, use `uv pip freeze --python /path/to/python`. #### Python Version @@ -91,6 +92,8 @@ file, then publish the HTML file to the Posit Connect server: rsconnect deploy notebook --static my-notebook.ipynb ``` +Install `nbconvert` in the selected Python environment to render static notebooks. + ### Creating a Manifest for Future Deployment You can create a `manifest.json` file for a Jupyter Notebook, then use that manifest @@ -98,7 +101,7 @@ in a later deployment. Use the `write-manifest` command to do this. The `write-manifest` command will also create a `requirements.txt` file, if it does not already exist or the `--force-generate` option is specified. It will contain the -package dependencies from the current Python environment, or from an alternative +package dependencies from the selected Python environment, or from an alternative Python executable specified in the `--python` option. Here is an example of the `write-manifest` command: @@ -203,9 +206,8 @@ included in the bundle. It must specify the package dependencies needed to execu the API or application. Posit Connect will reconstruct the Python environment using the specified package list. -If there is no `requirements.txt` file or the `--force-generate` option is specified, -the package dependencies will be determined from the current Python environment, or -from an alternative Python executable specified via the `--python` option: +With `--force-generate`, rsconnect-python determines package dependencies from the +selected Python environment. Use `--python` to select an interpreter explicitly: ```bash rsconnect deploy api --python /path/to/python my-api/ @@ -218,6 +220,8 @@ ensuring that you use the same Python that you use to run your API or applicatio /path/to/python -m pip list --format=freeze ``` +If the environment does not have pip, use `uv pip freeze --python /path/to/python` instead. + #### Python Version When deploying Python content to Posit Connect, @@ -284,7 +288,7 @@ manifest in a later deployment. Use the `write-manifest` command to do this. The `write-manifest` command will also create a `requirements.txt` file, if it does not already exist or the `--force-generate` option is specified. It will contain -the package dependencies from the current Python environment, or from an alternative +the package dependencies from the selected Python environment, or from an alternative Python executable specified in the `--python` option. Here is an example of the `write-manifest` command: @@ -643,8 +647,7 @@ rsconnect deploy notebook \ my-notebook.ipynb ``` -By default, rsconnect-python does not install Jupyter notebook-related depenencies. -To use these hide input features in rsconnect-python you need to install these extra dependencies: +To use these hide input features, install these dependencies in the selected Python environment: ``` notebook diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f8b74b9f..786fb449 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -260,9 +260,10 @@ def __init__( if environment: pm_name = str(environment.package_manager) pm_version_value = getattr(environment, pm_name, None) - if pm_version_value is None: + if pm_version_value is None and pm_name != "uv": # Fallback: use pip version if available; otherwise empty string - pm_version_value = getattr(environment, "pip", "") + pm_version_value = getattr(environment, "pip", None) or "" + pm_version_value = pm_version_value or "" pm: ManifestDataPythonPackageManager = { "name": pm_name, "version": pm_version_value, @@ -823,8 +824,16 @@ def make_notebook_html_bundle( else: cmd.append("--TagRemovePreprocessor.remove_input_tags=['hide_input']") try: - output = check_output(cmd) + output = check_output(cmd, stderr=subprocess.PIPE) except subprocess.CalledProcessError as error: + if error.stderr: + sys.__stderr__.write(error.stderr.decode("utf-8", errors="replace")) + sys.__stderr__.flush() + if error.stderr and b"No module named nbconvert" in error.stderr: + raise RSConnectException( + f"nbconvert is not installed in the selected Python environment ({python}). " + "Install nbconvert in that environment to deploy a static notebook." + ) from error raise RSConnectException(f"Rendering {filename} with nbconvert failed (see output above)") from error nb_name = basename(filename) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index f9dad6d3..00fc653e 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -3,8 +3,7 @@ Given a directory and a Python executable, this module inspects the environment and returns information about the Python version and the environment itself. -To inspect the environment it relies on a subprocess that runs the `rsconnect.subprocesses.inspect_environment` -module. This module is responsible for gathering the environment information and returning it in a JSON format. +The CLI runs the inspector and queries the selected project interpreter. """ import typing @@ -16,8 +15,10 @@ import pathlib import os.path import enum +import uv from . import pyproject +from .subprocesses import inspect_environment from .log import logger, warn_user from .exception import RSConnectException from .subprocesses.inspect_environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData @@ -60,6 +61,7 @@ def __init__( # Fields that are not loaded from the environment subprocess self.python_version_requirement = python_version_requirement self.python_interpreter = python_interpreter + self.uv: typing.Optional[str] = None # Optional override of server install behavior. If None, server-driven # default is used. self.package_manager_allow_uv: typing.Optional[bool] = None @@ -120,7 +122,7 @@ def create_python_environment( ) -> "Environment": """Given a project directory and a Python executable, return Environment information. - If no Python executable is provided, the current system Python executable is used. + If no Python executable is provided, select the project environment when available. :param directory: the project directory to inspect. :param requirements_file: requirements file name relative to the project directory. If None, @@ -143,14 +145,6 @@ def create_python_environment( requirements_file = _check_requirements_file(directory, requirements_file) - if python is not None: - # TODO: Remove the option in a future release - logger.warning( - "On modern Posit Connect versions, the --python option won't influence " - "the Python version used to deploy the application anymore. " - "Please use a .python-version file to force a specific interpreter version." - ) - if override_python_version: # TODO: Remove the option in a future release logger.warning( @@ -160,7 +154,9 @@ def create_python_environment( python_version_requirement = f"=={override_python_version}" # with cli_feedback("Inspecting Python environment"): - environment = cls._get_python_env_info(module_file, python, requirements_file=requirements_file) + environment = cls._get_python_env_info( + module_file, python, requirements_file=requirements_file, project_directory=directory + ) environment.python_version_requirement = python_version_requirement if override_python_version: @@ -178,6 +174,16 @@ def create_python_environment( # Derive allow_uv from selection environment.package_manager_allow_uv = selected_package_manager is PackageInstaller.UV + if environment.package_manager == PackageInstaller.UV: + try: + uv_version = ( + subprocess.check_output([environment.uv_executable, "--version"], text=True).strip().split() + ) + if len(uv_version) >= 2 and uv_version[0] == "uv": + environment.uv = uv_version[1] + except (OSError, subprocess.SubprocessError): + logger.debug("Could not determine uv version", exc_info=True) + if requirements_file is None: _warn_on_ignored_requirements(directory, environment.filename) @@ -189,6 +195,7 @@ def _get_python_env_info( file_name: str, python: typing.Optional[str], requirements_file: typing.Optional[str] = "requirements.txt", + project_directory: typing.Optional[str] = None, ) -> "Environment": """ Gathers the python and environment information relating to the specified file @@ -200,12 +207,11 @@ def _get_python_env_info( :return: information about the version of Python in use plus some environmental stuff. """ - python = which_python(python) - logger.debug("Python: %s" % python) + python = which_python(python, project_directory or os.path.dirname(file_name)) + logger.info("Using Python: %s", python) environment = cls._inspect_environment(python, os.path.dirname(file_name), requirements_file=requirements_file) if environment.error: raise RSConnectException(environment.error) - logger.debug("Python: %s" % python) logger.debug("Environment: %s" % pprint.pformat(environment._asdict())) return environment @@ -217,24 +223,34 @@ def _inspect_environment( requirements_file: typing.Optional[str] = "requirements.txt", check_output: typing.Callable[..., bytes] = subprocess.check_output, ) -> "Environment": - """Run the environment inspector using the specified python binary. + """Run the environment inspector against the specified python binary. Returns a dictionary of information about the environment, or containing an "error" field if an error occurred. """ - args = [python, "-m", "rsconnect.subprocesses.inspect_environment"] + uv_executable = _uv_bin() + # No -I/-E/-s: the inspector imports rsconnect's dependencies, which may live in user + # site-packages or on PYTHONPATH just like the running CLI's. + args = [ + sys.executable, + inspect_environment.__file__, + "--python", + os.path.abspath(python), + "--uv", + uv_executable, + ] args.extend(["--requirements-file", requirements_file or "none"]) args.append(directory) try: environment_json = check_output(args, text=True) except Exception as e: - raise RSConnectException("Error inspecting environment (subprocess failed)") from e + raise RSConnectException(f"Error inspecting environment with Python {python} (subprocess failed)") from e try: environment_data = json.loads(environment_json) except json.JSONDecodeError as e: - raise RSConnectException("Error parsing environment JSON") from e + raise RSConnectException(f"Error parsing environment JSON from Python {python}") from e if "error" in environment_data: system_error_message = environment_data.get("error") @@ -242,29 +258,79 @@ def _inspect_environment( raise RSConnectException(f"Error creating environment: {system_error_message}") try: - return cls.from_dict(environment_data, python_interpreter=python) + environment = cls.from_dict(environment_data, python_interpreter=python) + environment.uv_executable = uv_executable + return environment except TypeError as e: raise RSConnectException("Error constructing environment object") from e -def which_python(python: typing.Optional[str] = None) -> str: +def _uv_bin() -> str: + try: + return uv.find_uv_bin() + except FileNotFoundError: + return "uv" + + +def which_python(python: typing.Optional[str] = None, directory: typing.Optional[str] = None) -> str: """Determines which Python executable to use. - If the :param python: is provided, then validation is performed to check if the path is an executable file. If - None, the invoking system Python executable location is returned. + Explicit paths take precedence over active and project environments. + """ + if python is not None: + if not os.path.exists(python): + raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") + if os.path.isdir(python): + raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") + if not os.access(python, os.X_OK): + raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") + return python + + def interpreter(environment: typing.Union[str, pathlib.Path], conda: bool = False) -> typing.Optional[str]: + executable = ( + "python.exe" if os.name == "nt" and conda else "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + root = pathlib.Path(environment) + path = root / executable + if path.is_file() and os.access(path, os.X_OK): + if root.resolve() == pathlib.Path(sys.prefix).resolve(): + return sys.executable + return str(path) + return None + + active = os.environ.get("VIRTUAL_ENV") + if active and (selected := interpreter(active)): + return selected + + conda = os.environ.get("CONDA_PREFIX") + conda_is_base = bool(conda) and _is_conda_base(pathlib.Path(conda)) + if conda and not conda_is_base and (selected := interpreter(conda, conda=True)): + return selected + + project = pathlib.Path(os.path.abspath(directory or os.curdir)) + for parent in (project, *project.parents): + if selected := interpreter(parent / ".venv"): + return selected + + if conda and conda_is_base and (selected := interpreter(conda, conda=True)): + return selected + return sys.executable + + +def _is_conda_base(prefix: pathlib.Path) -> bool: + """Mirrors how uv tells the base conda environment from a named or path-based one. - :param python: (Optional) path to a python executable. - :return: :param python: or `sys.executable`. + uv treats the base environment like a system Python, so it ranks below project environments. """ - if python is None: - return sys.executable - if not os.path.exists(python): - raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") - if os.path.isdir(python): - raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") - if not os.access(python, os.X_OK): - raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") - return python + if (prefix / "conda-meta" / "pixi").is_file(): + return False + conda_root = os.environ.get("_CONDA_ROOT") + if conda_root and str(prefix) == conda_root: + return True + default_env = os.environ.get("CONDA_DEFAULT_ENV") + if not default_env or str(prefix) == default_env: + return False + return prefix.name != default_env def fake_module_file_from_directory(directory: str) -> str: diff --git a/rsconnect/main.py b/rsconnect/main.py index fda97767..e52863a8 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -312,6 +312,8 @@ def _validate_preflight_options(ctx: click.Context) -> None: if params["runtime"] == "nodejs": if params["fix"]: raise click.UsageError("--fix applies only to Python projects; Node.js metadata is left unchanged.") + if params["python"]: + raise click.UsageError("--python applies only to --runtime python.") elif params["node"]: raise click.UsageError("--node requires --runtime nodejs.") @@ -1388,6 +1390,11 @@ def details( @click.option("--app-id", "-a", help="Check an existing content item by GUID. Prefer --guid.") @click.option("--runtime", type=click.Choice(["python", "nodejs"]), default="python", show_default=True) @click.option("--node", type=click.Path(exists=True, dir_okay=False), help="Node.js executable for --runtime nodejs.") +@click.option( + "--python", + type=click.Path(exists=True), + help="Path to the Python interpreter whose version is checked. Defaults to the project environment.", +) @cli_exception_handler @preflight_exception_handler @click.pass_context @@ -1400,6 +1407,7 @@ def preflight( app_id: Optional[str], runtime: str, node: Optional[str], + python: Optional[str], name: Optional[str], server: Optional[str], api_key: Optional[str], @@ -1433,7 +1441,7 @@ def preflight( result = run_node_preflight(ce, directory, node) else: - result = run_preflight(ce, directory, fix) + result = run_preflight(ce, directory, fix, python) click.echo(json.dumps(result)) if result["status"] == "incompatible": sys.exit(3) @@ -2131,10 +2139,7 @@ def _warn_on_ignored_requirements(directory: str, requirements_file_name: str): "--python", "-p", type=click.Path(exists=True), - help=( - "Path to Python interpreter whose environment should be used. " - "The Python environment must have the rsconnect package installed." - ), + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -2328,10 +2333,7 @@ def deploy_notebook( "--python", "-p", type=click.Path(exists=True), - help=( - "Path to Python interpreter whose environment should be used. " - "The Python environment must have the rsconnect package installed." - ), + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -3028,10 +3030,7 @@ def deploy_git( "--python", "-p", type=click.Path(exists=True), - help=( - "Path to Python interpreter whose environment should be used. " - "The Python environment must have the rsconnect package installed." - ), + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -3132,6 +3131,7 @@ def deploy_quarto( environment = Environment.create_python_environment( base_dir, requirements_file=requirements_file, + python=python, override_python_version=override_python_version, ) @@ -3516,10 +3516,7 @@ def generate_deploy_python( "--python", "-p", type=click.Path(exists=True), - help=( - "Path to Python interpreter whose environment should be used. " - "The Python environment must have the rsconnect package installed." - ), + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -3897,8 +3894,7 @@ def write_manifest(): "--python", "-p", type=click.Path(exists=True), - help="Path to Python interpreter whose environment should be used. " - + "The Python environment must have the rsconnect package installed.", + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -4018,8 +4014,7 @@ def write_manifest_notebook( "--python", "-p", type=click.Path(exists=True), - help="Path to Python interpreter whose environment should be used. " - + "The Python environment must have the rsconnect package installed.", + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -4175,8 +4170,7 @@ def write_manifest_voila( "--python", "-p", type=click.Path(exists=True), - help="Path to Python interpreter whose environment should be used. " - + "The Python environment must have the rsconnect package installed.", + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", @@ -4598,8 +4592,7 @@ def generate_write_manifest_python( "--python", "-p", type=click.Path(exists=True), - help="Path to Python interpreter whose environment should be used. " - + "The Python environment must have the rsconnect-python package installed.", + help="Path to Python interpreter whose environment should be used.", ) @click.option( "--override-python-version", diff --git a/rsconnect/preflight.py b/rsconnect/preflight.py index 87ab9f28..ed7165b8 100644 --- a/rsconnect/preflight.py +++ b/rsconnect/preflight.py @@ -6,6 +6,8 @@ import os import platform import shutil +import subprocess +import sys from pathlib import Path from typing import Any, List, Mapping, Optional, Tuple, Union, cast @@ -13,7 +15,7 @@ from packaging.version import InvalidVersion, Version from .api import ConnectCloudServer, RSConnectClient, RSConnectExecutor, RSConnectServer, SPCSConnectServer -from .environment import fake_module_file_from_directory +from .environment import fake_module_file_from_directory, which_python from .exception import RSConnectException from .metadata import AppStore from .validation import require_posix @@ -496,6 +498,23 @@ def _warn_local_python_mismatch(constraint: Optional[str], local_python: str, wa ) +def _local_python_version(project_dir: str, python: Optional[str]) -> str: + interpreter = which_python(python, project_dir) + if interpreter == sys.executable: + return platform.python_version() + try: + result = subprocess.run( + [interpreter, "-I", "-c", "import platform; print(platform.python_version())"], + capture_output=True, + text=True, + timeout=30, + check=True, + ) + except (OSError, subprocess.SubprocessError) as err: + raise RSConnectException(f"Could not determine the version of Python at {interpreter}: {err}") + return result.stdout.strip() + + def _validated_python_executor( executor: RSConnectExecutor, ) -> Tuple[Union[RSConnectServer, SPCSConnectServer], RSConnectClient]: @@ -516,13 +535,14 @@ def _validated_python_executor( def _python_project_metadata( - target: str, local_python: str, warnings: list[str] -) -> tuple[str, Optional[str], Optional[str]]: + target: str, python: Optional[str], warnings: list[str] +) -> tuple[str, str, Optional[str], Optional[str]]: target_path = Path(target) project_dir = str(target_path.parent) if target_path.is_file() else target + local_python = _local_python_version(project_dir, python) constraint, invalid_metadata = _project_constraint(project_dir, warnings) _warn_local_python_mismatch(constraint, local_python, warnings) - return project_dir, constraint, invalid_metadata + return project_dir, local_python, constraint, invalid_metadata def _status_advice( @@ -542,15 +562,16 @@ def _status_advice( _unknown_advice(is_new, constraint, recommended, warnings, actions) -def run_preflight(executor: RSConnectExecutor, project: str, fix: bool = False) -> dict[str, Any]: +def run_preflight( + executor: RSConnectExecutor, project: str, fix: bool = False, python: Optional[str] = None +) -> dict[str, Any]: """Check project Python metadata against a validated Connect executor.""" require_posix("Deployment preflight") server, client = _validated_python_executor(executor) warnings: list[str] = [] actions: list[str] = [] - local_python = platform.python_version() - project_dir, constraint, invalid_metadata = _python_project_metadata(project, local_python, warnings) + project_dir, local_python, constraint, invalid_metadata = _python_project_metadata(project, python, warnings) is_existing, app_id, target_issue = _deployment_target(executor, project) is_new = not is_existing if target_issue: diff --git a/rsconnect/subprocesses/inspect_environment.py b/rsconnect/subprocesses/inspect_environment.py index 69354364..6703dab2 100644 --- a/rsconnect/subprocesses/inspect_environment.py +++ b/rsconnect/subprocesses/inspect_environment.py @@ -1,9 +1,9 @@ #!/usr/bin/env python """ -Environment data class abstraction that is usable as an executable module +Environment data class abstraction that is usable as an executable script ```bash -python -m rsconnect.subprocesses.inspect_environment +python -I path/to/inspect_environment.py --python /path/to/python --requirements-file none DIRECTORY ``` """ @@ -21,21 +21,16 @@ import subprocess import sys from dataclasses import asdict, dataclass, replace -from typing import Callable, List, Optional, cast +from typing import Any, Callable, List, Optional, cast from urllib.parse import parse_qs, urlsplit +from urllib.request import url2pathname from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name from packaging.version import Version -try: - import tomllib -except ImportError: - # Python <3.11 doesn't have tomllib in the standard library - import toml as tomllib # type: ignore[no-redef] - version_re = re.compile(r"\d+\.\d+(\.\d+)?") -exec_dir = os.path.dirname(sys.executable) +module_cwd = os.path.dirname(os.path.abspath(__file__)) rsconnect_project_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) excluded_distributions = {"rsconnect", "rsconnect-python"} @@ -46,7 +41,7 @@ class EnvironmentData: filename: str locale: str package_manager: str - pip: str + pip: Optional[str] python: str source: str python_requires: Optional[str] @@ -64,7 +59,7 @@ def MakeEnvironmentData( filename: str, locale: str, package_manager: str, - pip: str, + pip: Optional[str], python: str, source: str, python_requires: Optional[str] = None, @@ -78,7 +73,12 @@ class EnvironmentException(Exception): pass -def detect_environment(dirname: str, requirements_file: Optional[str] = "requirements.txt") -> EnvironmentData: +def detect_environment( + dirname: str, + requirements_file: Optional[str] = "requirements.txt", + uv_path: str = "uv", + python: str = sys.executable, +) -> EnvironmentData: """Determine the python dependencies in the environment. `pip freeze` will be used to introspect the environment. @@ -89,33 +89,43 @@ def detect_environment(dirname: str, requirements_file: Optional[str] = "require or a dictionary containing `error` on failure. """ + python_version = get_python_version(python) + pip_version = get_version("pip", python) + result: dict[str, Any] if requirements_file is None: # --force-generate sets requirements_file to None - result = pip_freeze() + result = pip_freeze(python, uv_path, pip_version, python_version) elif os.path.basename(requirements_file) == "uv.lock": - result = uv_export(dirname, requirements_file) + result = uv_export(dirname, requirements_file, uv_path) elif os.path.basename(requirements_file) == "pyproject.toml": result = pyproject_dependencies(dirname, requirements_file) else: - result = output_file(dirname, requirements_file, "pip") - if result is None: + file_result = output_file(dirname, requirements_file, "pip") + if file_result is None: raise EnvironmentException( "The requirements file '%s' was not found in '%s'. " "Please create it or use --force-generate to use pip freeze." % (requirements_file, dirname) ) + result = file_result if result is not None: check_editables_are_bundleable(result["contents"] or "", dirname) - result["python"] = get_python_version() - result["pip"] = get_version("pip") + result["python"] = python_version + result["pip"] = pip_version result["locale"] = get_default_locale() return MakeEnvironmentData(**result) -def get_python_version() -> str: - v = sys.version_info - return "%d.%d.%d" % (v[0], v[1], v[2]) +def get_python_version(python: str = sys.executable) -> str: + try: + return subprocess.check_output( + [python, "-I", "-c", "import sys; print('%d.%d.%d' % sys.version_info[:3])"], + cwd=module_cwd, + text=True, + ).strip() + except (OSError, subprocess.SubprocessError) as exception: + raise EnvironmentException(f"Error getting Python version from {python}: {exception}") from exception def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None]] = locale.getlocale): @@ -123,16 +133,19 @@ def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None return "" if result == "." else result -def get_version(module: str): +def get_version(module: str, python: str = sys.executable) -> Optional[str]: try: - args = [sys.executable, "-m", module, "--version"] + args = [python, "-E", "-m", module, "--version"] proc = subprocess.Popen( args, + cwd=module_cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) stdout, _stderr = proc.communicate() + if proc.returncode and module == "pip": + return None match = version_re.search(stdout) if match: return match.group() @@ -174,31 +187,43 @@ def output_file(dirname: str, filename: str, package_manager: str): raise EnvironmentException("Error reading %s: %s" % (filename, str(exception))) -def pip_freeze(): +def pip_freeze(python: str, uv_path: str, pip_version: Optional[str], python_version: str): """Inspect the environment using `pip freeze`. Returns a dictionary containing the filename (always 'requirements.txt') and contents if successful, or a dictionary containing 'error' on failure. """ - pip_version = get_version("pip") - if Version(pip_version) < Version("20.3"): - raise EnvironmentException( - f"Generating requirements requires pip 20.3 or later (found {pip_version}). " - f"Upgrade pip with '{sys.executable} -m pip install --upgrade pip', " - "or provide a requirements file." - ) - try: - proc = subprocess.Popen( + # Older pip suppresses these bootstrap packages; current pip suppresses + # only pip itself on Python 3.12 and later. + bootstrap_packages = {"pip"} + if pip_version is not None or Version(python_version) < Version("3.12"): + bootstrap_packages.update({"setuptools", "wheel", "distribute"}) + command = ( [ - sys.executable, + python, + "-E", "-m", "pip", "freeze", "--disable-pip-version-check", *(arg for name in sorted(excluded_distributions) for arg in ("--exclude", name)), - ], + ] + if pip_version and Version(pip_version) >= Version("20.3") + else [ + uv_path, + "pip", + "freeze", + "--no-cache", + "--python", + python, + *(arg for name in sorted(bootstrap_packages) for arg in ("--exclude", name)), + ] + ) + proc = subprocess.Popen( + command, + cwd=module_cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, @@ -230,7 +255,7 @@ def pip_freeze(): } -def uv_export(dirname: str, lock_filename: str): +def uv_export(dirname: str, lock_filename: str, uv_path: str = "uv"): """ Export requirements from a uv.lock file using `uv export`. """ @@ -246,7 +271,7 @@ def uv_export(dirname: str, lock_filename: str): try: result = subprocess.run( [ - "uv", + uv_path, "export", "--format", "requirements-txt", @@ -306,13 +331,17 @@ def pyproject_dependencies(dirname: str, pyproject_filename: str): raise EnvironmentException(f"pyproject.toml not found: {pyproject_filename}") try: + try: + import tomllib + except ImportError: + import toml as tomllib # type: ignore[no-redef] with open(pyproject_path, "r", encoding="utf-8-sig") as f: - pyproject = tomllib.loads(f.read()) + project_data = tomllib.loads(f.read()).get("project", {}) except Exception as exception: raise EnvironmentException(f"Error reading {pyproject_filename}: {exception}") - project = pyproject.get("project", {}) - dependencies: List[object] = project.get("dependencies", []) + project = project_data or {} + dependencies: list[object] = project.get("dependencies", []) if not isinstance(dependencies, list) or not all(isinstance(dep, str) for dep in dependencies): raise EnvironmentException(f"Invalid project.dependencies in {pyproject_filename}: expected a list of strings.") @@ -381,6 +410,8 @@ def is_rsconnect_editable(target: str) -> bool: egg = parse_qs(url.fragment).get("egg") if egg: return canonicalize_name(egg[0]) in excluded_distributions + if url.scheme.lower() == "file" and url.netloc in ("", "localhost"): + target = url2pathname(url.path) return is_path(target) and os.path.isdir(target) and os.path.samefile(target, rsconnect_project_dir) @@ -429,6 +460,8 @@ def main(): default="requirements.txt", help="Requirements file name (relative to the directory). Use 'none' to capture via pip freeze.", ) + parser.add_argument("--uv", default="uv", help="Path to the uv executable.") + parser.add_argument("--python", default=sys.executable, help="Python environment to inspect.") parser.add_argument("directory", help="Directory to inspect.") args = parser.parse_args() @@ -436,7 +469,7 @@ def main(): if requirements_file.lower() == "none": requirements_file = None - envinfo = detect_environment(args.directory, requirements_file=requirements_file)._asdict() + envinfo = detect_environment(args.directory, requirements_file, args.uv, args.python)._asdict() if "contents" in envinfo: keepers = list(map(strip_ref, envinfo["contents"].split("\n"))) keepers = [line for line in keepers if not exclude(line)] diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py index 0eb8eefa..310a916e 100644 --- a/tests/test_agent_cli.py +++ b/tests/test_agent_cli.py @@ -4,6 +4,7 @@ import json import logging import os +import sys from unittest.mock import Mock, patch import click @@ -197,7 +198,7 @@ def test_preflight_json_and_exit_status(tmp_path, status, exit_code): result = cli_runner().invoke(cli, ["preflight", "-n", "target", str(tmp_path), "--fix", "--new"]) assert result.exit_code == exit_code, result.output assert json.loads(result.stdout) == expected - check.assert_called_once_with(executor, str(tmp_path), True) + check.assert_called_once_with(executor, str(tmp_path), True, None) assert create.call_args.kwargs["path"] == str(tmp_path) assert create.call_args.kwargs["new"] is True @@ -256,6 +257,7 @@ def test_node_preflight_dispatch_and_incompatible_exit(tmp_path): [ (["--runtime", "nodejs", "--fix"], "--fix applies only to Python"), (["--runtime", "unsupported"], "Invalid value"), + (["--runtime", "nodejs", "--python", sys.executable], "--python applies only to --runtime python"), ], ) def test_node_preflight_invalid_options(arguments, message): diff --git a/tests/test_bundle.py b/tests/test_bundle.py index be95257d..6cef1139 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -4,6 +4,7 @@ import json import os import sys +import subprocess import tarfile import tempfile from os.path import abspath, basename, dirname, join @@ -12,6 +13,7 @@ from unittest import TestCase, mock import pytest +import uv import rsconnect.bundle from rsconnect.bundle import ( @@ -3552,6 +3554,17 @@ def check_output(command): assert callback_calls == [] +def test_static_notebook_reports_missing_nbconvert_in_selected_environment(tmp_path): + environment_dir = tmp_path / "env" + subprocess.run([uv.find_uv_bin(), "venv", "--python", sys.executable, str(environment_dir)], check=True) + python = environment_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + notebook = tmp_path / "notebook.ipynb" + notebook.write_text("{}") + + with pytest.raises(RSConnectException, match="nbconvert is not installed in the selected Python environment"): + make_notebook_html_bundle(str(notebook), str(python), hide_all_input=False, hide_tagged_input=False) + + def test_standalone_quarto_bundle_filters_config_extras(tmp_path, monkeypatch): project = tmp_path / "project" project.mkdir() diff --git a/tests/test_environment.py b/tests/test_environment.py index d1d03ec2..01d50339 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,16 +1,18 @@ import re import sys import os +import json import tempfile import shutil import subprocess +import zipfile from pathlib import Path from unittest import TestCase from unittest import mock import rsconnect.environment from rsconnect.exception import RSConnectException -from rsconnect.environment import Environment, which_python +from rsconnect.environment import Environment, PackageInstaller, which_python from rsconnect.subprocesses.inspect_environment import ( EnvironmentException, detect_environment, @@ -24,6 +26,7 @@ from .utils import get_dir import pytest +import uv version_re = re.compile(r"\d+\.\d+(\.\d+)?") TESTDATA = os.path.join(os.path.dirname(__file__), "testdata") @@ -241,42 +244,57 @@ def test_check_editables_reports_all_unbundleable_editables(tmp_path: Path): assert "-e /work/two" in str(error.value) -def test_inspect_environment_reports_local_editable(tmp_path: Path, monkeypatch): - pip_dir = tmp_path / "pip" +def missing_bundled_uv(): + raise FileNotFoundError("uv") + + +def make_environment_with_fake_pip(tmp_path: Path, version: str): + env_dir = tmp_path / "env" + subprocess.run([uv.find_uv_bin(), "venv", "--python", sys.executable, str(env_dir)], check=True) + python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + site = Path( + subprocess.check_output( + [str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], text=True + ).strip() + ) + pip_dir = site / "pip" pip_dir.mkdir() (pip_dir / "__init__.py").write_text("") (pip_dir / "__main__.py").write_text( "import sys\n" "if '--version' in sys.argv:\n" - " print('pip 20.3')\n" + f" print('pip {version}')\n" "else:\n" + " assert '--exclude' in sys.argv and 'rsconnect-python' in sys.argv and 'rsconnect' in sys.argv\n" " print('# Editable install with no version control (demo==1.0)')\n" " print('-e /work/demo')\n" ) - monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])])) + return python, site + + +def test_inspect_environment_reports_local_editable(tmp_path: Path): + python, _ = make_environment_with_fake_pip(tmp_path, "20.3") with pytest.raises( RSConnectException, match="Editable dependencies outside the deployment directory cannot be installed on Connect", ) as error: - Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None) + Environment._inspect_environment(str(python), str(tmp_path), requirements_file=None) assert "-e /work/demo" in str(error.value) assert "requirements file" in str(error.value) -def test_inspect_environment_rejects_old_pip_for_generated_requirements(tmp_path: Path, monkeypatch): - pip_dir = tmp_path / "pip" - pip_dir.mkdir() - (pip_dir / "__init__.py").write_text("") - (pip_dir / "__main__.py").write_text("print('pip 20.2.4')\n") - monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])])) +def test_inspect_environment_uses_uv_for_old_pip(tmp_path: Path): + python, site = make_environment_with_fake_pip(tmp_path, "20.2.4") + package = site / "demo_package-1.0.dist-info" + package.mkdir() + (package / "METADATA").write_text("Metadata-Version: 2.1\nName: demo-package\nVersion: 1.0\n") - with pytest.raises(RSConnectException, match=r"requires pip 20\.3 or later \(found 20\.2\.4\)") as error: - Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None) + environment = Environment._inspect_environment(str(python), str(tmp_path), requirements_file=None) - assert "install --upgrade pip" in str(error.value) - assert "requirements file" in str(error.value) + assert environment.pip == "20.2.4" + assert "demo-package==1.0" in environment.contents.splitlines() def test_requirements_file_filters_only_rsconnect_distribution(tmp_path: Path): @@ -302,7 +320,8 @@ def test_requirements_file_rejects_editable_outside_bundle(tmp_path: Path): assert "-e /Users/me/localpkg" in str(error.value) -def test_uv_lock_export(tmp_path): +@pytest.mark.parametrize("fallback", [False, True]) +def test_uv_lock_export(tmp_path, monkeypatch, fallback): project_dir = tmp_path / "project" project_dir.mkdir() (project_dir / "pyproject.toml").write_text( @@ -367,7 +386,19 @@ def test_uv_lock_export(tmp_path): encoding="utf-8", ) - env = detect_environment(str(project_dir), requirements_file="uv.lock") + from rsconnect.bundle import Manifest + + uv_path = uv.find_uv_bin() + expected_uv_version = subprocess.check_output([uv_path, "--version"], text=True).split()[1] + env_dir = project_dir / ".venv" + subprocess.run([uv_path, "venv", "--python", sys.executable, str(env_dir)], check=True) + python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + if fallback: + monkeypatch.setenv("PATH", os.path.dirname(uv_path)) + monkeypatch.setattr(rsconnect.environment.uv, "find_uv_bin", missing_bundled_uv) + else: + monkeypatch.setenv("PATH", "") + env = Environment.create_python_environment(str(project_dir), requirements_file="uv.lock", python=str(python)) assert env.filename == "requirements.txt.lock" assert "aiofiles==24.1.0" in env.contents @@ -375,6 +406,8 @@ def test_uv_lock_export(tmp_path): assert "annotated-types==0.7.0" in env.contents assert env.source == "uv_lock" assert env.package_manager == "uv" + assert env.pip is None + assert Manifest(environment=env).data["python"]["package_manager"]["version"] == expected_uv_version @pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) @@ -481,6 +514,17 @@ def test_pyproject_dependencies_no_project_table(tmp_path): class WhichPythonTestCase(TestCase): + def setUp(self): + # which_python() falls back to sys.executable only without an active or project environment. + environ = {name: value for name, value in os.environ.items() if name not in ("VIRTUAL_ENV", "CONDA_PREFIX")} + patcher = mock.patch.dict(os.environ, environ, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + workdir = tempfile.TemporaryDirectory() + self.addCleanup(workdir.cleanup) + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(workdir.name) + def test_default(self): self.assertEqual(which_python(), sys.executable) @@ -508,6 +552,342 @@ def test_is_not_executable(self): which_python(tmpfile.name) +def test_python_environment_lookup_order(tmp_path, monkeypatch): + for name in ("VIRTUAL_ENV", "CONDA_PREFIX"): + monkeypatch.delenv(name, raising=False) + + def make_interpreter(environment, conda=False): + executable = ( + "python.exe" if os.name == "nt" and conda else "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + path = environment / executable + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + path.chmod(0o755) + return str(path) + + project = tmp_path / "project" + nested = project / "content" / "nested" + nested.mkdir(parents=True) + activated = make_interpreter(tmp_path / "active") + parent = make_interpreter(project / ".venv") + nearest = make_interpreter(project / "content" / ".venv") + conda = make_interpreter(tmp_path / "conda", conda=True) + + assert which_python(directory=str(nested)) == nearest + assert which_python(directory=str(project)) == parent + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path / "conda")) + assert which_python(directory=str(nested)) == conda + monkeypatch.setenv("VIRTUAL_ENV", str(tmp_path / "active")) + assert which_python(directory=str(nested)) == activated + assert which_python(sys.executable, directory=str(nested)) == sys.executable + monkeypatch.delenv("VIRTUAL_ENV") + monkeypatch.delenv("CONDA_PREFIX") + assert which_python(directory=str(nested)) == nearest + + +@pytest.mark.skipif(os.name == "nt", reason="uses the POSIX environment layout") +def test_conda_environment_ranks_by_kind_like_uv(tmp_path, monkeypatch): + for name in ("VIRTUAL_ENV", "CONDA_PREFIX", "_CONDA_ROOT", "CONDA_DEFAULT_ENV"): + monkeypatch.delenv(name, raising=False) + + def make_interpreter(environment): + path = environment / "bin/python" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + path.chmod(0o755) + return str(path) + + project = tmp_path / "project" + project.mkdir() + venv = make_interpreter(project / ".venv") + prefix = tmp_path / "envs" / "analysis" + conda = make_interpreter(prefix) + monkeypatch.setenv("CONDA_PREFIX", str(prefix)) + + # No CONDA_DEFAULT_ENV, so the prefix is treated as a path-based (non-base) environment. + assert which_python(directory=str(project)) == conda + + monkeypatch.setenv("CONDA_DEFAULT_ENV", "analysis") + assert which_python(directory=str(project)) == conda + + # A base environment ranks below the project environment. + monkeypatch.setenv("CONDA_DEFAULT_ENV", "base") + assert which_python(directory=str(project)) == venv + + monkeypatch.setenv("_CONDA_ROOT", str(prefix)) + monkeypatch.setenv("CONDA_DEFAULT_ENV", "analysis") + assert which_python(directory=str(project)) == venv + + # Without a project environment, the base environment is still used. + (project / ".venv" / "bin" / "python").unlink() + assert which_python(directory=str(project)) == conda + + +def test_project_environment_lookup_ignores_non_ancestors(tmp_path, monkeypatch): + for name in ("VIRTUAL_ENV", "CONDA_PREFIX"): + monkeypatch.delenv(name, raising=False) + current = tmp_path / "current" + (tmp_path / "sibling").mkdir() + python = current / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + python.parent.mkdir(parents=True) + python.write_text("") + python.chmod(0o755) + monkeypatch.chdir(current) + + assert which_python(directory=os.path.join("..", "sibling")) == sys.executable + + +def test_project_environment_lookup_includes_ancestors_outside_content_project(tmp_path, monkeypatch): + for name in ("VIRTUAL_ENV", "CONDA_PREFIX"): + monkeypatch.delenv(name, raising=False) + content = tmp_path / "project" / "content" + content.mkdir(parents=True) + (tmp_path / "project" / "pyproject.toml").write_text("[project]\nname = 'example'\n") + python = tmp_path / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + python.parent.mkdir(parents=True) + python.write_text("") + python.chmod(0o755) + + assert which_python(directory=str(content)) == str(python) + + +def test_inspect_environment_without_bundled_uv(monkeypatch): + monkeypatch.setattr(rsconnect.environment.uv, "find_uv_bin", missing_bundled_uv) + environment = Environment._inspect_environment(sys.executable, get_dir("pip1")) + assert environment.source == "file" + + +def test_inspect_environment_reports_invalid_pyproject(tmp_path): + (tmp_path / "pyproject.toml").write_text("[project\n") + with pytest.raises(RSConnectException, match="Error reading pyproject.toml"): + Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file="pyproject.toml") + + +@pytest.mark.parametrize("output", [b"banner\n", b""]) +def test_inspection_json_errors_name_interpreter(tmp_path, output): + with pytest.raises(RSConnectException, match="Error parsing environment JSON from Python /selected/python"): + Environment._inspect_environment("/selected/python", str(tmp_path), check_output=lambda *args, **kwargs: output) + + +def test_inspection_subprocess_errors_name_interpreter(tmp_path): + def fail(*args, **kwargs): + raise subprocess.CalledProcessError(1, args[0]) + + with pytest.raises(RSConnectException, match="Error inspecting environment with Python /selected/python"): + Environment._inspect_environment("/selected/python", str(tmp_path), check_output=fail) + + +def test_isolated_uv_environment_inspection_without_pip_or_rsconnect(tmp_path, monkeypatch, caplog): + from rsconnect.bundle import Manifest + + for name in ("VIRTUAL_ENV", "CONDA_PREFIX"): + monkeypatch.delenv(name, raising=False) + project = tmp_path / "project" + project.mkdir() + environment_dir = project / ".venv" + subprocess.run([uv.find_uv_bin(), "venv", "--python", sys.executable, str(environment_dir)], check=True) + python = environment_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + wheel = tmp_path / "example_project_dependency-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("example_project_dependency.py", "") + archive.writestr( + "example_project_dependency-1.2.3.dist-info/METADATA", + "Metadata-Version: 2.1\nName: example-project-dependency\nVersion: 1.2.3\n", + ) + archive.writestr( + "example_project_dependency-1.2.3.dist-info/WHEEL", + "Wheel-Version: 1.0\nGenerator: test\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + archive.writestr("example_project_dependency-1.2.3.dist-info/RECORD", "") + subprocess.run( + [ + uv.find_uv_bin(), + "pip", + "install", + "--offline", + "--no-index", + "--find-links", + str(tmp_path), + "--python", + str(python), + "example-project-dependency==1.2.3", + ], + check=True, + ) + foreign_path = tmp_path / "foreign" + foreign_path.mkdir() + (foreign_path / "pip.py").write_text("raise RuntimeError('foreign pip must not be imported')\n") + monkeypatch.setenv("PYTHONPATH", str(foreign_path)) + assert ( + subprocess.run( + [ + str(python), + "-I", + "-c", + "import importlib.util; assert not any(importlib.util.find_spec(x) for x in ('pip', 'rsconnect'))", + ], + capture_output=True, + ).returncode + == 0 + ) + + monkeypatch.setenv("PATH", "") + with caplog.at_level("INFO"): + generated = Environment.create_python_environment(str(project), requirements_file=None) + assert generated.python_interpreter == str(python) + assert f"Using Python: {python}" in caplog.text + assert ( + generated.python + == subprocess.check_output( + [str(python), "-I", "-c", "import sys; print('%d.%d.%d' % sys.version_info[:3])"], text=True + ).strip() + ) + assert generated.pip is None + assert "example-project-dependency==1.2.3" in generated.contents + assert "click==" not in generated.contents + assert Manifest(environment=generated).data["python"]["package_manager"]["version"] == "" + + (project / "requirements.txt").write_text("project-package==1.0\n") + monkeypatch.chdir(tmp_path) + existing = Environment.create_python_environment(str(project), python=os.path.relpath(python)) + assert existing.pip is None + assert existing.contents.splitlines() == ["project-package==1.0"] + uv_environment = Environment.create_python_environment( + str(project), python=str(python), package_manager=PackageInstaller.UV + ) + expected_uv_version = subprocess.check_output([uv.find_uv_bin(), "--version"], text=True).split()[1] + assert Manifest(environment=uv_environment).data["python"]["package_manager"]["version"] == expected_uv_version + + (project / "pyproject.toml").write_text('[project]\nname = "demo"\ndependencies = ["other-package>=2"]\n') + pyproject = Environment.create_python_environment(str(project), requirements_file="pyproject.toml") + assert "other-package>=2" in pyproject.contents + assert pyproject.pip is None + + +def test_inspection_when_rsconnect_is_importable_only_from_pythonpath(tmp_path): + # Mirrors `pip install --user` and PYTHONPATH installs: the interpreter running the CLI has + # no packages of its own, so the inspector must inherit the CLI's import path. + cli_env = tmp_path / "cli-env" + subprocess.run([uv.find_uv_bin(), "venv", "--python", sys.executable, str(cli_env)], check=True) + cli_python = cli_env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + project = tmp_path / "project" + project.mkdir() + (project / "requirements.txt").write_text("project-package==1.0\n") + + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(path for path in sys.path if path) + result = subprocess.run( + [ + str(cli_python), + "-c", + "import json, sys; from rsconnect.environment import Environment; " + "e = Environment.create_python_environment(sys.argv[1], python=sys.argv[2]); " + "print(json.dumps(e.contents))", + str(project), + str(cli_python), + ], + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.splitlines()[-1]).splitlines() == ["project-package==1.0"] + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_uv_freeze_filters_distributions_and_rejects_editable_file_urls(tmp_path, monkeypatch, fallback): + uv_path = uv.find_uv_bin() + env_dir = tmp_path / "env" + subprocess.run([uv_path, "venv", "--python", sys.executable, str(env_dir)], check=True) + python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + site = Path( + subprocess.check_output( + [str(python), "-I", "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], text=True + ).strip() + ) + for name in ("rsconnect-python", "rsconnect-utils", "pip", "setuptools", "wheel", "distribute"): + dist = site / f"{name.replace('-', '_')}-1.0.dist-info" + dist.mkdir() + (dist / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") + if fallback: + monkeypatch.setenv("PATH", os.path.dirname(uv_path)) + monkeypatch.setattr(rsconnect.environment.uv, "find_uv_bin", missing_bundled_uv) + else: + monkeypatch.setenv("PATH", "") + + environment = Environment._inspect_environment(str(python), str(tmp_path), requirements_file=None) + assert "rsconnect-utils==1.0" in environment.contents.splitlines() + assert "rsconnect-python==1.0" not in environment.contents.splitlines() + assert "pip==1.0" not in environment.contents.splitlines() + for name in ("setuptools", "wheel", "distribute"): + assert (f"{name}==1.0" in environment.contents.splitlines()) == (sys.version_info >= (3, 12)) + + local = tmp_path / "editable project" + local.mkdir() + dist = site / "editable_project-1.0.dist-info" + dist.mkdir() + (dist / "METADATA").write_text("Metadata-Version: 2.1\nName: editable-project\nVersion: 1.0\n") + (dist / "direct_url.json").write_text(json.dumps({"url": local.as_uri(), "dir_info": {"editable": True}})) + with pytest.raises(RSConnectException, match="Editable dependencies outside the deployment directory"): + Environment._inspect_environment(str(python), str(tmp_path), requirements_file=None) + + +def test_filter_uv_editable_rsconnect_file_url(): + line = f"-e {Path(rsconnect_project_dir).as_uri()}" + assert filter_pip_freeze_output(line) == "" + + +def test_uv_freeze_does_not_create_relative_cache_in_inspector_directory(tmp_path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + env_dir = project / ".venv" + subprocess.run([uv.find_uv_bin(), "venv", "--python", sys.executable, str(env_dir)], check=True) + python = env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + inspector_dir = tmp_path / "inspector" + inspector_dir.mkdir() + inspector = inspector_dir / "inspect_environment.py" + shutil.copyfile(rsconnect.environment.inspect_environment.__file__, inspector) + monkeypatch.setattr(rsconnect.environment.inspect_environment, "__file__", str(inspector)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("UV_CACHE_DIR", ".cache/uv") + + result = Environment.create_python_environment(str(project), requirements_file=None, python=str(python)) + + assert result.source == "pip_freeze" + assert not (inspector_dir / ".cache").exists() + + +@pytest.mark.skipif(os.name != "posix", reason="Requires a POSIX user site directory") +def test_isolated_inspection_finds_pip_in_user_site(tmp_path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + env_dir = tmp_path / "environment" + subprocess.run( + [uv.find_uv_bin(), "venv", "--system-site-packages", "--python", sys.executable, str(env_dir)], check=True + ) + python = env_dir / "bin/python" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("PYTHONUSERBASE", raising=False) + user_site = Path( + subprocess.check_output( + [str(python), "-E", "-c", "import site; print(site.getusersitepackages())"], text=True + ).strip() + ) + pip = user_site / "pip" + pip.mkdir(parents=True) + (pip / "__init__.py").write_text("") + (pip / "__main__.py").write_text( + "import sys\nprint('pip 50.0' if '--version' in sys.argv else 'user-project-dependency==1.2.3')\n" + ) + + result = Environment.create_python_environment(str(project), requirements_file=None, python=str(python)) + + assert result.pip == "50.0" + assert "user-project-dependency==1.2.3" in result.contents + + class TestPythonVersionRequirements: def test_malformed_pyproject(self, tmp_path): (tmp_path / "pyproject.toml").write_text('[project\nname = "x"\n') @@ -640,7 +1020,7 @@ def test_get_python_env_info( expected_python, expected_environment, ): - def fake_which_python(python, env=os.environ): + def fake_which_python(python, directory): return expected_python def fake_inspect_environment( @@ -691,12 +1071,7 @@ def test_python_interpreter(self): with mock.patch.object(rsconnect.environment.logger, "warning") as mock_warning: result = Environment.create_python_environment(get_dir("pip1"), python=sys.executable) - assert mock_warning.call_count == 1 - mock_warning.assert_called_once_with( - "On modern Posit Connect versions, the --python option won't influence " - "the Python version used to deploy the application anymore. " - "Please use a .python-version file to force a specific interpreter version." - ) + mock_warning.assert_not_called() assert result.python == current_python_version diff --git a/tests/test_main.py b/tests/test_main.py index f66d7766..39fb5fae 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -49,6 +49,14 @@ def _load_json(data): return json.loads(data) +@pytest.mark.parametrize("command", [["deploy", "notebook"], ["deploy", "api"], ["write-manifest", "notebook"]]) +def test_python_option_help_does_not_require_rsconnect_in_target_environment(command): + result = CliRunner().invoke(cli, [*command, "--help"]) + assert result.exit_code == 0 + assert "--python" in result.output + assert "must have the rsconnect" not in result.output + + class TestMain: # Every env var ``metadata.config_dirname()`` consults. ``HOME`` alone only # isolates macOS: Linux prefers ``XDG_CONFIG_HOME`` and Windows prefers @@ -384,7 +392,10 @@ def test_deploy_static_notebook_render_failure(self, tmp_path, capfd, contents, stderr = capfd.readouterr().err assert result.exit_code == 1 assert diagnostic in re.sub(r"\x1b\[[0-9;]*m", "", stderr) - assert f"Error: Rendering {notebook} with nbconvert failed (see output above)" in result.output + if missing_nbconvert: + assert f"nbconvert is not installed in the selected Python environment ({interpreter})" in result.output + else: + assert f"Error: Rendering {notebook} with nbconvert failed (see output above)" in result.output assert "Traceback (most recent call last)" not in result.output assert "Internal error" not in result.output + stderr assert "subprocess.CalledProcessError" not in result.output + stderr diff --git a/tests/test_pip_compatibility.py b/tests/test_pip_compatibility.py index 6aa03a44..f548db80 100644 --- a/tests/test_pip_compatibility.py +++ b/tests/test_pip_compatibility.py @@ -2,20 +2,36 @@ import json import os +import shutil import subprocess +import sys from pathlib import Path import pytest -from packaging.version import Version +import uv @pytest.mark.skipif(not os.environ.get("RSCONNECT_COMPAT_PYTHON"), reason="No compatibility interpreter selected") def test_dependency_inspection_with_selected_pip(tmp_path: Path): - python = os.path.abspath(os.environ["RSCONNECT_COMPAT_PYTHON"]) + base_python = os.path.abspath(os.environ["RSCONNECT_COMPAT_PYTHON"]) repo = Path(__file__).resolve().parents[1] - site = tmp_path / "site" - site.mkdir() - for name in ("rsconnect", "rsconnect-python", "rsconnect-utils", "demo-url"): + env_dir = tmp_path / "env" + subprocess.run([base_python, "-m", "venv", "--without-pip", str(env_dir)], check=True) + python = str(env_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python")) + site = Path( + subprocess.check_output( + [python, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], text=True + ).strip() + ) + base_site = Path( + subprocess.check_output( + [base_python, "-c", "import os,pip; print(os.path.dirname(os.path.dirname(pip.__file__)))"], text=True + ).strip() + ) + shutil.copytree(base_site / "pip", site / "pip") + for metadata in base_site.glob("pip-*.dist-info"): + shutil.copytree(metadata, site / metadata.name) + for name in ("rsconnect", "rsconnect-python", "rsconnect-utils", "demo-url", "setuptools", "wheel", "distribute"): dist = site / f"{name.replace('-', '_')}-1.0.dist-info" dist.mkdir() (dist / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") @@ -25,12 +41,21 @@ def test_dependency_inspection_with_selected_pip(tmp_path: Path): ) env = os.environ.copy() - env["PYTHONPATH"] = os.pathsep.join((str(repo), str(site))) env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" def inspect(requirements_file): result = subprocess.run( - [python, "-m", "rsconnect.subprocesses.inspect_environment", "-r", requirements_file, str(tmp_path)], + [ + sys.executable, + str(repo / "rsconnect/subprocesses/inspect_environment.py"), + "--python", + python, + "--uv", + uv.find_uv_bin(), + "-r", + requirements_file, + str(tmp_path), + ], capture_output=True, text=True, check=True, @@ -39,27 +64,28 @@ def inspect(requirements_file): ) return json.loads(result.stdout) - version = subprocess.check_output([python, "-m", "pip", "--version"], text=True).split()[1] generated = inspect("none") - if Version(version) < Version("20.3"): - assert f"requires pip 20.3 or later (found {version})" in generated["error"] - assert "install --upgrade pip" in generated["error"] - else: - lines = generated["contents"].splitlines() - assert "rsconnect-utils==1.0" in lines - assert "demo-url @ https://example.com/rsconnect/demo.whl" in lines - assert "rsconnect==1.0" not in lines - assert "rsconnect-python==1.0" not in lines + assert generated["pip"] == subprocess.check_output([python, "-m", "pip", "--version"], text=True).split()[1] + lines = generated["contents"].splitlines() + assert "rsconnect-utils==1.0" in lines + assert "demo-url @ https://example.com/rsconnect/demo.whl" in lines + assert "rsconnect==1.0" not in lines + assert "rsconnect-python==1.0" not in lines + pip_lines = subprocess.check_output([python, "-m", "pip", "freeze"], text=True, env=env).splitlines() + for name in ("pip", "setuptools", "wheel", "distribute"): + assert any(line.startswith(f"{name}==") for line in lines) == any( + line.startswith(f"{name}==") for line in pip_lines + ) - local = tmp_path / "rsconnect-local" - metadata = local / "demo_local.egg-info" - metadata.mkdir(parents=True) - (metadata / "PKG-INFO").write_text("Metadata-Version: 2.1\nName: demo-local\nVersion: 1.0\n") - (site / "demo-local.egg-link").write_text(f"{local}\n.\n") - env["PYTHONPATH"] += os.pathsep + str(local) - rejected = inspect("none") - assert "Editable dependencies outside the deployment directory" in rejected["error"] - assert str(local.resolve()) in rejected["error"] + local = tmp_path / "rsconnect-local" + local.mkdir() + metadata = site / "demo_local-1.0.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text("Metadata-Version: 2.1\nName: demo-local\nVersion: 1.0\n") + (metadata / "direct_url.json").write_text(json.dumps({"url": local.as_uri(), "dir_info": {"editable": True}})) + rejected = inspect("none") + assert "Editable dependencies outside the deployment directory" in rejected["error"] + assert "rsconnect-local" in rejected["error"] (tmp_path / "requirements.txt").write_text( "rsconnect==1.0\nrsconnect-python==1.0\nrsconnect-utils==1.0\n" diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 2c337ee4..ce33e820 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -92,10 +92,40 @@ def save_deployment_record(app_file: Path, app_id: str) -> None: @pytest.fixture(autouse=True) def preflight_environment(monkeypatch): + for variable in ("VIRTUAL_ENV", "CONDA_PREFIX"): + monkeypatch.delenv(variable, raising=False) monkeypatch.setattr("rsconnect.preflight.platform.python_version", lambda: "3.12.2") monkeypatch.setattr("rsconnect.preflight.shutil.which", lambda _: "/usr/bin/quarto") +def make_interpreter(path: Path, version: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\necho {version}\n") + path.chmod(0o755) + return path + + +def test_local_python_comes_from_project_environment(tmp_path): + make_interpreter(tmp_path / ".venv" / "bin" / "python", "3.11.4") + executor, _, _ = make_executor(tmp_path, new=True) + + result = run_preflight(executor, str(tmp_path), fix=True) + + assert result["local_python"] == "3.11.4" + assert result["recommended_python"] == "3.11" + assert (tmp_path / ".python-version").read_text().strip() == "3.11" + + +def test_explicit_python_overrides_project_environment(tmp_path): + make_interpreter(tmp_path / ".venv" / "bin" / "python", "3.11.4") + explicit = make_interpreter(tmp_path / "other" / "python", "3.12.1") + executor, _, _ = make_executor(tmp_path, new=True) + + result = run_preflight(executor, str(tmp_path), python=str(explicit)) + + assert result["local_python"] == "3.12.1" + + def test_metadata_precedence_and_explicit_constraint_are_preserved(tmp_path): (tmp_path / ".python-version").write_text("3.10\n") (tmp_path / "pyproject.toml").write_text('[project]\nrequires-python = ">=3.12"\n') @@ -700,11 +730,24 @@ def test_unpinned_local_version_mismatch_is_unknown_until_pin_applied(tmp_path, result = run_preflight(executor, str(tmp_path)) assert result["status"] == "unknown" + assert result["local_python_publishable"] is False assert result["recommended_python"] == "3.12" assert result["changed_files"] == [] assert any("dependency compatibility has not been tested" in warning for warning in result["warnings"]) +def test_project_environment_with_missing_publishable_flags_is_unknown(tmp_path): + make_interpreter(tmp_path / ".venv" / "bin" / "python", "3.9.6") + executor, _, _ = make_executor(tmp_path, new=True, settings={"installations": [{"version": "3.12.8"}]}) + + result = run_preflight(executor, str(tmp_path)) + + assert result["local_python"] == "3.9.6" + assert result["status"] == "unknown" + assert result["local_python_publishable"] is None + assert any("publishable flag" in warning for warning in result["warnings"]) + + def test_existing_python_version_pin_warns_when_local_python_does_not_match(tmp_path): (tmp_path / ".python-version").write_text("3.11\n", encoding="utf-8") executor, _, _ = make_executor(