From e9b459279a53d91db3538ff96248a266d1f6e015 Mon Sep 17 00:00:00 2001 From: Taylor Steinberg Date: Wed, 4 Nov 2026 14:12:13 -0500 Subject: [PATCH] fix: require pip 20.3 for dependency inspection --- docs/CHANGELOG.md | 5 + docs/deploying.md | 7 + pyproject.toml | 2 +- rsconnect/subprocesses/inspect_environment.py | 101 ++++++++++- tests/test_environment.py | 171 +++++++++++++++++- tests/test_pip_compatibility.py | 71 ++++++++ uv.lock | 2 +- 7 files changed, 351 insertions(+), 8 deletions(-) create mode 100644 tests/test_pip_compatibility.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ba110352..ec678ad0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Generating requirements from an environment now requires pip 20.3 or later. + Older environments can upgrade pip or provide a requirements file. +- Reports editable dependencies outside the deployment directory, which Posit Connect + cannot install, in both generated and user-provided requirements. Requirement + filtering no longer drops unrelated packages whose name or path contains `rsconnect`. - `rsconnect token` prints the OAuth access token stored by `rsconnect login` for a Posit Connect server, so other tools, such as the Connect SDKs, can make API requests with your login. `--refresh` gets a new token after the server rejects diff --git a/docs/deploying.md b/docs/deploying.md index 7e5291ad..d8bc88f3 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -35,6 +35,13 @@ ensuring that you use the same Python that you use to run your Jupyter Notebook: /path/to/python -m pip list --format=freeze ``` +Generating requirements from an environment requires pip 20.3 or later in the +selected Python environment. Pip 20.3 introduced named `pip freeze --exclude` +options and supports Python 3.8, the oldest Python version supported by +rsconnect-python. Upgrade pip in older environments or supply a requirements +file. Newer Python versions may need a later pip release. This requirement +applies to inspection, not the installer used by Posit Connect. + #### Python Version When deploying Python content to Posit Connect, diff --git a/pyproject.toml b/pyproject.toml index 9a3b9ac2..e304b537 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ version = "1.32.1.dev1" dependencies = [ "typing-extensions>=4.8.0", - "pip>=10.0.0", + "pip>=20.3", "uv>=0.9.0", "semver>=2.0.0,<4.0.0", "pyjwt>=2.4.0; python_version < '3.9'", diff --git a/rsconnect/subprocesses/inspect_environment.py b/rsconnect/subprocesses/inspect_environment.py index b24bf851..b21b3642 100644 --- a/rsconnect/subprocesses/inspect_environment.py +++ b/rsconnect/subprocesses/inspect_environment.py @@ -13,13 +13,20 @@ import datetime import json import locale +import ntpath import os +import posixpath import tempfile import re import subprocess import sys from dataclasses import asdict, dataclass, replace from typing import Callable, List, Optional, cast +from urllib.parse import parse_qs, urlsplit + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version try: import tomllib @@ -29,6 +36,8 @@ version_re = re.compile(r"\d+\.\d+(\.\d+)?") exec_dir = os.path.dirname(sys.executable) +rsconnect_project_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +excluded_distributions = {"rsconnect", "rsconnect-python"} @dataclass(frozen=True) @@ -96,6 +105,7 @@ def detect_environment(dirname: str, requirements_file: Optional[str] = "require ) if result is not None: + check_editables_are_bundleable(result["contents"] or "", dirname) result["python"] = get_python_version() result["pip"] = get_version("pip") result["locale"] = get_default_locale() @@ -152,7 +162,7 @@ def output_file(dirname: str, filename: str, package_manager: str): with open(path, "r", newline="") as f: data = f.read() - data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line]) + data = "\n".join(line for line in data.split("\n") if not is_rsconnect_requirement(line)) return { "filename": filename, @@ -165,15 +175,33 @@ def output_file(dirname: str, filename: str, package_manager: str): def pip_freeze(): - """Inspect the environment using `pip freeze --disable-pip-version-check version`. + """Inspect the environment using `pip freeze`. Returns a dictionary containing the filename (always 'requirements.txt') and contents if successful, or a dictionary containing 'error' on failure. """ + pip_version = get_version("pip") + if Version(pip_version) < Version("20.3"): + raise EnvironmentException( + f"Generating requirements requires pip 20.3 or later (found {pip_version}). " + f"Upgrade pip with '{sys.executable} -m pip install --upgrade pip', " + "or provide a requirements file." + ) + try: proc = subprocess.Popen( - [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"], + [ + sys.executable, + "-m", + "pip", + "freeze", + "--disable-pip-version-check", + "--exclude", + "rsconnect-python", + "--exclude", + "rsconnect", + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, @@ -309,12 +337,75 @@ def pyproject_dependencies(dirname: str, pyproject_filename: str): def filter_pip_freeze_output(pip_stdout: str): - # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]` return "\n".join( - [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))] + line + for line in pip_stdout.split("\n") + if not is_rsconnect_requirement(line) + and not line.startswith("[notice]") + and not line.startswith("# Editable install with no version control (") ) +def check_editables_are_bundleable(requirements: str, dirname: str): + unbundleable = [line.strip() for line in requirements.split("\n") if is_unbundleable_editable(line, dirname)] + if unbundleable: + raise EnvironmentException( + "Editable dependencies outside the deployment directory cannot be installed on Connect:\n" + + "\n".join(unbundleable) + + "\nProvide a requirements file that installs them from a package index, a VCS URL, " + "or a relative path inside the deployment directory." + ) + + +def is_rsconnect_requirement(line: str) -> bool: + target = editable_target(line) + if target is not None: + return is_rsconnect_editable(target) + try: + requirement = Requirement(line.strip()) + except InvalidRequirement: + return False + return canonicalize_name(requirement.name) in excluded_distributions + + +def editable_target(line: str) -> Optional[str]: + line = line.strip() + if line.startswith("--editable="): + return line[len("--editable=") :].strip() + for flag in ("-e", "--editable"): + if line.startswith(flag) and line[len(flag) : len(flag) + 1].isspace(): + return line[len(flag) :].strip() + return None + + +def is_rsconnect_editable(target: str) -> bool: + url = urlsplit(target) + # pip freeze names VCS editables with an #egg= fragment; local checkouts are only identifiable by path. + egg = parse_qs(url.fragment).get("egg") + if egg: + return canonicalize_name(egg[0]) in excluded_distributions + return is_path(target) and os.path.isdir(target) and os.path.samefile(target, rsconnect_project_dir) + + +def is_unbundleable_editable(line: str, dirname: str) -> bool: + target = editable_target(line) + if not target: + return False + if not is_path(target): + scheme = urlsplit(target).scheme.lower() + return scheme == "file" or scheme.endswith("+file") + # Requirements may come from Windows or POSIX publishers; either kind of absolute path only exists locally. + if posixpath.isabs(target) or ntpath.isabs(target): + return True + bundle_dir = os.path.abspath(dirname) + return os.path.commonpath([bundle_dir, os.path.abspath(os.path.join(bundle_dir, target))]) != bundle_dir + + +def is_path(target: str) -> bool: + # PEP 508 URLs use RFC 3986 schemes; a one-letter scheme is a Windows drive, so the target is a path. + return len(urlsplit(target).scheme) <= 1 + + def strip_ref(line: str): # remove erroneous conda build paths that will break pip install # Keep a CRLF line ending. The caller splits on "\n", so the "\r" stays on the line. diff --git a/tests/test_environment.py b/tests/test_environment.py index ff841b64..d1d03ec2 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -14,7 +14,9 @@ from rsconnect.subprocesses.inspect_environment import ( EnvironmentException, detect_environment, + check_editables_are_bundleable, filter_pip_freeze_output, + rsconnect_project_dir, get_default_locale, get_python_version, ) @@ -75,7 +77,7 @@ def test_requirements_override(self): os.makedirs(os.path.join(project_dir, "alt"), exist_ok=True) custom_requirements = os.path.join(project_dir, "alt", "custom.txt") with open(custom_requirements, "w", newline="\n") as f: - f.write("foo==1.0\nbar>=2.0\nrsconnect==0.1\n") + f.write("foo==1.0\nbar>=2.0\nrsconnect-python==0.1\nrsconnect==0.1\n") result = Environment.create_python_environment( project_dir, requirements_file=os.path.join("alt", "custom.txt") @@ -133,6 +135,173 @@ def test_filter_pip_freeze_output(self): self.assertEqual(filtered, expected) +def test_filter_generated_requirements(): + raw = "\n".join( + [ + "rsconnect-python==1.0", + "rsconnect==1.0", + "RSCONNECT @ https://example.com/rsconnect.whl", + "RSCONNECT_PYTHON @ https://example.com/rsconnect.whl", + "rsconnect.python>=1", + "rsconnect-python", + " Rsconnect_-Python[extra] ; python_version >= '3.8'", + "-rsconnect-python", + "my-rsconnect-utils==0.1", + "rsconnect-utils==0.1", + "rsconnect-python-extra==1.0", + "demo @ file:///work/rsconnect-demo", + "# Editable install with no version control (demo==1.0)", + "-e git+https://example.com/repo.git#egg=remote", + "-e git+ssh://git@example.com/repo.git#egg=remote-ssh", + "-e bzr+lp:project#egg=remote-bzr", + "-e git+https:repo#egg=remote-git", + "-e hg+http://example.com/repo#egg=remote-hg", + "--editable=svn+svn://example.com/repo#egg=remote-svn", + "-e https://example.com/project.tar.gz#egg=remote-archive", + "# my rsconnect project", + ] + ) + + assert filter_pip_freeze_output(raw).splitlines() == [ + "-rsconnect-python", + "my-rsconnect-utils==0.1", + "rsconnect-utils==0.1", + "rsconnect-python-extra==1.0", + "demo @ file:///work/rsconnect-demo", + "-e git+https://example.com/repo.git#egg=remote", + "-e git+ssh://git@example.com/repo.git#egg=remote-ssh", + "-e bzr+lp:project#egg=remote-bzr", + "-e git+https:repo#egg=remote-git", + "-e hg+http://example.com/repo#egg=remote-hg", + "--editable=svn+svn://example.com/repo#egg=remote-svn", + "-e https://example.com/project.tar.gz#egg=remote-archive", + "# my rsconnect project", + ] + + +@pytest.mark.parametrize( + "requirement", + [ + "-e /work/rsconnect-demo", + "-e ../outside-project", + "-e ./libs/../../outside-project", + "--editable=file:///work/project", + "-e git+file:///work/repo#egg=project", + "-e git+file:/work/repo#egg=project", + "-e C:\\work\\project", + "-e D:/work/project", + "-e FILE:///work/project", + ], +) +def test_check_editables_rejects_editables_outside_bundle(tmp_path: Path, requirement): + with pytest.raises(EnvironmentException) as error: + check_editables_are_bundleable(f"six==1.17.0\n{requirement}\n", str(tmp_path)) + + assert requirement in str(error.value) + assert "requirements file" in str(error.value) + + +@pytest.mark.parametrize( + "requirement", + [ + "-e ./libs/foo", + "-e libs/foo", + "--editable=.", + "-e git+https://example.com/repo.git#egg=remote", + "-e bzr+lp:project#egg=remote-bzr", + "six @ file:///work/six", + ], +) +def test_check_editables_allows_bundleable_requirements(tmp_path: Path, requirement): + check_editables_are_bundleable(f"{requirement}\r\n", str(tmp_path)) + + +def test_filter_generated_requirements_drops_editable_rsconnect(): + raw = "\n".join( + [ + "six==1.17.0", + "-e git+https://github.com/posit-dev/rsconnect-python.git@abc123#egg=rsconnect_python", + "-e git+https://example.com/rsconnect.git#egg=RSCONNECT", + "-e git+https://example.com/rsconnect-utils.git#egg=rsconnect-utils", + "# Editable install with no version control (rsconnect-python==1.0)", + f"-e {rsconnect_project_dir}", + ] + ) + + assert filter_pip_freeze_output(raw) == ( + "six==1.17.0\n-e git+https://example.com/rsconnect-utils.git#egg=rsconnect-utils" + ) + + +def test_check_editables_reports_all_unbundleable_editables(tmp_path: Path): + with pytest.raises(EnvironmentException) as error: + check_editables_are_bundleable("-e /work/one\n-e /work/two", str(tmp_path)) + + assert "-e /work/one" in str(error.value) + assert "-e /work/two" in str(error.value) + + +def test_inspect_environment_reports_local_editable(tmp_path: Path, monkeypatch): + pip_dir = tmp_path / "pip" + pip_dir.mkdir() + (pip_dir / "__init__.py").write_text("") + (pip_dir / "__main__.py").write_text( + "import sys\n" + "if '--version' in sys.argv:\n" + " print('pip 20.3')\n" + "else:\n" + " print('# Editable install with no version control (demo==1.0)')\n" + " print('-e /work/demo')\n" + ) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])])) + + with pytest.raises( + RSConnectException, + match="Editable dependencies outside the deployment directory cannot be installed on Connect", + ) as error: + Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None) + + assert "-e /work/demo" in str(error.value) + assert "requirements file" in str(error.value) + + +def test_inspect_environment_rejects_old_pip_for_generated_requirements(tmp_path: Path, monkeypatch): + pip_dir = tmp_path / "pip" + pip_dir.mkdir() + (pip_dir / "__init__.py").write_text("") + (pip_dir / "__main__.py").write_text("print('pip 20.2.4')\n") + monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])])) + + with pytest.raises(RSConnectException, match=r"requires pip 20\.3 or later \(found 20\.2\.4\)") as error: + Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None) + + assert "install --upgrade pip" in str(error.value) + assert "requirements file" in str(error.value) + + +def test_requirements_file_filters_only_rsconnect_distribution(tmp_path: Path): + (tmp_path / "requirements.txt").write_bytes( + b"RSCONNECT_PYTHON==1.0\r\nrsconnect==1.0\r\nmy-rsconnect-utils==0.1\r\n" + b"demo @ file:///work/rsconnect-demo\r\n# rsconnect note\r\n-e ./libs/local\r\n" + ) + + env = Environment.create_python_environment(str(tmp_path)) + + assert env.contents == "my-rsconnect-utils==0.1\r\ndemo\r\n# rsconnect note\r\n-e ./libs/local\r\n" + + +def test_requirements_file_rejects_editable_outside_bundle(tmp_path: Path): + (tmp_path / "requirements.txt").write_text("six\n-e /Users/me/localpkg\n") + + with pytest.raises( + RSConnectException, + match="Editable dependencies outside the deployment directory cannot be installed on Connect", + ) as error: + Environment.create_python_environment(str(tmp_path)) + + assert "-e /Users/me/localpkg" in str(error.value) + + def test_uv_lock_export(tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() diff --git a/tests/test_pip_compatibility.py b/tests/test_pip_compatibility.py new file mode 100644 index 00000000..6aa03a44 --- /dev/null +++ b/tests/test_pip_compatibility.py @@ -0,0 +1,71 @@ +"""Opt-in integration checks against an isolated Python with pip installed.""" + +import json +import os +import subprocess +from pathlib import Path + +import pytest +from packaging.version import Version + + +@pytest.mark.skipif(not os.environ.get("RSCONNECT_COMPAT_PYTHON"), reason="No compatibility interpreter selected") +def test_dependency_inspection_with_selected_pip(tmp_path: Path): + python = os.path.abspath(os.environ["RSCONNECT_COMPAT_PYTHON"]) + repo = Path(__file__).resolve().parents[1] + site = tmp_path / "site" + site.mkdir() + for name in ("rsconnect", "rsconnect-python", "rsconnect-utils", "demo-url"): + dist = site / f"{name.replace('-', '_')}-1.0.dist-info" + dist.mkdir() + (dist / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") + if name == "demo-url": + (dist / "direct_url.json").write_text( + json.dumps({"url": "https://example.com/rsconnect/demo.whl", "archive_info": {}}) + ) + + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join((str(repo), str(site))) + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + def inspect(requirements_file): + result = subprocess.run( + [python, "-m", "rsconnect.subprocesses.inspect_environment", "-r", requirements_file, str(tmp_path)], + capture_output=True, + text=True, + check=True, + cwd=tmp_path, + env=env, + ) + return json.loads(result.stdout) + + version = subprocess.check_output([python, "-m", "pip", "--version"], text=True).split()[1] + generated = inspect("none") + if Version(version) < Version("20.3"): + assert f"requires pip 20.3 or later (found {version})" in generated["error"] + assert "install --upgrade pip" in generated["error"] + else: + lines = generated["contents"].splitlines() + assert "rsconnect-utils==1.0" in lines + assert "demo-url @ https://example.com/rsconnect/demo.whl" in lines + assert "rsconnect==1.0" not in lines + assert "rsconnect-python==1.0" not in lines + + local = tmp_path / "rsconnect-local" + metadata = local / "demo_local.egg-info" + metadata.mkdir(parents=True) + (metadata / "PKG-INFO").write_text("Metadata-Version: 2.1\nName: demo-local\nVersion: 1.0\n") + (site / "demo-local.egg-link").write_text(f"{local}\n.\n") + env["PYTHONPATH"] += os.pathsep + str(local) + rejected = inspect("none") + assert "Editable dependencies outside the deployment directory" in rejected["error"] + assert str(local.resolve()) in rejected["error"] + + (tmp_path / "requirements.txt").write_text( + "rsconnect==1.0\nrsconnect-python==1.0\nrsconnect-utils==1.0\n" + "demo-url @ https://example.com/rsconnect/demo.whl\n-e ./rsconnect-local\n" + ) + requirements = inspect("requirements.txt") + assert requirements["contents"] == ( + "rsconnect-utils==1.0\ndemo-url @ https://example.com/rsconnect/demo.whl\n-e ./rsconnect-local\n" + ) diff --git a/uv.lock b/uv.lock index 109da124..69fad309 100644 --- a/uv.lock +++ b/uv.lock @@ -5532,7 +5532,7 @@ requires-dist = [ { name = "click", specifier = ">=8.0.0" }, { name = "keyring", marker = "extra == 'keyring'", specifier = ">=23.0.0" }, { name = "packaging", specifier = ">=20.0" }, - { name = "pip", specifier = ">=10.0.0" }, + { name = "pip", specifier = ">=20.3" }, { name = "pyjwt", marker = "python_full_version < '3.9'", specifier = ">=2.4.0" }, { name = "pyjwt", marker = "python_full_version >= '3.9'", specifier = ">=2.15.0" }, { name = "semver", specifier = ">=2.0.0,<4.0.0" },