Require pip 20.3 for dependency inspection

docs/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- Generating requirements from an environment now requires pip 20.3 or later.
+  Older environments can upgrade pip or provide a requirements file.
+- Reports editable dependencies outside the deployment directory, which Posit Connect
+  cannot install, in both generated and user-provided requirements. Requirement
+  filtering no longer drops unrelated packages whose name or path contains `rsconnect`.
 - `rsconnect token` prints the OAuth access token stored by `rsconnect login` for a
   Posit Connect server, so other tools, such as the Connect SDKs, can make API
   requests with your login. `--refresh` gets a new token after the server rejects

docs/deploying.md

@@ -35,6 +35,13 @@ ensuring that you use the same Python that you use to run your Jupyter Notebook:
 /path/to/python -m pip list --format=freeze
 ```
 
+Generating requirements from an environment requires pip 20.3 or later in the
+selected Python environment. Pip 20.3 introduced named `pip freeze --exclude`
+options and supports Python 3.8, the oldest Python version supported by
+rsconnect-python. Upgrade pip in older environments or supply a requirements
+file. Newer Python versions may need a later pip release. This requirement
+applies to inspection, not the installer used by Posit Connect.
+
 #### Python Version
 
 When deploying Python content to Posit Connect,

pyproject.toml

@@ -10,7 +10,7 @@ version = "1.32.1.dev1"
 
 dependencies = [
     "typing-extensions>=4.8.0",
-    "pip>=10.0.0",
+    "pip>=20.3",
     "uv>=0.9.0",
     "semver>=2.0.0,<4.0.0",
     "pyjwt>=2.4.0; python_version < '3.9'",

rsconnect/subprocesses/inspect_environment.py

@@ -13,13 +13,20 @@
 import datetime
 import json
 import locale
+import ntpath
 import os
+import posixpath
 import tempfile
 import re
 import subprocess
 import sys
 from dataclasses import asdict, dataclass, replace
 from typing import Callable, List, Optional, cast
+from urllib.parse import parse_qs, urlsplit
+
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+from packaging.version import Version
 
 try:
     import tomllib
@@ -29,6 +36,8 @@
 
 version_re = re.compile(r"\d+\.\d+(\.\d+)?")
 exec_dir = os.path.dirname(sys.executable)
+rsconnect_project_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+excluded_distributions = {"rsconnect", "rsconnect-python"}
 
 
 @dataclass(frozen=True)
@@ -96,6 +105,7 @@ def detect_environment(dirname: str, requirements_file: Optional[str] = "require
             )
 
     if result is not None:
+        check_editables_are_bundleable(result["contents"] or "", dirname)
         result["python"] = get_python_version()
         result["pip"] = get_version("pip")
         result["locale"] = get_default_locale()
@@ -152,7 +162,7 @@ def output_file(dirname: str, filename: str, package_manager: str):
         with open(path, "r", newline="") as f:
             data = f.read()
 
-        data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line])
+        data = "\n".join(line for line in data.split("\n") if not is_rsconnect_requirement(line))
 
         return {
             "filename": filename,
@@ -165,15 +175,33 @@ def output_file(dirname: str, filename: str, package_manager: str):
 
 
 def pip_freeze():
-    """Inspect the environment using `pip freeze --disable-pip-version-check version`.
+    """Inspect the environment using `pip freeze`.
 
     Returns a dictionary containing the filename
     (always 'requirements.txt') and contents if successful,
     or a dictionary containing 'error' on failure.
     """
+    pip_version = get_version("pip")
+    if Version(pip_version) < Version("20.3"):
+        raise EnvironmentException(
+            f"Generating requirements requires pip 20.3 or later (found {pip_version}). "
+            f"Upgrade pip with '{sys.executable} -m pip install --upgrade pip', "
+            "or provide a requirements file."
+        )
+
     try:
         proc = subprocess.Popen(
-            [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"],
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "freeze",
+                "--disable-pip-version-check",
+                "--exclude",
+                "rsconnect-python",
+                "--exclude",
+                "rsconnect",
+            ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             universal_newlines=True,
@@ -309,12 +337,75 @@ def pyproject_dependencies(dirname: str, pyproject_filename: str):
 
 
 def filter_pip_freeze_output(pip_stdout: str):
-    # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]`
     return "\n".join(
-        [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))]
+        line
+        for line in pip_stdout.split("\n")
+        if not is_rsconnect_requirement(line)
+        and not line.startswith("[notice]")
+        and not line.startswith("# Editable install with no version control (")
     )
 
 
+def check_editables_are_bundleable(requirements: str, dirname: str):
+    unbundleable = [line.strip() for line in requirements.split("\n") if is_unbundleable_editable(line, dirname)]
+    if unbundleable:
+        raise EnvironmentException(
+            "Editable dependencies outside the deployment directory cannot be installed on Connect:\n"
+            + "\n".join(unbundleable)
+            + "\nProvide a requirements file that installs them from a package index, a VCS URL, "
+            "or a relative path inside the deployment directory."
+        )
+
+
+def is_rsconnect_requirement(line: str) -> bool:
+    target = editable_target(line)
+    if target is not None:
+        return is_rsconnect_editable(target)
+    try:
+        requirement = Requirement(line.strip())
+    except InvalidRequirement:
+        return False
+    return canonicalize_name(requirement.name) in excluded_distributions
+
+
+def editable_target(line: str) -> Optional[str]:
+    line = line.strip()
+    if line.startswith("--editable="):
+        return line[len("--editable=") :].strip()
+    for flag in ("-e", "--editable"):
+        if line.startswith(flag) and line[len(flag) : len(flag) + 1].isspace():
+            return line[len(flag) :].strip()
+    return None
+
+
+def is_rsconnect_editable(target: str) -> bool:
+    url = urlsplit(target)
+    # pip freeze names VCS editables with an #egg= fragment; local checkouts are only identifiable by path.
+    egg = parse_qs(url.fragment).get("egg")
+    if egg:
+        return canonicalize_name(egg[0]) in excluded_distributions
+    return is_path(target) and os.path.isdir(target) and os.path.samefile(target, rsconnect_project_dir)
+
+
+def is_unbundleable_editable(line: str, dirname: str) -> bool:
+    target = editable_target(line)
+    if not target:
+        return False
+    if not is_path(target):
+        scheme = urlsplit(target).scheme.lower()
+        return scheme == "file" or scheme.endswith("+file")
+    # Requirements may come from Windows or POSIX publishers; either kind of absolute path only exists locally.
+    if posixpath.isabs(target) or ntpath.isabs(target):
+        return True
+    bundle_dir = os.path.abspath(dirname)
+    return os.path.commonpath([bundle_dir, os.path.abspath(os.path.join(bundle_dir, target))]) != bundle_dir
+
+
+def is_path(target: str) -> bool:
+    # PEP 508 URLs use RFC 3986 schemes; a one-letter scheme is a Windows drive, so the target is a path.
+    return len(urlsplit(target).scheme) <= 1
+
+
 def strip_ref(line: str):
     # remove erroneous conda build paths that will break pip install
     # Keep a CRLF line ending. The caller splits on "\n", so the "\r" stays on the line.

tests/test_environment.py

@@ -14,7 +14,9 @@
 from rsconnect.subprocesses.inspect_environment import (
     EnvironmentException,
     detect_environment,
+    check_editables_are_bundleable,
     filter_pip_freeze_output,
+    rsconnect_project_dir,
     get_default_locale,
     get_python_version,
 )
@@ -75,7 +77,7 @@ def test_requirements_override(self):
             os.makedirs(os.path.join(project_dir, "alt"), exist_ok=True)
             custom_requirements = os.path.join(project_dir, "alt", "custom.txt")
             with open(custom_requirements, "w", newline="\n") as f:
-                f.write("foo==1.0\nbar>=2.0\nrsconnect==0.1\n")
+                f.write("foo==1.0\nbar>=2.0\nrsconnect-python==0.1\nrsconnect==0.1\n")
 
             result = Environment.create_python_environment(
                 project_dir, requirements_file=os.path.join("alt", "custom.txt")
@@ -133,6 +135,173 @@ def test_filter_pip_freeze_output(self):
         self.assertEqual(filtered, expected)
 
 
+def test_filter_generated_requirements():
+    raw = "\n".join(
+        [
+            "rsconnect-python==1.0",
+            "rsconnect==1.0",
+            "RSCONNECT @ https://example.com/rsconnect.whl",
+            "RSCONNECT_PYTHON @ https://example.com/rsconnect.whl",
+            "rsconnect.python>=1",
+            "rsconnect-python",
+            "  Rsconnect_-Python[extra] ; python_version >= '3.8'",
+            "-rsconnect-python",
+            "my-rsconnect-utils==0.1",
+            "rsconnect-utils==0.1",
+            "rsconnect-python-extra==1.0",
+            "demo @ file:///work/rsconnect-demo",
+            "# Editable install with no version control (demo==1.0)",
+            "-e git+https://example.com/repo.git#egg=remote",
+            "-e git+ssh://git@example.com/repo.git#egg=remote-ssh",
+            "-e bzr+lp:project#egg=remote-bzr",
+            "-e git+https:repo#egg=remote-git",
+            "-e hg+http://example.com/repo#egg=remote-hg",
+            "--editable=svn+svn://example.com/repo#egg=remote-svn",
+            "-e https://example.com/project.tar.gz#egg=remote-archive",
+            "# my rsconnect project",
+        ]
+    )
+
+    assert filter_pip_freeze_output(raw).splitlines() == [
+        "-rsconnect-python",
+        "my-rsconnect-utils==0.1",
+        "rsconnect-utils==0.1",
+        "rsconnect-python-extra==1.0",
+        "demo @ file:///work/rsconnect-demo",
+        "-e git+https://example.com/repo.git#egg=remote",
+        "-e git+ssh://git@example.com/repo.git#egg=remote-ssh",
+        "-e bzr+lp:project#egg=remote-bzr",
+        "-e git+https:repo#egg=remote-git",
+        "-e hg+http://example.com/repo#egg=remote-hg",
+        "--editable=svn+svn://example.com/repo#egg=remote-svn",
+        "-e https://example.com/project.tar.gz#egg=remote-archive",
+        "# my rsconnect project",
+    ]
+
+
+@pytest.mark.parametrize(
+    "requirement",
+    [
+        "-e /work/rsconnect-demo",
+        "-e ../outside-project",
+        "-e ./libs/../../outside-project",
+        "--editable=file:///work/project",
+        "-e git+file:///work/repo#egg=project",
+        "-e git+file:/work/repo#egg=project",
+        "-e C:\\work\\project",
+        "-e D:/work/project",
+        "-e FILE:///work/project",
+    ],
+)
+def test_check_editables_rejects_editables_outside_bundle(tmp_path: Path, requirement):
+    with pytest.raises(EnvironmentException) as error:
+        check_editables_are_bundleable(f"six==1.17.0\n{requirement}\n", str(tmp_path))
+
+    assert requirement in str(error.value)
+    assert "requirements file" in str(error.value)
+
+
+@pytest.mark.parametrize(
+    "requirement",
+    [
+        "-e ./libs/foo",
+        "-e libs/foo",
+        "--editable=.",
+        "-e git+https://example.com/repo.git#egg=remote",
+        "-e bzr+lp:project#egg=remote-bzr",
+        "six @ file:///work/six",
+    ],
+)
+def test_check_editables_allows_bundleable_requirements(tmp_path: Path, requirement):
+    check_editables_are_bundleable(f"{requirement}\r\n", str(tmp_path))
+
+
+def test_filter_generated_requirements_drops_editable_rsconnect():
+    raw = "\n".join(
+        [
+            "six==1.17.0",
+            "-e git+https://github.com/posit-dev/rsconnect-python.git@abc123#egg=rsconnect_python",
+            "-e git+https://example.com/rsconnect.git#egg=RSCONNECT",
+            "-e git+https://example.com/rsconnect-utils.git#egg=rsconnect-utils",
+            "# Editable install with no version control (rsconnect-python==1.0)",
+            f"-e {rsconnect_project_dir}",
+        ]
+    )
+
+    assert filter_pip_freeze_output(raw) == (
+        "six==1.17.0\n-e git+https://example.com/rsconnect-utils.git#egg=rsconnect-utils"
+    )
+
+
+def test_check_editables_reports_all_unbundleable_editables(tmp_path: Path):
+    with pytest.raises(EnvironmentException) as error:
+        check_editables_are_bundleable("-e /work/one\n-e /work/two", str(tmp_path))
+
+    assert "-e /work/one" in str(error.value)
+    assert "-e /work/two" in str(error.value)
+
+
+def test_inspect_environment_reports_local_editable(tmp_path: Path, monkeypatch):
+    pip_dir = tmp_path / "pip"
+    pip_dir.mkdir()
+    (pip_dir / "__init__.py").write_text("")
+    (pip_dir / "__main__.py").write_text(
+        "import sys\n"
+        "if '--version' in sys.argv:\n"
+        "    print('pip 20.3')\n"
+        "else:\n"
+        "    print('# Editable install with no version control (demo==1.0)')\n"
+        "    print('-e /work/demo')\n"
+    )
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])]))
+
+    with pytest.raises(
+        RSConnectException,
+        match="Editable dependencies outside the deployment directory cannot be installed on Connect",
+    ) as error:
+        Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None)
+
+    assert "-e /work/demo" in str(error.value)
+    assert "requirements file" in str(error.value)
+
+
+def test_inspect_environment_rejects_old_pip_for_generated_requirements(tmp_path: Path, monkeypatch):
+    pip_dir = tmp_path / "pip"
+    pip_dir.mkdir()
+    (pip_dir / "__init__.py").write_text("")
+    (pip_dir / "__main__.py").write_text("print('pip 20.2.4')\n")
+    monkeypatch.setenv("PYTHONPATH", os.pathsep.join([str(tmp_path), str(Path(__file__).resolve().parents[1])]))
+
+    with pytest.raises(RSConnectException, match=r"requires pip 20\.3 or later \(found 20\.2\.4\)") as error:
+        Environment._inspect_environment(sys.executable, str(tmp_path), requirements_file=None)
+
+    assert "install --upgrade pip" in str(error.value)
+    assert "requirements file" in str(error.value)
+
+
+def test_requirements_file_filters_only_rsconnect_distribution(tmp_path: Path):
+    (tmp_path / "requirements.txt").write_bytes(
+        b"RSCONNECT_PYTHON==1.0\r\nrsconnect==1.0\r\nmy-rsconnect-utils==0.1\r\n"
+        b"demo @ file:///work/rsconnect-demo\r\n# rsconnect note\r\n-e ./libs/local\r\n"
+    )
+
+    env = Environment.create_python_environment(str(tmp_path))
+
+    assert env.contents == "my-rsconnect-utils==0.1\r\ndemo\r\n# rsconnect note\r\n-e ./libs/local\r\n"
+
+
+def test_requirements_file_rejects_editable_outside_bundle(tmp_path: Path):
+    (tmp_path / "requirements.txt").write_text("six\n-e /Users/me/localpkg\n")
+
+    with pytest.raises(
+        RSConnectException,
+        match="Editable dependencies outside the deployment directory cannot be installed on Connect",
+    ) as error:
+        Environment.create_python_environment(str(tmp_path))
+
+    assert "-e /Users/me/localpkg" in str(error.value)
+
+
 def test_uv_lock_export(tmp_path):
     project_dir = tmp_path / "project"
     project_dir.mkdir()