From 377672788290417d9e9258b5323371f866ade8a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Fri, 23 Oct 2026 14:02:54 -0400 Subject: [PATCH 01/10] Use importlib for __import__ --- src/core/IronPython/Modules/Builtin.cs | 72 +++++--- src/core/IronPython/Runtime/Importer.cs | 169 +++++++++++++++++- src/core/IronPython/Runtime/PythonContext.cs | 7 + src/core/IronPython/Runtime/PythonModule.cs | 2 +- tests/IronPython.Tests/Stress/Engine.cs | 1 + .../import_extant/replaced_by_builtin.py | 3 + .../suite/import_extant/replaced_by_class.py | 5 + .../replaced_by_namespace/__init__.py | 4 + .../replaced_by_namespace/sub.py | 1 + .../import_extant/replaced_sub/__init__.py | 0 .../import_extant/replaced_sub/a/__init__.py | 0 tests/suite/import_extant/replaced_sub/a/b.py | 3 + tests/suite/modules/io_related/test_codecs.py | 9 +- tests/suite/test_doc.py | 4 + tests/suite/test_imp.py | 141 ++++++++++++--- tests/suite/test_importext.py | 49 +++++ tests/suite/test_namebinding.py | 2 + tests/suite/test_regressions.py | 3 + 18 files changed, 409 insertions(+), 66 deletions(-) create mode 100644 tests/suite/import_extant/replaced_by_builtin.py create mode 100644 tests/suite/import_extant/replaced_by_class.py create mode 100644 tests/suite/import_extant/replaced_by_namespace/__init__.py create mode 100644 tests/suite/import_extant/replaced_by_namespace/sub.py create mode 100644 tests/suite/import_extant/replaced_sub/__init__.py create mode 100644 tests/suite/import_extant/replaced_sub/a/__init__.py create mode 100644 tests/suite/import_extant/replaced_sub/a/b.py diff --git a/src/core/IronPython/Modules/Builtin.cs b/src/core/IronPython/Modules/Builtin.cs index 13f306951..f258a3de7 100644 --- a/src/core/IronPython/Modules/Builtin.cs +++ b/src/core/IronPython/Modules/Builtin.cs @@ -49,15 +49,56 @@ public static partial class Builtin { [Documentation("__import__(name, globals, locals, fromlist, level) -> module\n\nImport a module.")] [LightThrowing] public static object __import__(CodeContext/*!*/ context, [NotNone] string name, object? globals = null, object? locals = null, object? fromlist = null, int level = 0) { - if (fromlist is string || fromlist is Extensible) { - fromlist = new List { fromlist }; + PythonContext pc = context.LanguageContext; + PythonModule? bootstrap = pc.ImportLibBootstrap; + + if (bootstrap is not null) { + var hasFromList = PythonOps.IsTrue(fromlist); + + object module = Importer.ImportModule(context, globals, name, hasFromList, level); + if (module is null) return ModuleNotFound(context, name); + + if (hasFromList && HasPath(module)) { + return PythonOps.Invoke(context, bootstrap, "_handle_fromlist", module, fromlist, PythonOps.GetBoundAttr(context, bootstrap, "_gcd_import"))!; + } + return module; + } else { + if (fromlist is string || fromlist is Extensible) { + fromlist = new List { fromlist }; + } + IList? from = fromlist as IList; + var hasFromList = from != null && from.Count > 0; + + object module = Importer.ImportModule(context, globals, name, hasFromList, level); + if (module is null) return ModuleNotFound(context, name); + + if (hasFromList && from is not null && module is PythonModule mod && mod.__dict__._storage.TryGetPath(out _)) { + string strAttrName; + for (int i = 0; i < from.Count; i++) { + object? attrName = from[i]; + + if (pc.TryConvertToString(attrName, out strAttrName) && + !string.IsNullOrEmpty(strAttrName) && + strAttrName != "*") { + try { + Importer.ImportFrom(context, mod, strAttrName); + } catch (ImportException) { + continue; + } + } + } + } + + return module; } - IList? from = fromlist as IList; - PythonContext pc = context.LanguageContext; + bool HasPath(object module) { + // the module type has no __path__ or __getattr__, so only the dict matters (and a miss through HasAttr is slow) + if (module is PythonModule mod && mod.GetType() == typeof(PythonModule)) return mod.__dict__._storage.TryGetPath(out _); + return PythonOps.HasAttr(context, module, "__path__"); + } - object ret = Importer.ImportModule(context, globals, name, from != null && from.Count > 0, level); - if (ret == null) { + static object ModuleNotFound(CodeContext context, string name) { #if PYTHON_36_OR_GREATER // ModuleNotFoundError since Python 3.6 var err = PythonOps.ModuleNotFoundError("No module named {0}", PythonOps.Repr(context, name)); #else @@ -66,25 +107,6 @@ public static object __import__(CodeContext/*!*/ context, [NotNone] string name, ((PythonExceptions._ImportError)err.GetPythonException()!).name = name; return LightExceptions.Throw(err); } - - if (ret is PythonModule mod && from != null) { - string strAttrName; - for (int i = 0; i < from.Count; i++) { - object? attrName = from[i]; - - if (pc.TryConvertToString(attrName, out strAttrName) && - !string.IsNullOrEmpty(strAttrName) && - strAttrName != "*") { - try { - Importer.ImportFrom(context, mod, strAttrName); - } catch (ImportException) { - continue; - } - } - } - } - - return ret; } [Documentation("abs(number) -> number\n\nReturn the absolute value of the argument.")] diff --git a/src/core/IronPython/Runtime/Importer.cs b/src/core/IronPython/Runtime/Importer.cs index 8a7984808..bed7035a8 100644 --- a/src/core/IronPython/Runtime/Importer.cs +++ b/src/core/IronPython/Runtime/Importer.cs @@ -6,6 +6,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Reflection; using System.Text; @@ -71,6 +72,7 @@ internal static object ImportLightThrow(CodeContext/*!*/ context, string fullNam /// result. /// public static object ImportFrom(CodeContext/*!*/ context, object from, string name) { + PythonContext pc = context.LanguageContext; if (from is PythonModule scope) { object ret; if (scope.GetType() == typeof(PythonModule)) { @@ -84,7 +86,7 @@ public static object ImportFrom(CodeContext/*!*/ context, object from, string na } } - if (scope.__dict__._storage.TryGetPath(out object path)) { + if (pc.ImportLibBootstrap is null && scope.__dict__._storage.TryGetPath(out object path)) { if (path is PythonList listPath) { return ImportNestedModule(context, scope, new ArraySegment(new[] { name }), listPath, scope.GetName()); } @@ -109,6 +111,15 @@ public static object ImportFrom(CodeContext/*!*/ context, object from, string na } } + if (pc.ImportLibBootstrap is not null) { + // like CPython 3.6, __import__ already imported submodules, but they may only be in sys.modules (e.g. circular imports) + if (PythonOps.TryGetBoundAttr(context, from, "__name__", out object pkgName) && pkgName is string strPkgName + && pc.SystemStateModules.TryGetValue(strPkgName + "." + name, out object module)) { + return module; + } + throw PythonOps.ImportError("cannot import name {0}", PythonOps.Repr(context, name)); + } + throw PythonOps.ImportError("Cannot import name {0}", name); } @@ -141,13 +152,17 @@ private static object ImportModuleFrom(CodeContext/*!*/ context, object from, Ar /// /// Called by the __builtin__.__import__ functions (general importing) and ScriptEngine (for site.py) /// - /// level indiciates whether to perform absolute or relative imports. + /// level indicates whether to perform absolute or relative imports. /// 0 indicates only absolute imports should be performed /// Positive numbers indicate the # of parent directories to search relative to the calling module /// public static object ImportModule(CodeContext/*!*/ context, object globals, string/*!*/ modName, bool bottom, int level) { if (level < 0) throw PythonOps.ValueError("level must be >= 0"); + if (context.LanguageContext.ImportLibBootstrap is PythonModule bootstrap) { + return ImportModuleWithImportLib(context, bootstrap, globals, modName, bottom, level); + } + if (modName.Contains(Path.DirectorySeparatorChar)) { throw PythonOps.ImportError("Import by filename is not supported."); } @@ -222,7 +237,6 @@ public static object ImportModule(CodeContext/*!*/ context, object globals, stri context.ShowCls = true; } } - } } @@ -278,6 +292,147 @@ public static object ImportModule(CodeContext/*!*/ context, object globals, stri return bottom ? next : newmod; } +#nullable enable + + /// + /// Imports a module using importlib, like importlib's __import__ but with a fast path for + /// imports of modules that are already in sys.modules. + /// + private static object? ImportModuleWithImportLib(CodeContext context, PythonModule bootstrap, object? globals, string modName, bool bottom, int level) { + PythonContext pc = context.LanguageContext; + var modules = pc.SystemStateModules; + + // TODO: strip the importlib._bootstrap frames from the traceback of exceptions raised during import (like CPython's remove_importlib_frames) + string? absName = modName; + if (level != 0) { + Debug.Assert(level > 0); + if (globals is not PythonDictionary dict || !TryCalcPackage(context, dict, out object? package)) { + package = PythonOps.Invoke(context, bootstrap, "_calc___package__", globals ?? new PythonDictionary()); + } + // resolve the name like CPython 3.6, so that the 3.4 bootstrap's _sanity_check doesn't require the package to be loaded + if (package is not string strPackage) throw PythonOps.TypeError("package must be a string"); +#if !PYTHON_36_OR_GREATER + // force a SystemError like CPython 3.4 + if (!modules.ContainsKey(strPackage)) { + throw PythonOps.SystemError("Parent module {0} not loaded, cannot perform relative import", PythonOps.Repr(context, strPackage)); + } +#endif + if (strPackage.Length == 0) throw PythonOps.ImportError("attempted relative import with no known parent package"); + if (!TryResolveName(modName, strPackage, level, out absName)) { + throw PythonOps.ValueError("attempted relative import beyond top-level package"); + } + } + + if (modules.TryGetValue(absName, out object? module) && module is not null) { + WaitForInitializing(module, absName); + } else { + // anything unusual goes through _gcd_import for its errors + try { + module = PythonOps.Invoke(context, bootstrap, "_gcd_import", absName); + } catch (ImportException e) when (level == 0 && !modName.Contains('.') + && e.GetPythonException() is PythonExceptions._ImportError { name: string errName } && errName == modName + && PythonOps.ScopeTryGetMember(context, pc.DomainManager.Globals, modName, out _)) { + // IronPython: fall back on runtime globals (e.g. COM type libraries), unless the ImportError came from a module that was found + module = ImportReflected(context, modName); + } + } + + object? top = module; + if (level == 0) { + int firstDot = modName.IndexOf('.'); + if (firstDot == -1) { + // IronPython: refresh .NET namespaces and types, clr.AddReference could have invalidated them + if (IsReflected(module)) { + module = top = ImportReflected(context, modName, allowSourceFile: false) ?? module; + } + } else if (!bottom) { + // import the top-level package like CPython, it may have been removed from sys.modules + top = ImportModuleWithImportLib(context, bootstrap, globals, modName.Substring(0, firstDot), bottom: true, level: 0); + } + // IronPython: importing clr or a .NET namespace or type makes .NET members visible on builtin types in this module (e.g. "abc".Trim()) + if (top is MemberTracker || top is PythonType { IsSystemType: true } || top == pc.ClrModule) { + context.ShowCls = true; + } + } else if (!bottom && modName.Length != 0) { + // slice the resolved name like CPython's C implementation, not __name__ like _bootstrap.__import__ + int firstDot = modName.IndexOf('.'); + int cutOff = firstDot == -1 ? 0 : modName.Length - firstDot; + string topName = absName.Substring(0, absName.Length - cutOff); + if (!modules.TryGetValue(topName, out top)) { + throw PythonOps.KeyError($"{PythonOps.Repr(context, topName)} not in sys.modules as expected"); + } + } + + return bottom ? module : top; + + void WaitForInitializing(object module, string name) { + // skip .NET namespaces and types, where the __spec__ lookup is a slow miss + if (module is PythonModule + && PythonOps.TryGetBoundAttr(context, module, "__spec__", out object? spec) + && PythonOps.TryGetBoundAttr(context, spec, "_initializing", out object? initializing) + && PythonOps.IsTrue(initializing)) { + // the module may still be initializing in another thread, wait for it like CPython does +#if !PYTHON_36_OR_GREATER + // 3.4's _lock_unlock_module expects the import lock to be held and releases it + PythonImport.acquire_lock(context); +#endif + PythonOps.Invoke(context, bootstrap, "_lock_unlock_module", name); + } + } + } + + /// + /// Same as importlib._bootstrap._calc___package__. Returns false where it would warn or raise, + /// so that the Python version can do it. + /// + private static bool TryCalcPackage(CodeContext context, PythonDictionary globals, out object? package) { +#if PYTHON_36_OR_GREATER + globals.TryGetValue("__package__", out package); + globals.TryGetValue("__spec__", out object? spec); + if (package is not null) { + // warns when __package__ != __spec__.parent + return spec is null || package is string strPackage + && PythonOps.TryGetBoundAttr(context, spec, "parent", out object? parent) + && parent is string strParent && strPackage == strParent; + } + if (spec is not null) { + return PythonOps.TryGetBoundAttr(context, spec, "parent", out package); + } + // warns when falling back on __name__ and __path__ + return false; +#else + globals.TryGetValue("__package__", out package); + if (package is null) { + if (!globals.TryGetValue("__name__", out package)) return false; + if (!globals.ContainsKey("__path__")) { + if (package is not string strName) return false; + package = strName.Substring(0, Math.Max(strName.LastIndexOf('.'), 0)); + } + } + return true; +#endif + } + + /// + /// Same as importlib._bootstrap._resolve_name. Returns false for relative imports beyond the top-level package. + /// + private static bool TryResolveName(string name, string package, int level, [NotNullWhen(true)] out string? absName) { + int end = package.Length; + for (int i = 1; i < level; i++) { + end = end == 0 ? -1 : package.LastIndexOf('.', end - 1); + if (end == -1) { + // attempted relative import beyond top-level package + absName = null; + return false; + } + } + string baseName = package.Substring(0, end); + absName = name.Length == 0 ? baseName : baseName + "." + name; + return true; + } + +#nullable restore + /// /// Interrogates the importing module for __name__ and __path__, which determine /// whether the imported module (whose name is 'name') is being imported as nested @@ -553,7 +708,7 @@ private static object ImportTopAbsolute(CodeContext/*!*/ context, string/*!*/ na if (IsReflected(ret)) { // Even though we found something in sys.modules, we need to check if a // clr.AddReference has invalidated it. So try ImportReflected again. - ret = ImportReflected(context, name) ?? ret; + ret = ImportReflected(context, name, allowSourceFile: false) ?? ret; } if (ret is NamespaceTracker rp || ret == context.LanguageContext.ClrModule) { @@ -667,11 +822,11 @@ internal static object ImportBuiltin(CodeContext/*!*/ context, string/*!*/ name) return pc.GetBuiltinModule(name); } - private static object ImportReflected(CodeContext/*!*/ context, string/*!*/ name) { + private static object ImportReflected(CodeContext/*!*/ context, string/*!*/ name, bool allowSourceFile = true) { object ret; PythonContext pc = context.LanguageContext; if (!PythonOps.ScopeTryGetMember(context, pc.DomainManager.Globals, name, out ret) && - (ret = pc.TopNamespace.TryGetPackageAny(name)) == null) { + (ret = pc.TopNamespace.TryGetPackageAny(name)) == null && allowSourceFile) { ret = TryImportSourceFile(pc, name); } @@ -770,7 +925,7 @@ internal static SourceUnit TryFindSourceFile(PythonContext/*!*/ context, string/ private static bool IsReflected(object module) { // corresponds to the list of types that can be returned by ImportReflected return module is MemberTracker - || module is PythonType + || module is PythonType { IsSystemType: true } || module is ReflectedEvent || module is ReflectedField || module is BuiltinFunction; diff --git a/src/core/IronPython/Runtime/PythonContext.cs b/src/core/IronPython/Runtime/PythonContext.cs index 29cbd56df..0a4ce2574 100644 --- a/src/core/IronPython/Runtime/PythonContext.cs +++ b/src/core/IronPython/Runtime/PythonContext.cs @@ -308,6 +308,7 @@ void BootstrapImportLib() { #endif var moduleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec"); (GetSystemStateValue("meta_path") as PythonList)?.append(new Importer.ClrImporter(moduleSpec)); + ImportLibBootstrap = _frozen_importlib; } catch { } PythonModule LoadModuleFromResource(string name, string resourceName) { @@ -1147,6 +1148,12 @@ internal PythonModule GetReloadableModule(PythonModule/*!*/ module) { #endregion + /// + /// The _frozen_importlib module, which drives imports once it has been bootstrapped. + /// When null (e.g. NoImportLib), the legacy importer is used instead. + /// + internal PythonModule ImportLibBootstrap { get; private set; } + private object copyRegModule; internal object GetCopyRegModule() { diff --git a/src/core/IronPython/Runtime/PythonModule.cs b/src/core/IronPython/Runtime/PythonModule.cs index 4cad4ca6f..b7395707f 100755 --- a/src/core/IronPython/Runtime/PythonModule.cs +++ b/src/core/IronPython/Runtime/PythonModule.cs @@ -145,7 +145,7 @@ public void __setattr__(CodeContext/*!*/ context, string name, object value) { switch (name) { case "__dict__": throw PythonOps.AttributeError("readonly attribute"); - case "__class__": throw PythonOps.TypeError("__class__ assignment: only for heap types"); + case "__class__": throw PythonOps.TypeError("__class__ assignment only supported for heap types or ModuleType subclasses"); } Debug.Assert(value != Uninitialized.Instance); diff --git a/tests/IronPython.Tests/Stress/Engine.cs b/tests/IronPython.Tests/Stress/Engine.cs index 003cb57ca..be10df258 100644 --- a/tests/IronPython.Tests/Stress/Engine.cs +++ b/tests/IronPython.Tests/Stress/Engine.cs @@ -47,6 +47,7 @@ private static long GetTotalMemory() { #endif public void ScenarioXGC() { Assert.AreEqual(1, 1); // prevents nunit from counting towards memory usage + _pe.Execute("import gc"); // the first import initializes importlib's import machinery long initialMemory = GetTotalMemory(); diff --git a/tests/suite/import_extant/replaced_by_builtin.py b/tests/suite/import_extant/replaced_by_builtin.py new file mode 100644 index 000000000..69897d564 --- /dev/null +++ b/tests/suite/import_extant/replaced_by_builtin.py @@ -0,0 +1,3 @@ +import sys + +sys.modules[__name__] = len diff --git a/tests/suite/import_extant/replaced_by_class.py b/tests/suite/import_extant/replaced_by_class.py new file mode 100644 index 000000000..6e2cf2d54 --- /dev/null +++ b/tests/suite/import_extant/replaced_by_class.py @@ -0,0 +1,5 @@ +import sys + +class ReplacedByClass: pass + +sys.modules[__name__] = ReplacedByClass diff --git a/tests/suite/import_extant/replaced_by_namespace/__init__.py b/tests/suite/import_extant/replaced_by_namespace/__init__.py new file mode 100644 index 000000000..8c7d88a0a --- /dev/null +++ b/tests/suite/import_extant/replaced_by_namespace/__init__.py @@ -0,0 +1,4 @@ +import sys +import types + +sys.modules[__name__] = types.SimpleNamespace(__name__=__name__, __path__=__path__, __spec__=__spec__) diff --git a/tests/suite/import_extant/replaced_by_namespace/sub.py b/tests/suite/import_extant/replaced_by_namespace/sub.py new file mode 100644 index 000000000..4beddd4e1 --- /dev/null +++ b/tests/suite/import_extant/replaced_by_namespace/sub.py @@ -0,0 +1 @@ +value = 'sub' diff --git a/tests/suite/import_extant/replaced_sub/__init__.py b/tests/suite/import_extant/replaced_sub/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/suite/import_extant/replaced_sub/a/__init__.py b/tests/suite/import_extant/replaced_sub/a/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/suite/import_extant/replaced_sub/a/b.py b/tests/suite/import_extant/replaced_sub/a/b.py new file mode 100644 index 000000000..6f0ac6c6d --- /dev/null +++ b/tests/suite/import_extant/replaced_sub/a/b.py @@ -0,0 +1,3 @@ +import sys + +sys.modules[__name__] = type("b", (), {}) diff --git a/tests/suite/modules/io_related/test_codecs.py b/tests/suite/modules/io_related/test_codecs.py index aaa7d58d2..85099bac0 100644 --- a/tests/suite/modules/io_related/test_codecs.py +++ b/tests/suite/modules/io_related/test_codecs.py @@ -14,6 +14,7 @@ import array import codecs +import importlib import os import re import shutil @@ -22,12 +23,9 @@ import sys import unittest -from iptest import IronPythonTestCase, is_cli, is_cpython, is_mono, is_netcoreapp, is_posix, is_linux, is_windows, run_test, skipUnlessIronPython +from iptest import IronPythonTestCase, is_cli, is_cpython, is_mono, is_netcoreapp, is_posix, is_windows, run_test, skipUnlessIronPython from iptest.misc_util import ip_supported_encodings -if is_cpython and is_linux: - import time - class CodecTest(IronPythonTestCase): def test_escape_decode(self): # escape_decode decodes bytes to bytes, but when given a string it encodes it first with UTF-8 @@ -1840,8 +1838,7 @@ def test_file_encodings(self): f.write("# coding: %s" % (coding)) - if is_cpython and is_linux: - time.sleep(0.01) + importlib.invalidate_caches() __import__(temp_mod_name) os.remove(os.path.join(self.temporary_dir, "tmp_encodings", temp_mod_name + ".py")) diff --git a/tests/suite/test_doc.py b/tests/suite/test_doc.py index afd4e9a3e..1ad21097a 100644 --- a/tests/suite/test_doc.py +++ b/tests/suite/test_doc.py @@ -4,6 +4,8 @@ "module doc" +import importlib + from iptest import IronPythonTestCase, is_cli, path_modifier, retryOnFailure, run_test, skipUnlessIronPython class DocTest(IronPythonTestCase): @@ -208,11 +210,13 @@ def test_module_doc_cp21360(self): try: with path_modifier('.'): self.write_to_file(temp_filename_empty, "") + importlib.invalidate_caches() import cp21360_empty self.assertEqual(cp21360_empty.__doc__, None) with path_modifier('.'): self.write_to_file(temp_filename, "x = 3.14") + importlib.invalidate_caches() import cp21360 self.assertEqual(cp21360.__doc__, None) self.assertEqual(cp21360.x, 3.14) diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index b2cafb280..c91b10300 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -4,6 +4,7 @@ import gc import imp +import importlib import os import sys import threading @@ -31,13 +32,7 @@ def __init__(self, s): def find_module(self, fullname, path=None): self.s.assertIsNone(path) if fullname == 'does_not_exist_throw': raise Exception('hello') - elif fullname == 'does_not_exist_abc': return meta_loader('abc') elif fullname == 'does_not_exist_loader_throw': return meta_loader(Exception('loader')) - elif fullname == 'does_not_exist_None': return meta_loader(None) - elif fullname == 'does_not_exist_X': - class X(object): - abc = 3 - return meta_loader(X) class ImpTest(IronPythonTestCase): def setUp(self): @@ -595,10 +590,45 @@ def f():exec("from import abc") get_builtins_dict()['__import__'] = old_import #TODO:@skip("multiple_execute") #http://ironpython.codeplex.com/WorkItem/View.aspx?WorkItemId=26829 + @unittest.skipIf(not is_cli and sys.version_info[:2] == (3, 6), "CPython 3.6 doesn't raise when run as a script") def test_import_relative_error(self): - with self.assertRaises(SystemError): + with self.assertRaises(ImportError if sys.version_info >= (3, 6) else SystemError): exec('from . import *') + def test_import_relative_loaded(self): + """relative imports of modules that are already in sys.modules""" + import testpkg1.mod1 + pkg, mod = sys.modules['testpkg1'], sys.modules['testpkg1.mod1'] + glb = {'__name__': 'testpkg1.x', '__package__': 'testpkg1'} + try: + self.assertIs(__import__('mod1', glb, None, ['x'], 1), mod) + self.assertIs(__import__('mod1', glb, None, None, 1), mod) + self.assertIs(__import__('', glb, None, ['mod1'], 1), pkg) + self.assertIs(__import__('mod1', {'__name__': 'testpkg1.mod1.x', '__package__': 'testpkg1.mod1'}, None, ['x'], 2), mod) + + sys.modules['testpkg1.mod1'] = None + self.assertRaises(ImportError, __import__, 'mod1', glb, None, ['x'], 1) + + # the submodule is imported again once it's gone + del sys.modules['testpkg1.mod1'] + self.assertIsNot(__import__('mod1', glb, None, ['x'], 1), mod) + + del sys.modules['testpkg1'] + if sys.version_info >= (3, 5): + self.assertIs(__import__('mod1', glb, None, ['x'], 1), sys.modules['testpkg1.mod1']) + + # the parent of the resolved name is imported, __package__ doesn't need to exist + sys.modules.pop('testpkg1', None) + del sys.modules['testpkg1.mod1'] + self.assertIs(__import__('mod1', {'__name__': 'testpkg1.missing.x', '__package__': 'testpkg1.missing'}, None, ['x'], 2), sys.modules['testpkg1.mod1']) + + self.assertRaises(ImportError, __import__, 'mod1', {'__name__': 'missing_pkg.x', '__package__': 'missing_pkg'}, None, ['x'], 1) + else: + self.assertRaises(SystemError, __import__, 'mod1', glb, None, ['x'], 1) + finally: + sys.modules.pop('testpkg1', None) + sys.modules.pop('testpkg1.mod1', None) + @unittest.skip('No access to CPython stdlib') def test_import_hooks_import_precence(self): """__import__ takes precedence over import hooks""" @@ -744,10 +774,6 @@ def f(): import does_not_exist_throw self.assertRaisesMessage(Exception, 'hello again', f) - if is_cli: - def f(): import does_not_exist_return_none - self.assertRaises(ImportError, f) - import does_not_exist_create self.assertEqual(does_not_exist_create.__file__, '') self.assertEqual(does_not_exist_create.fullname, 'does_not_exist_create') @@ -796,10 +822,7 @@ def hook(*args): raise ImportError('foo') # returning None def hook(*args): pass prepare(hook) - if is_cli: - f() - else: - self.assertRaises(ImportError, f) + self.assertRaises(ImportError, f) finally: sys.path_hooks = old_path_hooks @@ -807,20 +830,12 @@ def common_meta_import_tests(self): def f(): import does_not_exist_throw self.assertRaisesMessage(Exception, 'hello', f) - if is_cli: - import does_not_exist_abc - self.assertEqual(does_not_exist_abc, 'abc') - def f(): import does_not_exist_loader_throw self.assertRaisesMessage(Exception, 'loader', f) def f(): import does_not_exist_loader_None self.assertRaisesMessage(ImportError, "No module named 'does_not_exist_loader_None'", f) - if is_cli: - from does_not_exist_X import abc - self.assertEqual(abc, 3) - def test_path_hooks_importer_and_loader(self): path = list(sys.path) hooks = list(sys.path_hooks) @@ -874,7 +889,7 @@ def ok_loader(name): # dynamically resolve find_module to None with self.assertRaises(TypeError): import xyz - self.assertEqual(loaderInst.calls, ['find_module'] if is_cli else ['find_spec', 'find_module']) + self.assertEqual(loaderInst.calls, ['find_spec', 'find_module']) loaderInst.calls = [] # dynamically resolve find_module to a function, @@ -882,14 +897,14 @@ def ok_loader(name): finder = ok_finder with self.assertRaises(TypeError): import xyz - self.assertEqual(loaderInst.calls, ['find_module', ('xyz', None), 'load_module'] if is_cli else ['find_spec', 'find_module', ('xyz', None), 'get_filename', 'is_package', 'exec_module', 'load_module']) + self.assertEqual(loaderInst.calls, ['find_spec', 'find_module', ('xyz', None), 'get_filename', 'is_package', 'exec_module', 'load_module']) loaderInst.calls = [] loader = ok_loader import xyz self.assertEqual(xyz, 'abc') - self.assertEqual(loaderInst.calls, ['find_module', ('xyz', None), 'load_module', 'xyz'] if is_cli else ['find_spec', 'find_module', ('xyz', None), 'get_filename', 'is_package', 'exec_module', 'load_module', 'xyz']) + self.assertEqual(loaderInst.calls, ['find_spec', 'find_module', ('xyz', None), 'get_filename', 'is_package', 'exec_module', 'load_module', 'xyz']) finally: sys.meta_path.remove(loaderInst) @@ -943,6 +958,7 @@ def test_file_coding(self): import os with open(os.path.join(tmpdir, 'test_coding_mod.py'), 'wb+') as f: f.write(b"# coding: utf-8\nx = '\xc3\xa6ble'\n") + importlib.invalidate_caches() with path_modifier(tmpdir): import test_coding_mod self.assertEqual(test_coding_mod.x[0], '\xe6') @@ -953,6 +969,7 @@ def test_file_coding(self): with open(os.path.join(tmpdir, 'test_coding_2.py'), 'wb+') as f: f.write(b"\xef\xbb\xbf# -*- coding: utf-8 -*-\n") f.write(b"x = u'ABCDE'\n") + importlib.invalidate_caches() with path_modifier(tmpdir): import test_coding_2 self.assertEqual(test_coding_2.x, 'ABCDE') @@ -964,6 +981,7 @@ def test_file_coding(self): f.write(b"# -*- coding: utf-8 -*-\n") f.write(b"raise Exception()") f.close() + importlib.invalidate_caches() try: with path_modifier(tmpdir): import test_coding_3 @@ -1166,6 +1184,23 @@ def test_clr_importer_globals(self): del runtime_globals.System sys.modules['System'] = system + @skipUnlessIronPython() + def test_clr_importer_globals_import_error(self): + # an ImportError from a module that was found isn't replaced by the runtime's globals + import clr + runtime_globals = clr.GetCurrentRuntime().Globals + self.write_to_file(os.path.join(self._imptestdir, 'globals_import_error.py'), 'import globals_missing_dependency') + importlib.invalidate_caches() + runtime_globals.globals_import_error = object() + try: + with path_modifier(self._imptestdir): + with self.assertRaises(ImportError) as cm: + exec("import globals_import_error", {}) + self.assertEqual(cm.exception.name, 'globals_missing_dependency') + finally: + del runtime_globals.globals_import_error + sys.modules.pop('globals_import_error', None) + def test_get_frozen_object(self): # frozen objects not supported, this always fails self.assertRaises(ImportError, imp.get_frozen_object, 'foo') @@ -1326,7 +1361,8 @@ class M2(type(sys)): pass # __setattr__/__delattr__ no non-derived type m = type(sys)('foo') - self.assertRaisesMessage(TypeError, "__class__ assignment: only for heap types", m.__setattr__, '__class__', int) + msg = "__class__ assignment only supported for heap types or ModuleType subclasses" if is_cli or sys.version_info >= (3, 5) else "__class__ assignment: only for heap types" + self.assertRaisesMessage(TypeError, msg, m.__setattr__, '__class__', int) self.assertRaisesMessage(AttributeError, "readonly attribute", m.__setattr__, '__dict__', int) self.assertRaisesMessage(TypeError, "can't delete __class__ attribute", m.__delattr__, '__class__') self.assertRaisesMessage(AttributeError, "readonly attribute", m.__delattr__, '__dict__') @@ -1351,6 +1387,57 @@ def test_ximp_load_module(self): def test_import_string_from_list_cp26098(self): self.assertEqual(__import__('email.mime.application', globals(), locals(), 'MIMEApplication').__name__, 'email.mime.application') + def test_import_fromlist_iterable(self): + sys.modules.pop('testpkg1', None) + sys.modules.pop('testpkg1.mod1', None) + try: + # a string is iterated by character, so no submodule named 'mod1' is imported + m = __import__('testpkg1', fromlist='mod1') + self.assertEqual(m.__name__, 'testpkg1') + self.assertNotIn('testpkg1.mod1', sys.modules) + + # an empty fromlist returns the top-level package + self.assertEqual(__import__('testpkg1.mod1', fromlist='').__name__, 'testpkg1') + + # any iterable is accepted + m = __import__('testpkg1', fromlist={'mod1': None}.keys()) + self.assertIs(m.mod1, sys.modules['testpkg1.mod1']) + + self.assertRaises(TypeError, __import__, 'testpkg1', fromlist=[1]) + finally: + sys.modules.pop('testpkg1', None) + sys.modules.pop('testpkg1.mod1', None) + + def test_import_from_sys_modules(self): + sys.modules.pop('testpkg1', None) + mod = imp.new_module('testpkg1.cached') + sys.modules['testpkg1.cached'] = mod + try: + # IronPython behaves like CPython 3.6, the submodule is found in sys.modules (e.g. circular imports) + if is_cli or sys.version_info >= (3, 5): + g = {} + exec('from testpkg1 import cached', g) + self.assertIs(g['cached'], mod) + + with self.assertRaisesRegex(ImportError, "^cannot import name 'missing'$"): + exec('from testpkg1 import missing', {}) + finally: + sys.modules.pop('testpkg1', None) + sys.modules.pop('testpkg1.cached', None) + + def test_import_missing_parent(self): + mod = imp.new_module('missing_parent.child') + sys.modules['missing_parent.child'] = mod + try: + # the parent isn't needed when a fromlist is given + self.assertIs(__import__('missing_parent.child', fromlist=['x']), mod) + + # otherwise the parent is imported again + with self.assertRaises(ImportError): + __import__('missing_parent.child') + finally: + sys.modules.pop('missing_parent.child', None) + @skipUnlessIronPython() def test_new_builtin_modules(self): import clr diff --git a/tests/suite/test_importext.py b/tests/suite/test_importext.py index e23f153bb..d9cbf1c4a 100644 --- a/tests/suite/test_importext.py +++ b/tests/suite/test_importext.py @@ -41,5 +41,54 @@ def test_gh936(self): finally: sys.path.remove(extra_import_path) + def test_replaced_in_sys_modules(self): + # a module that replaced itself in sys.modules must not be executed again + extra_import_path = os.path.join(self.test_dir, "import_extant") + sys.path.insert(0, extra_import_path) + try: + import replaced_by_class + first = replaced_by_class + self.assertIsInstance(first, type) + import replaced_by_class + self.assertIs(replaced_by_class, first) + self.assertIs(sys.modules["replaced_by_class"], first) + + import replaced_by_builtin + self.assertIs(replaced_by_builtin, len) + import replaced_by_builtin + self.assertIs(replaced_by_builtin, len) + self.assertIs(sys.modules["replaced_by_builtin"], len) + finally: + sys.path.remove(extra_import_path) + sys.modules.pop("replaced_by_class", None) + sys.modules.pop("replaced_by_builtin", None) + + def test_fromlist_path_attribute(self): + # fromlist submodules are imported for anything with a __path__, not only modules + extra_import_path = os.path.join(self.test_dir, "import_extant") + sys.path.insert(0, extra_import_path) + try: + from replaced_by_namespace import sub + self.assertEqual(sub.value, "sub") + self.assertIs(sys.modules["replaced_by_namespace.sub"], sub) + finally: + sys.path.remove(extra_import_path) + sys.modules.pop("replaced_by_namespace", None) + sys.modules.pop("replaced_by_namespace.sub", None) + + def test_relative_replaced_submodule(self): + # the top-level module of a relative import comes from the resolved name, not the submodule's __name__ + extra_import_path = os.path.join(self.test_dir, "import_extant") + sys.path.insert(0, extra_import_path) + try: + import replaced_sub + m = __import__("a.b", {"__name__": "replaced_sub", "__package__": "replaced_sub"}, None, (), 1) + self.assertIs(m, sys.modules["replaced_sub.a"]) + self.assertIsInstance(sys.modules["replaced_sub.a.b"], type) + finally: + sys.path.remove(extra_import_path) + for name in ("replaced_sub", "replaced_sub.a", "replaced_sub.a.b"): + sys.modules.pop(name, None) + run_test(__name__) diff --git a/tests/suite/test_namebinding.py b/tests/suite/test_namebinding.py index bcba654b8..1f64eab4a 100644 --- a/tests/suite/test_namebinding.py +++ b/tests/suite/test_namebinding.py @@ -2,6 +2,7 @@ # The .NET Foundation licenses this file to you under the Apache 2.0 License. # See the LICENSE file in the project root for more information. +import importlib import sys import unittest @@ -1093,6 +1094,7 @@ def f(): fname = module_name + '.py' with open(fname, 'w+') as f: f.write('foo = 42') + importlib.invalidate_caches() try: with path_modifier('.'): foo = __import__(module_name).foo diff --git a/tests/suite/test_regressions.py b/tests/suite/test_regressions.py index 9c6868c35..0dcd906e3 100644 --- a/tests/suite/test_regressions.py +++ b/tests/suite/test_regressions.py @@ -14,6 +14,7 @@ def test_cp1234(): ... Work Item number. """ +import importlib import os import sys import unittest @@ -434,6 +435,7 @@ def test_module_alias_cp19656(self): try: self.write_to_file(stuff_mod, "Keys = 3") self.write_to_file(check_mod, "def check(module):\n return module.Keys") + importlib.invalidate_caches() stuff = __import__(stuff_name) check = __import__(check_name).check self.assertEqual(check(stuff), 3) @@ -643,6 +645,7 @@ def _get_x(self): return x cp20174_property = os.path.join(cp20174_path, "property.py") self.write_to_file(cp20174_property, "x=1") + importlib.invalidate_caches() import cp20174 self.assertEqual(cp20174.property.x, 1) From 48b7a8ef380c0541717e06ef615e72c3b0f5740c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 28 Oct 2026 17:07:16 -0400 Subject: [PATCH 02/10] Fix failing test --- src/core/IronPython/Modules/Builtin.cs | 4 ++-- tests/suite/test_stdconsole.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/IronPython/Modules/Builtin.cs b/src/core/IronPython/Modules/Builtin.cs index f258a3de7..94a7dcb5e 100644 --- a/src/core/IronPython/Modules/Builtin.cs +++ b/src/core/IronPython/Modules/Builtin.cs @@ -58,7 +58,7 @@ public static object __import__(CodeContext/*!*/ context, [NotNone] string name, object module = Importer.ImportModule(context, globals, name, hasFromList, level); if (module is null) return ModuleNotFound(context, name); - if (hasFromList && HasPath(module)) { + if (hasFromList && HasPath(context, module)) { return PythonOps.Invoke(context, bootstrap, "_handle_fromlist", module, fromlist, PythonOps.GetBoundAttr(context, bootstrap, "_gcd_import"))!; } return module; @@ -92,7 +92,7 @@ public static object __import__(CodeContext/*!*/ context, [NotNone] string name, return module; } - bool HasPath(object module) { + static bool HasPath(CodeContext context, object module) { // the module type has no __path__ or __getattr__, so only the dict matters (and a miss through HasAttr is slow) if (module is PythonModule mod && mod.GetType() == typeof(PythonModule)) return mod.__dict__._storage.TryGetPath(out _); return PythonOps.HasAttr(context, module, "__path__"); diff --git a/tests/suite/test_stdconsole.py b/tests/suite/test_stdconsole.py index 51205e701..a25a42fe6 100644 --- a/tests/suite/test_stdconsole.py +++ b/tests/suite/test_stdconsole.py @@ -267,7 +267,7 @@ def test_E(self): with open(tmpscript2, "w") as f: f.write("print(2+2)\n") - self.TestCommandLine(('-E', tmpscript2), "4\n") + self.TestCommandLine(('-E', '-S', tmpscript2), "4\n") tmpscript3 = os.path.join(self.tmpdir, "startupdie.py") with open(tmpscript3, "w") as f: From 099d082ff156ef8147e92ba8efe04c2534f7d197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 28 Oct 2026 21:08:02 -0400 Subject: [PATCH 03/10] Move Globals import to ClrImporter --- src/core/IronPython/Runtime/Importer.cs | 27 +++++++++------------- tests/suite/test_imp.py | 30 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/core/IronPython/Runtime/Importer.cs b/src/core/IronPython/Runtime/Importer.cs index bed7035a8..7d41d10ca 100644 --- a/src/core/IronPython/Runtime/Importer.cs +++ b/src/core/IronPython/Runtime/Importer.cs @@ -326,15 +326,8 @@ public static object ImportModule(CodeContext/*!*/ context, object globals, stri if (modules.TryGetValue(absName, out object? module) && module is not null) { WaitForInitializing(module, absName); } else { - // anything unusual goes through _gcd_import for its errors - try { - module = PythonOps.Invoke(context, bootstrap, "_gcd_import", absName); - } catch (ImportException e) when (level == 0 && !modName.Contains('.') - && e.GetPythonException() is PythonExceptions._ImportError { name: string errName } && errName == modName - && PythonOps.ScopeTryGetMember(context, pc.DomainManager.Globals, modName, out _)) { - // IronPython: fall back on runtime globals (e.g. COM type libraries), unless the ImportError came from a module that was found - module = ImportReflected(context, modName); - } + // not imported yet (or blocked by None): _gcd_import finds and loads it, or raises + module = PythonOps.Invoke(context, bootstrap, "_gcd_import", absName); } object? top = module; @@ -740,7 +733,7 @@ private static object ImportTopAbsolute(CodeContext/*!*/ context, string/*!*/ na return ret; } - private static string [] SubArray(string[] t, int len) { + private static string[] SubArray(string[] t, int len) { var ret = new string[len]; Array.Copy(t, ret, len); return ret; @@ -1080,7 +1073,7 @@ internal static PythonModule LoadPackageFromSource(CodeContext/*!*/ context, str return null; } - if(context.LanguageContext.DomainManager.Platform.DirectoryExists(path) && !context.LanguageContext.DomainManager.Platform.FileExists(context.LanguageContext.DomainManager.Platform.CombinePaths(path, "__init__.py"))) { + if (context.LanguageContext.DomainManager.Platform.DirectoryExists(path) && !context.LanguageContext.DomainManager.Platform.FileExists(context.LanguageContext.DomainManager.Platform.CombinePaths(path, "__init__.py"))) { PythonOps.Warn(context, PythonExceptions.ImportWarning, "Not importing directory '{0}': missing __init__.py", path); } @@ -1095,8 +1088,8 @@ internal static PythonModule LoadPackageFromSource(CodeContext/*!*/ context, str #nullable enable /// - /// sys.meta_path finder for .NET namespaces and types. It is added after importlib's PathFinder - /// so Python modules take precedence over .NET namespaces and types. + /// sys.meta_path finder for the runtime's globals and .NET namespaces and types. It is added after + /// importlib's PathFinder so Python modules take precedence over them. /// [PythonType] public sealed class ClrImporter { @@ -1132,7 +1125,7 @@ public object load_module(CodeContext/*!*/ context, [NotNone] string fullname) { } /// - /// Looks up a top-level namespace or type, or a child of a namespace that has been imported. + /// Looks up a name in the runtime's globals, a top-level namespace or type, or a child of a namespace that has been imported. /// Returns the same object as attribute access on the namespace. /// private static object? FindMember(CodeContext/*!*/ context, string fullname) { @@ -1143,8 +1136,10 @@ public object load_module(CodeContext/*!*/ context, [NotNone] string fullname) { int lastDot = fullname.LastIndexOf('.'); if (lastDot == -1) { // names in the runtime's globals (e.g. COM type libraries) take precedence over namespaces - // and are left to the legacy importer - if (PythonOps.ScopeTryGetMember(context, pc.DomainManager.Globals, fullname, out _)) return null; + if (PythonOps.ScopeTryGetMember(context, pc.DomainManager.Globals, fullname, out object? value)) { + // use the shared CLS context to avoid setting ShowCls on importlib._bootstrap + return MemberTrackerToPython(pc.SharedClsContext, value); + } parent = pc.TopNamespace; name = fullname; } else { diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index c91b10300..8ee3d6837 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -1201,6 +1201,36 @@ def test_clr_importer_globals_import_error(self): del runtime_globals.globals_import_error sys.modules.pop('globals_import_error', None) + @skipUnlessIronPython() + def test_clr_importer_globals_blocked(self): + # a None in sys.modules blocks the import even if the name is in the runtime's globals + import clr + runtime_globals = clr.GetCurrentRuntime().Globals + sys.modules['blocked_name'] = None + runtime_globals.blocked_name = object() + try: + with self.assertRaises(ImportError) as cm: + import blocked_name + self.assertEqual(cm.exception.name, 'blocked_name') + self.assertIsNone(sys.modules['blocked_name']) + finally: + del runtime_globals.blocked_name + sys.modules.pop('blocked_name', None) + + @skipUnlessIronPython() + def test_clr_importer_globals_import_module(self): + # names in the runtime's globals are visible to importlib too + import clr + runtime_globals = clr.GetCurrentRuntime().Globals + value = object() + runtime_globals.globals_import_module = value + try: + self.assertIs(importlib.import_module('globals_import_module'), value) + self.assertIs(sys.modules['globals_import_module'], value) + finally: + del runtime_globals.globals_import_module + sys.modules.pop('globals_import_module', None) + def test_get_frozen_object(self): # frozen objects not supported, this always fails self.assertRaises(ImportError, imp.get_frozen_object, 'foo') From 82c31e79dedf59405058f3a8ab8b367252242578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 28 Oct 2026 21:28:08 -0400 Subject: [PATCH 04/10] Remove invalid null check --- src/core/IronPython/Modules/Builtin.cs | 1 - tests/suite/test_imp.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/core/IronPython/Modules/Builtin.cs b/src/core/IronPython/Modules/Builtin.cs index 94a7dcb5e..4189fd56a 100644 --- a/src/core/IronPython/Modules/Builtin.cs +++ b/src/core/IronPython/Modules/Builtin.cs @@ -56,7 +56,6 @@ public static object __import__(CodeContext/*!*/ context, [NotNone] string name, var hasFromList = PythonOps.IsTrue(fromlist); object module = Importer.ImportModule(context, globals, name, hasFromList, level); - if (module is null) return ModuleNotFound(context, name); if (hasFromList && HasPath(context, module)) { return PythonOps.Invoke(context, bootstrap, "_handle_fromlist", module, fromlist, PythonOps.GetBoundAttr(context, bootstrap, "_gcd_import"))!; diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index 8ee3d6837..b1fae7448 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -629,6 +629,19 @@ def test_import_relative_loaded(self): sys.modules.pop('testpkg1', None) sys.modules.pop('testpkg1.mod1', None) + def test_import_relative_blocked_top(self): + """a relative import without a fromlist returns the top module from sys.modules, even if it's None""" + import types + names = ['p', 'p.a', 'p.a.b'] + for name in names: + sys.modules[name] = types.ModuleType(name) + try: + sys.modules['p.a'] = None + self.assertIsNone(__import__('a.b', {'__package__': 'p'}, None, None, 1)) + finally: + for name in names: + sys.modules.pop(name, None) + @unittest.skip('No access to CPython stdlib') def test_import_hooks_import_precence(self): """__import__ takes precedence over import hooks""" From 92ec8eba400581c28d9923420e4f0e87804c6e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 28 Oct 2026 21:28:58 -0400 Subject: [PATCH 05/10] Re-enable test_import_hooks_import_precedence --- tests/suite/test_imp.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index b1fae7448..810862ad3 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -642,8 +642,7 @@ def test_import_relative_blocked_top(self): for name in names: sys.modules.pop(name, None) - @unittest.skip('No access to CPython stdlib') - def test_import_hooks_import_precence(self): + def test_import_hooks_import_precedence(self): """__import__ takes precedence over import hooks""" global myimpCalled myimpCalled = None @@ -658,7 +657,7 @@ def myimport(*args): import distutils import distutils.command mi = myimp() - sys.meta_path.append(mi) + sys.meta_path.insert(0, mi) builtinimp = get_builtins_dict()['__import__'] try: get_builtins_dict()['__import__'] = myimport @@ -666,16 +665,18 @@ def myimport(*args): import abc self.assertEqual(abc, 'myimport') self.assertEqual(myimpCalled, None) + finally: + get_builtins_dict()['__import__'] = builtinimp - # reload on a built-in hits the loader protocol + try: + # reload hits the import hooks imp.reload(distutils) self.assertEqual(myimpCalled, ('distutils', None)) imp.reload(distutils.command) self.assertEqual(myimpCalled[0], 'distutils.command') - self.assertEqual(myimpCalled[1][0][-7:], 'distutils') + self.assertEqual(myimpCalled[1][0][-9:], 'distutils') finally: - get_builtins_dict()['__import__'] = builtinimp sys.meta_path.remove(mi) def test_import_hooks_bad_importer(self): From 6d54516ad985690a1c38a65b4dd4e405189f7021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 28 Oct 2026 21:51:24 -0400 Subject: [PATCH 06/10] ValueError on import of empty name --- src/core/IronPython/Runtime/Importer.cs | 14 +++++++++----- tests/suite/test_imp.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/core/IronPython/Runtime/Importer.cs b/src/core/IronPython/Runtime/Importer.cs index 7d41d10ca..622363f17 100644 --- a/src/core/IronPython/Runtime/Importer.cs +++ b/src/core/IronPython/Runtime/Importer.cs @@ -321,6 +321,9 @@ public static object ImportModule(CodeContext/*!*/ context, object globals, stri if (!TryResolveName(modName, strPackage, level, out absName)) { throw PythonOps.ValueError("attempted relative import beyond top-level package"); } + } else if (modName.Length == 0) { + // check before the sys.modules lookup, like CPython + throw PythonOps.ValueError("Empty module name"); } if (modules.TryGetValue(absName, out object? module) && module is not null) { @@ -346,13 +349,14 @@ public static object ImportModule(CodeContext/*!*/ context, object globals, stri if (top is MemberTracker || top is PythonType { IsSystemType: true } || top == pc.ClrModule) { context.ShowCls = true; } - } else if (!bottom && modName.Length != 0) { + } else if (!bottom) { // slice the resolved name like CPython's C implementation, not __name__ like _bootstrap.__import__ int firstDot = modName.IndexOf('.'); - int cutOff = firstDot == -1 ? 0 : modName.Length - firstDot; - string topName = absName.Substring(0, absName.Length - cutOff); - if (!modules.TryGetValue(topName, out top)) { - throw PythonOps.KeyError($"{PythonOps.Repr(context, topName)} not in sys.modules as expected"); + if (firstDot != -1) { + string topName = absName.Substring(0, absName.Length - (modName.Length - firstDot)); + if (!modules.TryGetValue(topName, out top)) { + throw PythonOps.KeyError($"{PythonOps.Repr(context, topName)} not in sys.modules as expected"); + } } } diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index 810862ad3..7031ffd3b 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -642,6 +642,16 @@ def test_import_relative_blocked_top(self): for name in names: sys.modules.pop(name, None) + def test_import_empty_name(self): + """an absolute import of an empty name raises, even if it's in sys.modules""" + self.assertRaises(ValueError, __import__, '') + sys.modules[''] = sys + try: + self.assertRaises(ValueError, __import__, '') + self.assertRaises(ValueError, __import__, '', None, None, ['x']) + finally: + del sys.modules[''] + def test_import_hooks_import_precedence(self): """__import__ takes precedence over import hooks""" global myimpCalled From fed72f779a71ad2a3004a7206028d2fdfbe96356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Sat, 24 Oct 2026 23:25:25 -0400 Subject: [PATCH 07/10] Move ModuleSpec to PythonContext --- src/core/IronPython/Runtime/Importer.cs | 8 ++------ src/core/IronPython/Runtime/PythonContext.cs | 10 ++++++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/core/IronPython/Runtime/Importer.cs b/src/core/IronPython/Runtime/Importer.cs index 622363f17..df9f2dd8a 100644 --- a/src/core/IronPython/Runtime/Importer.cs +++ b/src/core/IronPython/Runtime/Importer.cs @@ -1097,11 +1097,7 @@ internal static PythonModule LoadPackageFromSource(CodeContext/*!*/ context, str /// [PythonType] public sealed class ClrImporter { - private readonly object _moduleSpec; - - internal ClrImporter(object moduleSpec) { - _moduleSpec = moduleSpec; - } + internal ClrImporter() { } public object? find_spec(CodeContext/*!*/ context, [NotNone] string fullname, object? path = null, object? target = null) { object? member = FindMember(context, fullname); @@ -1110,7 +1106,7 @@ internal ClrImporter(object moduleSpec) { // we return "this" as the loader for the module, which uses the legacy load_module // rather than exec_module since types and namespaces don't support setting // the module attributes (__name__, etc.) - return PythonCalls.CallWithKeywordArgs(context, _moduleSpec, [fullname, this, member is NamespaceTracker], ["is_package"]); + return PythonCalls.CallWithKeywordArgs(context, context.LanguageContext.ImportLibModuleSpec, [fullname, this, member is NamespaceTracker], ["is_package"]); } public object? find_module(CodeContext/*!*/ context, [NotNone] string fullname, object? path = null) { diff --git a/src/core/IronPython/Runtime/PythonContext.cs b/src/core/IronPython/Runtime/PythonContext.cs index 0a4ce2574..d8d3a8977 100644 --- a/src/core/IronPython/Runtime/PythonContext.cs +++ b/src/core/IronPython/Runtime/PythonContext.cs @@ -306,8 +306,9 @@ void BootstrapImportLib() { #if PYTHON_37_OR_GREATER PythonOps.Invoke(SharedClsContext, _frozen_importlib, "_install_external_importers"); #endif - var moduleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec"); - (GetSystemStateValue("meta_path") as PythonList)?.append(new Importer.ClrImporter(moduleSpec)); + ImportLibModuleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec"); + var metaPath = GetSystemStateValue("meta_path") as PythonList; + metaPath?.append(new Importer.ClrImporter()); ImportLibBootstrap = _frozen_importlib; } catch { } @@ -1154,6 +1155,11 @@ internal PythonModule GetReloadableModule(PythonModule/*!*/ module) { /// internal PythonModule ImportLibBootstrap { get; private set; } + /// + /// importlib's ModuleSpec type, used by our sys.meta_path finders to create specs. + /// + internal object ImportLibModuleSpec { get; private set; } + private object copyRegModule; internal object GetCopyRegModule() { From 38bf06ca6947740af4a2a1932580dce1fdf021ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Thu, 29 Oct 2026 19:36:43 -0400 Subject: [PATCH 08/10] Give .NET namespaces precedence over namespace packages --- src/core/IronPython/Runtime/Importer.cs | 28 ++++++++++++++++--- src/core/IronPython/Runtime/PythonContext.cs | 17 +++++++++--- tests/suite/test_imp.py | 29 +++++++++++++++++++- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/src/core/IronPython/Runtime/Importer.cs b/src/core/IronPython/Runtime/Importer.cs index df9f2dd8a..eaf081935 100644 --- a/src/core/IronPython/Runtime/Importer.cs +++ b/src/core/IronPython/Runtime/Importer.cs @@ -1092,17 +1092,26 @@ internal static PythonModule LoadPackageFromSource(CodeContext/*!*/ context, str #nullable enable /// - /// sys.meta_path finder for the runtime's globals and .NET namespaces and types. It is added after - /// importlib's PathFinder so Python modules take precedence over them. + /// sys.meta_path finder for the runtime's globals and .NET namespaces and types. It can optionally be + /// given importlib's PathFinder, in which case it should be added before PathFinder so that these take + /// precedence over namespace packages, but defers to it for Python modules and regular packages. /// [PythonType] public sealed class ClrImporter { - internal ClrImporter() { } + private readonly object? _pathFinder; + + internal ClrImporter(object? pathFinder = null) { + _pathFinder = pathFinder; + } public object? find_spec(CodeContext/*!*/ context, [NotNone] string fullname, object? path = null, object? target = null) { object? member = FindMember(context, fullname); if (member is null) return null; + // a Python module or regular package takes precedence, so return its spec rather than + // deferring, which would run the path finder a second time + if (FindPathSpec(context, fullname, path, target) is object spec) return spec; + // we return "this" as the loader for the module, which uses the legacy load_module // rather than exec_module since types and namespaces don't support setting // the module attributes (__name__, etc.) @@ -1110,7 +1119,18 @@ internal ClrImporter() { } } public object? find_module(CodeContext/*!*/ context, [NotNone] string fullname, object? path = null) { - return FindMember(context, fullname) is null ? null : this; + object? spec = find_spec(context, fullname, path); + if (spec is null) return null; + return PythonOps.GetBoundAttr(context, spec, "loader"); + } + + /// + /// Returns the spec of a Python module or regular package found by the path finder, ignoring namespace packages. + /// + private object? FindPathSpec(CodeContext/*!*/ context, string fullname, object? path, object? target) { + if (_pathFinder is null) return null; + object? spec = PythonOps.Invoke(context, _pathFinder, "find_spec", fullname, path, target); + return spec is not null && PythonOps.GetBoundAttr(context, spec, "loader") is not null ? spec : null; } public object load_module(CodeContext/*!*/ context, [NotNone] string fullname) { diff --git a/src/core/IronPython/Runtime/PythonContext.cs b/src/core/IronPython/Runtime/PythonContext.cs index d8d3a8977..e1f8923be 100644 --- a/src/core/IronPython/Runtime/PythonContext.cs +++ b/src/core/IronPython/Runtime/PythonContext.cs @@ -300,17 +300,26 @@ void BootstrapImportLib() { try { var _frozen_importlib = LoadModuleFromResource("_frozen_importlib", "IronPython.Modules._bootstrap.py"); #if PYTHON_36_OR_GREATER - LoadModuleFromResource("_frozen_importlib_external", "IronPython.Modules._bootstrap_external.py"); + var _frozen_importlib_external = LoadModuleFromResource("_frozen_importlib_external", "IronPython.Modules._bootstrap_external.py"); #endif PythonOps.Invoke(SharedClsContext, _frozen_importlib, "_install", SystemState, GetBuiltinModule("_imp")); #if PYTHON_37_OR_GREATER PythonOps.Invoke(SharedClsContext, _frozen_importlib, "_install_external_importers"); #endif ImportLibModuleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec"); - var metaPath = GetSystemStateValue("meta_path") as PythonList; - metaPath?.append(new Importer.ClrImporter()); + var metaPath = (PythonList)GetSystemStateValue("meta_path"); +#if PYTHON_36_OR_GREATER + var pathFinder = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib_external, "PathFinder"); +#else + var pathFinder = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "PathFinder"); +#endif + // Python modules take precedence over .NET namespaces and types; namespace packages come last, + // so that a directory on sys.path doesn't hide a .NET namespace + metaPath.insert(metaPath.index(pathFinder), new Importer.ClrImporter(pathFinder)); ImportLibBootstrap = _frozen_importlib; - } catch { } + } catch { + Debug.Fail("Failed to bootstrap importlib"); + } PythonModule LoadModuleFromResource(string name, string resourceName) { var sourceUnit = CreateSourceUnit(new ResourceStreamContentProvider(resourceName), null, DefaultEncoding, SourceCodeKind.File); diff --git a/tests/suite/test_imp.py b/tests/suite/test_imp.py index 7031ffd3b..66085c9cd 100644 --- a/tests/suite/test_imp.py +++ b/tests/suite/test_imp.py @@ -1145,7 +1145,7 @@ def test_clr_importer(self): import System.Collections.Generic names = [type(finder).__name__ if not isinstance(finder, type) else finder.__name__ for finder in sys.meta_path] - self.assertLess(names.index('PathFinder'), names.index('ClrImporter')) + self.assertLess(names.index('ClrImporter'), names.index('PathFinder')) # namespaces and types are imported as the same objects as attribute access gives self.assertIs(importlib.import_module('System.Collections.Generic'), System.Collections.Generic) @@ -1225,6 +1225,33 @@ def test_clr_importer_globals_import_error(self): del runtime_globals.globals_import_error sys.modules.pop('globals_import_error', None) + @skipUnlessIronPython() + def test_clr_importer_namespace_package(self): + # .NET namespaces take precedence over namespace packages, but not over regular packages and modules + import clr + from System.Reflection import AssemblyName, TypeAttributes + from System.Reflection.Emit import AssemblyBuilder, AssemblyBuilderAccess + names = ['ClrImporterNsDir', 'ClrImporterNsPkg', 'ClrImporterNsMod'] + ab = AssemblyBuilder.DefineDynamicAssembly(AssemblyName('ClrImporterNamespace'), AssemblyBuilderAccess.Run) + mb = ab.DefineDynamicModule('ClrImporterNamespace') + for name in names: + mb.DefineType(name + '.Type', TypeAttributes.Public).CreateType() + clr.AddReference(ab) + + self.ensure_directory_present(os.path.join(self._imptestdir, 'ClrImporterNsDir')) + self.write_to_file(os.path.join(self._imptestdir, 'ClrImporterNsPkg', '__init__.py'), 'value = 1') + self.write_to_file(os.path.join(self._imptestdir, 'ClrImporterNsMod.py'), 'value = 2') + importlib.invalidate_caches() + try: + with path_modifier(self._imptestdir): + import ClrImporterNsDir, ClrImporterNsPkg, ClrImporterNsMod + self.assertTrue(hasattr(ClrImporterNsDir, 'Type')) + self.assertEqual(ClrImporterNsPkg.value, 1) + self.assertEqual(ClrImporterNsMod.value, 2) + finally: + for name in names: + sys.modules.pop(name, None) + @skipUnlessIronPython() def test_clr_importer_globals_blocked(self): # a None in sys.modules blocks the import even if the name is in the runtime's globals From e49c59c88ffe9326e1791adcbd9a487f7a9834b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Tue, 27 Oct 2026 21:59:22 -0400 Subject: [PATCH 09/10] Enable importlib tests --- .../Cases/CPythonCasesManifest.ini | 51 +++++++++++++++++-- tests/IronPython.Tests/Cases/CaseExecuter.cs | 8 ++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini b/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini index 41ddd9a4c..0486cefb1 100644 --- a/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini +++ b/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini @@ -504,6 +504,54 @@ Ignore=true Reason=ImportError: No module named _multiprocessing [CPython.test_importlib] +Arguments=-m $(TEST_MODULE) # relative imports +IsolationLevel=PROCESS # required for Arguments to have effect + +[CPython.test_importlib.extension.test_finder] +Ignore=true + +[CPython.test_importlib.extension.test_loader] +Ignore=true + +[CPython.test_importlib.frozen.test_finder] +Ignore=true + +[CPython.test_importlib.frozen.test_loader] +Ignore=true + +[CPython.test_importlib.import_.test_fromlist] +Ignore=true +Reason=3.6 test_star_in_all: hasattr(module, '*') raises SystemError (trailing * is a .NET reflection wildcard) + +[CPython.test_importlib.source.test_file_loader] +Ignore=true + +[CPython.test_importlib.source.test_finder] +Ignore=true + +[CPython.test_importlib.test_abc] +Ignore=true + +[CPython.test_importlib.test_api] +Ignore=true + +[CPython.test_importlib.test_lazy] +Ignore=true +Reason=TypeError: __class__ assignment only supported for heap types or ModuleType subclasses + +[CPython.test_importlib.test_locks] +Ignore=true + +[CPython.test_importlib.test_namespace_pkgs] +Ignore=true + +[CPython.test_importlib.test_spec] +Ignore=true + +[CPython.test_importlib.test_util] +Ignore=true + +[CPython.test_importlib.test_windows] Ignore=true [CPython.test_inspect] @@ -889,9 +937,6 @@ Ignore=true Ignore=true Reason=Blocking -[CPython.test_threaded_import] -Ignore=true - [CPython.test_threading] IsolationLevel=PROCESS RunCondition=NOT $(IS_MONO) # TODO: figure out diff --git a/tests/IronPython.Tests/Cases/CaseExecuter.cs b/tests/IronPython.Tests/Cases/CaseExecuter.cs index 3a377753d..0cdfa1a09 100644 --- a/tests/IronPython.Tests/Cases/CaseExecuter.cs +++ b/tests/IronPython.Tests/Cases/CaseExecuter.cs @@ -157,6 +157,11 @@ private static string GetIronPythonPath() { return string.Empty; } + // the module name of a StdLib file, for packages with relative imports to run with -m + private static string GetModuleName(string path) { + return Path.GetRelativePath(IRONPYTHONPATH, Path.ChangeExtension(path, null)).Replace(Path.DirectorySeparatorChar, '.'); + } + private string ReplaceVariables(string input, Dictionary replacements) { Regex variableRegex = new Regex(@"\$\(([^}]+)\)", RegexOptions.Compiled); @@ -189,7 +194,8 @@ private int GetProcessTest(TestInfo testcase) { int exitCode = -1; var argReplacements = new Dictionary() { { "TEST_FILE", testcase.Path }, - { "TEST_FILE_DIR", Path.GetDirectoryName(testcase.Path) } + { "TEST_FILE_DIR", Path.GetDirectoryName(testcase.Path) }, + { "TEST_MODULE", GetModuleName(testcase.Path) } }; var wdReplacements = new Dictionary() { From d5437064898230366543004ca4ce10bdabab26e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Thu, 29 Oct 2026 21:04:57 -0400 Subject: [PATCH 10/10] Set RetryCount=2 for test_ssl --- tests/IronPython.Tests/Cases/CPythonCasesManifest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini b/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini index 0486cefb1..f5e2467cf 100644 --- a/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini +++ b/tests/IronPython.Tests/Cases/CPythonCasesManifest.ini @@ -848,6 +848,9 @@ IsolationLevel=PROCESS RunCondition=NOT $(IS_OSX) # https://github.com/IronLanguages/ironpython3/issues/1483 NotParallelSafe=true +[CPython.test_ssl] +RetryCount=2 # test_dealloc_warn is flaky on Mono + [CPython.test_startfile] RunCondition=NOT $(IS_POSIX) # TODO: debug?