Use importlib for __import__

src/core/IronPython/Modules/Builtin.cs

@@ -49,15 +49,55 @@ public static partial class Builtin {
         [Documentation("__import__(name, globals, locals, fromlist, level) -> module\n\nImport a module.")]
         [LightThrowing]
         public static object __import__(CodeContext/*!*/ context, [NotNone] string name, object? globals = null, object? locals = null, object? fromlist = null, int level = 0) {
-            if (fromlist is string || fromlist is Extensible<string>) {
-                fromlist = new List<object> { fromlist };
+            PythonContext pc = context.LanguageContext;
+            PythonModule? bootstrap = pc.ImportLibBootstrap;
+
+            if (bootstrap is not null) {
+                var hasFromList = PythonOps.IsTrue(fromlist);
+
+                object module = Importer.ImportModule(context, globals, name, hasFromList, level);
+
+                if (hasFromList && HasPath(context, module)) {
+                    return PythonOps.Invoke(context, bootstrap, "_handle_fromlist", module, fromlist, PythonOps.GetBoundAttr(context, bootstrap, "_gcd_import"))!;
+                }
+                return module;
+            } else {
+                if (fromlist is string || fromlist is Extensible<string>) {
+                    fromlist = new List<object> { fromlist };
+                }
+                IList? from = fromlist as IList;
+                var hasFromList = from != null && from.Count > 0;
+
+                object module = Importer.ImportModule(context, globals, name, hasFromList, level);
+                if (module is null) return ModuleNotFound(context, name);
+
+                if (hasFromList && from is not null && module is PythonModule mod && mod.__dict__._storage.TryGetPath(out _)) {
+                    string strAttrName;
+                    for (int i = 0; i < from.Count; i++) {
+                        object? attrName = from[i];
+
+                        if (pc.TryConvertToString(attrName, out strAttrName) &&
+                            !string.IsNullOrEmpty(strAttrName) &&
+                            strAttrName != "*") {
+                            try {
+                                Importer.ImportFrom(context, mod, strAttrName);
+                            } catch (ImportException) {
+                                continue;
+                            }
+                        }
+                    }
+                }
+
+                return module;
             }
 
-            IList? from = fromlist as IList;
-            PythonContext pc = context.LanguageContext;
+            static bool HasPath(CodeContext context, object module) {
+                // the module type has no __path__ or __getattr__, so only the dict matters (and a miss through HasAttr is slow)
+                if (module is PythonModule mod && mod.GetType() == typeof(PythonModule)) return mod.__dict__._storage.TryGetPath(out _);
+                return PythonOps.HasAttr(context, module, "__path__");
+            }
 
-            object ret = Importer.ImportModule(context, globals, name, from != null && from.Count > 0, level);
-            if (ret == null) {
+            static object ModuleNotFound(CodeContext context, string name) {
 #if PYTHON_36_OR_GREATER // ModuleNotFoundError since Python 3.6
                 var err = PythonOps.ModuleNotFoundError("No module named {0}", PythonOps.Repr(context, name));
 #else
@@ -66,25 +106,6 @@ public static object __import__(CodeContext/*!*/ context, [NotNone] string name,
                 ((PythonExceptions._ImportError)err.GetPythonException()!).name = name;
                 return LightExceptions.Throw(err);
             }
-
-            if (ret is PythonModule mod && from != null) {
-                string strAttrName;
-                for (int i = 0; i < from.Count; i++) {
-                    object? attrName = from[i];
-
-                    if (pc.TryConvertToString(attrName, out strAttrName) &&
-                        !string.IsNullOrEmpty(strAttrName) &&
-                        strAttrName != "*") {
-                        try {
-                            Importer.ImportFrom(context, mod, strAttrName);
-                        } catch (ImportException) {
-                            continue;
-                        }
-                    }
-                }
-            }
-
-            return ret;
         }
 
         [Documentation("abs(number) -> number\n\nReturn the absolute value of the argument.")]

src/core/IronPython/Runtime/PythonContext.cs

@@ -300,15 +300,26 @@ void BootstrapImportLib() {
                 try {
                     var _frozen_importlib = LoadModuleFromResource("_frozen_importlib", "IronPython.Modules._bootstrap.py");
 #if PYTHON_36_OR_GREATER
-                    LoadModuleFromResource("_frozen_importlib_external", "IronPython.Modules._bootstrap_external.py");
+                    var _frozen_importlib_external = LoadModuleFromResource("_frozen_importlib_external", "IronPython.Modules._bootstrap_external.py");
 #endif
                     PythonOps.Invoke(SharedClsContext, _frozen_importlib, "_install", SystemState, GetBuiltinModule("_imp"));
 #if PYTHON_37_OR_GREATER
                     PythonOps.Invoke(SharedClsContext, _frozen_importlib, "_install_external_importers");
 #endif
-                    var moduleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec");
-                    (GetSystemStateValue("meta_path") as PythonList)?.append(new Importer.ClrImporter(moduleSpec));
-                } catch { }
+                    ImportLibModuleSpec = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "ModuleSpec");
+                    var metaPath = (PythonList)GetSystemStateValue("meta_path");
+#if PYTHON_36_OR_GREATER
+                    var pathFinder = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib_external, "PathFinder");
+#else
+                    var pathFinder = PythonOps.GetBoundAttr(SharedClsContext, _frozen_importlib, "PathFinder");
+#endif
+                    // Python modules take precedence over .NET namespaces and types; namespace packages come last,
+                    // so that a directory on sys.path doesn't hide a .NET namespace
+                    metaPath.insert(metaPath.index(pathFinder), new Importer.ClrImporter(pathFinder));
+                    ImportLibBootstrap = _frozen_importlib;
+                } catch {
+                    Debug.Fail("Failed to bootstrap importlib");
+                }
 
                 PythonModule LoadModuleFromResource(string name, string resourceName) {
                     var sourceUnit = CreateSourceUnit(new ResourceStreamContentProvider(resourceName), null, DefaultEncoding, SourceCodeKind.File);
@@ -1147,6 +1158,17 @@ internal PythonModule GetReloadableModule(PythonModule/*!*/ module) {
 
         #endregion
 
+        /// <summary>
+        /// The _frozen_importlib module, which drives imports once it has been bootstrapped.
+        /// When null (e.g. NoImportLib), the legacy importer is used instead.
+        /// </summary>
+        internal PythonModule ImportLibBootstrap { get; private set; }
+
+        /// <summary>
+        /// importlib's ModuleSpec type, used by our sys.meta_path finders to create specs.
+        /// </summary>
+        internal object ImportLibModuleSpec { get; private set; }
+
         private object copyRegModule;
 
         internal object GetCopyRegModule() {

src/core/IronPython/Runtime/PythonModule.cs

@@ -145,7 +145,7 @@ public void __setattr__(CodeContext/*!*/ context, string name, object value) {
 
             switch (name) {
                 case "__dict__": throw PythonOps.AttributeError("readonly attribute");
-                case "__class__": throw PythonOps.TypeError("__class__ assignment: only for heap types");
+                case "__class__": throw PythonOps.TypeError("__class__ assignment only supported for heap types or ModuleType subclasses");
             }
 
             Debug.Assert(value != Uninitialized.Instance);

tests/IronPython.Tests/Cases/CPythonCasesManifest.ini

@@ -504,6 +504,54 @@ Ignore=true
 Reason=ImportError: No module named _multiprocessing
 
 [CPython.test_importlib]
+Arguments=-m $(TEST_MODULE) # relative imports
+IsolationLevel=PROCESS # required for Arguments to have effect
+
+[CPython.test_importlib.extension.test_finder]
+Ignore=true
+
+[CPython.test_importlib.extension.test_loader]
+Ignore=true
+
+[CPython.test_importlib.frozen.test_finder]
+Ignore=true
+
+[CPython.test_importlib.frozen.test_loader]
+Ignore=true
+
+[CPython.test_importlib.import_.test_fromlist]
+Ignore=true
+Reason=3.6 test_star_in_all: hasattr(module, '*') raises SystemError (trailing * is a .NET reflection wildcard)
+
+[CPython.test_importlib.source.test_file_loader]
+Ignore=true
+
+[CPython.test_importlib.source.test_finder]
+Ignore=true
+
+[CPython.test_importlib.test_abc]
+Ignore=true
+
+[CPython.test_importlib.test_api]
+Ignore=true
+
+[CPython.test_importlib.test_lazy]
+Ignore=true
+Reason=TypeError: __class__ assignment only supported for heap types or ModuleType subclasses
+
+[CPython.test_importlib.test_locks]
+Ignore=true
+
+[CPython.test_importlib.test_namespace_pkgs]
+Ignore=true
+
+[CPython.test_importlib.test_spec]
+Ignore=true
+
+[CPython.test_importlib.test_util]
+Ignore=true
+
+[CPython.test_importlib.test_windows]
 Ignore=true
 
 [CPython.test_inspect]
@@ -800,6 +848,9 @@ IsolationLevel=PROCESS
 RunCondition=NOT $(IS_OSX) # https://github.com/IronLanguages/ironpython3/issues/1483
 NotParallelSafe=true
 
+[CPython.test_ssl]
+RetryCount=2 # test_dealloc_warn is flaky on Mono
+
 [CPython.test_startfile]
 RunCondition=NOT $(IS_POSIX) # TODO: debug?
 
@@ -889,9 +940,6 @@ Ignore=true
 Ignore=true
 Reason=Blocking
 
-[CPython.test_threaded_import]
-Ignore=true
-
 [CPython.test_threading]
 IsolationLevel=PROCESS
 RunCondition=NOT $(IS_MONO) # TODO: figure out

tests/IronPython.Tests/Cases/CaseExecuter.cs

@@ -157,6 +157,11 @@ private static string GetIronPythonPath() {
             return string.Empty;
         }
 
+        // the module name of a StdLib file, for packages with relative imports to run with -m
+        private static string GetModuleName(string path) {
+            return Path.GetRelativePath(IRONPYTHONPATH, Path.ChangeExtension(path, null)).Replace(Path.DirectorySeparatorChar, '.');
+        }
+
         private string ReplaceVariables(string input, Dictionary<string, string> replacements) {
             Regex variableRegex = new Regex(@"\$\(([^}]+)\)", RegexOptions.Compiled);
 
@@ -189,7 +194,8 @@ private int GetProcessTest(TestInfo testcase) {
             int exitCode = -1;
             var argReplacements = new Dictionary<string, string>() {
                 { "TEST_FILE", testcase.Path },
-                { "TEST_FILE_DIR", Path.GetDirectoryName(testcase.Path) }
+                { "TEST_FILE_DIR", Path.GetDirectoryName(testcase.Path) },
+                { "TEST_MODULE", GetModuleName(testcase.Path) }
             };
 
             var wdReplacements = new Dictionary<string, string>() {

tests/IronPython.Tests/Stress/Engine.cs

@@ -47,6 +47,7 @@ private static long GetTotalMemory() {
 #endif
         public void ScenarioXGC() {
             Assert.AreEqual(1, 1); // prevents nunit from counting towards memory usage
+            _pe.Execute("import gc"); // the first import initializes importlib's import machinery
 
             long initialMemory = GetTotalMemory();
 

tests/suite/import_extant/replaced_by_builtin.py

@@ -0,0 +1,3 @@
+import sys
+
+sys.modules[__name__] = len

tests/suite/import_extant/replaced_by_class.py

@@ -0,0 +1,5 @@
+import sys
+
+class ReplacedByClass: pass
+
+sys.modules[__name__] = ReplacedByClass

tests/suite/import_extant/replaced_by_namespace/__init__.py

@@ -0,0 +1,4 @@
+import sys
+import types
+
+sys.modules[__name__] = types.SimpleNamespace(__name__=__name__, __path__=__path__, __spec__=__spec__)

tests/suite/import_extant/replaced_by_namespace/sub.py

@@ -0,0 +1 @@
+value = 'sub'

tests/suite/import_extant/replaced_sub/a/b.py

@@ -0,0 +1,3 @@
+import sys
+
+sys.modules[__name__] = type("b", (), {})

tests/suite/modules/io_related/test_codecs.py

@@ -14,6 +14,7 @@
 
 import array
 import codecs
+import importlib
 import os
 import re
 import shutil
@@ -22,12 +23,9 @@
 import sys
 import unittest
 
-from iptest import IronPythonTestCase, is_cli, is_cpython, is_mono, is_netcoreapp, is_posix, is_linux, is_windows, run_test, skipUnlessIronPython
+from iptest import IronPythonTestCase, is_cli, is_cpython, is_mono, is_netcoreapp, is_posix, is_windows, run_test, skipUnlessIronPython
 from iptest.misc_util import ip_supported_encodings
 
-if is_cpython and is_linux:
-    import time
-
 class CodecTest(IronPythonTestCase):
     def test_escape_decode(self):
         # escape_decode decodes bytes to bytes, but when given a string it encodes it first with UTF-8
@@ -1840,8 +1838,7 @@ def test_file_encodings(self):
 
                     f.write("# coding: %s" % (coding))
 
-                if is_cpython and is_linux:
-                    time.sleep(0.01)
+                importlib.invalidate_caches()
                 __import__(temp_mod_name)
                 os.remove(os.path.join(self.temporary_dir, "tmp_encodings", temp_mod_name + ".py"))
 

tests/suite/test_doc.py

@@ -4,6 +4,8 @@
 
 "module doc"
 
+import importlib
+
 from iptest import IronPythonTestCase, is_cli, path_modifier, retryOnFailure, run_test, skipUnlessIronPython
 
 class DocTest(IronPythonTestCase):
@@ -208,11 +210,13 @@ def test_module_doc_cp21360(self):
         try:
             with path_modifier('.'):
                 self.write_to_file(temp_filename_empty, "")
+                importlib.invalidate_caches()
                 import cp21360_empty
                 self.assertEqual(cp21360_empty.__doc__, None)
 
             with path_modifier('.'):
                 self.write_to_file(temp_filename, "x = 3.14")
+                importlib.invalidate_caches()
                 import cp21360
                 self.assertEqual(cp21360.__doc__, None)
                 self.assertEqual(cp21360.x, 3.14)