diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d72a3a0..4c2fe6fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Extension methods declared in non-framework assemblies are bound to the type they extend, so they can be called from the instance like in C# (e.g. `order_status.is_open()`) - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine diff --git a/src/embed_tests/TestExtensionMethods.cs b/src/embed_tests/TestExtensionMethods.cs new file mode 100644 index 000000000..316b49bf6 --- /dev/null +++ b/src/embed_tests/TestExtensionMethods.cs @@ -0,0 +1,193 @@ +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public enum ExtendedStatus { Open, Closed } + + public interface IExtendedShape + { + double Side { get; } + } + + public class ExtendedModel + { + public string Name { get; set; } = "model"; + + public string Existing() => "member"; + } + + public class DerivedExtendedModel : ExtendedModel + { + } + + public class ExtendedSquare : IExtendedShape + { + public double Side => 2; + } + + public static class ExtendedStatusExtensions + { + public static bool IsOpen(this ExtendedStatus status) => status == ExtendedStatus.Open; + + public static string Describe(this ExtendedStatus status, string prefix, int times = 1) + => string.Concat(Enumerable.Repeat(prefix + status, times)); + } + + public static class ExtendedModelExtensions + { + public static string Describe(this ExtendedModel model, string prefix = "") => prefix + model.Name; + + // collides with the real instance method, which must win + public static string Existing(this ExtendedModel model) => "extension"; + + public static double Area(this IExtendedShape shape) => shape.Side * shape.Side; + + public static int Total(this List values) => values.Sum(); + + public static T Identity(this T value) => value; + + public static string Twice(this string value) => value + value; + } + + public class TestExtensionMethods + { + private static dynamic module; + + private const string TestModule = @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") +from System.Collections.Generic import List +from Python.EmbeddingTest import * + +def enum_instance(): + return ExtendedStatus.Open.IsOpen(), ExtendedStatus.Open.is_open(), ExtendedStatus.Closed.is_open() + +def enum_with_args(): + return ExtendedStatus.Open.describe('s-', times=2) + +def enum_static_forms(): + return ExtendedStatusExtensions.is_open(ExtendedStatus.Open), ExtendedStatus.is_open(ExtendedStatus.Closed) + +def class_instance(): + return ExtendedModel().describe(), ExtendedModel().describe(prefix='p-') + +def derived_inherits(): + return DerivedExtendedModel().describe('d-') + +def interface_target(): + return ExtendedSquare().area() + +def member_wins(): + return ExtendedModel().existing() + +def closed_generic(): + values = List[int]() + values.Add(1) + values.Add(2) + return values.total() + +def open_generic_not_bound(): + return hasattr(ExtendedModel(), 'identity') + +def python_native_not_bound(): + return hasattr('ab', 'twice'), ExtendedModelExtensions.twice('ab') +"; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + PythonEngine.Initialize(); + using var _ = Py.GIL(); + module = PyModule.FromString("extension_methods", TestModule); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void EnumExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.enum_instance(); + Assert.IsTrue((bool)result[0]); + Assert.IsTrue((bool)result[1]); + Assert.IsFalse((bool)result[2]); + } + + [Test] + public void ExtensionArgumentsFollowTheInstance() + { + using var _ = Py.GIL(); + Assert.AreEqual("s-Opens-Open", (string)module.enum_with_args()); + } + + [Test] + public void StaticFormsKeepWorking() + { + using var _ = Py.GIL(); + var result = module.enum_static_forms(); + Assert.IsTrue((bool)result[0]); + Assert.IsFalse((bool)result[1]); + } + + [Test] + public void ClassExtensionIsCallableFromTheInstance() + { + using var _ = Py.GIL(); + var result = module.class_instance(); + Assert.AreEqual("model", (string)result[0]); + Assert.AreEqual("p-model", (string)result[1]); + } + + [Test] + public void DerivedClassInheritsBaseClassExtensions() + { + using var _ = Py.GIL(); + Assert.AreEqual("d-model", (string)module.derived_inherits()); + } + + [Test] + public void InterfaceExtensionIsBoundToImplementingClass() + { + using var _ = Py.GIL(); + Assert.AreEqual(4.0, (double)module.interface_target()); + } + + [Test] + public void RealMemberWinsOverExtension() + { + using var _ = Py.GIL(); + Assert.AreEqual("member", (string)module.member_wins()); + } + + [Test] + public void ClosedGenericTargetIsBound() + { + using var _ = Py.GIL(); + Assert.AreEqual(3, (int)module.closed_generic()); + } + + [Test] + public void OpenGenericTargetIsNotBound() + { + using var _ = Py.GIL(); + Assert.IsFalse((bool)module.open_generic_not_bound()); + } + + [Test] + public void PythonNativeTargetKeepsTheStaticForm() + { + using var _ = Py.GIL(); + var result = module.python_native_not_bound(); + Assert.IsFalse((bool)result[0]); + Assert.AreEqual("abab", (string)result[1]); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index dd7061c2b..5efef4906 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index 3370e4410..76b252652 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -62,6 +62,7 @@ internal static void Initialize() // test/usage after the first init cycle. Clear so the scan runs fresh. assembliesNamesCache.Clear(); assemblies.Clear(); + ExtensionMethods.Clear(); AppDomain domain = AppDomain.CurrentDomain; @@ -356,7 +357,8 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + var types = GetTypes(assembly); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -383,6 +385,8 @@ internal static void ScanAssembly(Assembly assembly) GenericUtil.Register(t); } } + + ExtensionMethods.Register(assembly, types); } public static AssemblyName[] ListAssemblies() diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b88a6a6b6..835f0ec82 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -416,6 +416,17 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) } } + // Extension methods are bound as members so they can be called from the instance. A real member with the same name wins, as in C# + var extensionMethods = ExtensionMethods.GetMethodsToBind(type).Where(em => !originalMemberNames.Contains(em.Name)).ToArray(); + if (extensionMethods.Length > 0) + { + foreach (var em in extensionMethods) + { + local.Add(em.Name); + } + info = info.Concat(extensionMethods).ToArray(); + } + // Now again to filter w/o losing overloaded member info for (i = 0; i < info.Length; i++) { diff --git a/src/runtime/ExtensionMethods.cs b/src/runtime/ExtensionMethods.cs new file mode 100644 index 000000000..2458057c3 --- /dev/null +++ b/src/runtime/ExtensionMethods.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Python.Runtime +{ + /// + /// Index of the C# extension methods found in the scanned assemblies, keyed by the type they extend. + /// Classes bind them as members so they are callable from an instance, like C# callers do. + /// + internal static class ExtensionMethods + { + private static readonly ConcurrentDictionary> _methodsByExtendedType = new(); + + // The framework ships thousands of extension methods (LINQ above all); binding them onto + // every collection would bloat each class, so only user assemblies are indexed. + private static readonly string[] _frameworkAssemblyPrefixes = { "System", "Microsoft", "mscorlib", "netstandard", "WindowsBase", "Python.Runtime" }; + + /// + /// Drops the index so a re-initialized runtime rebuilds it while re-scanning the assemblies + /// + internal static void Clear() + { + _methodsByExtendedType.Clear(); + } + + internal static bool IsExtensionMethod(MethodBase method) + { + return method.IsStatic && method.IsDefined(typeof(ExtensionAttribute), false); + } + + /// + /// Indexes the extension methods declared by the given exported types of an assembly + /// + internal static void Register(Assembly assembly, IEnumerable exportedTypes) + { + if (IsFrameworkAssembly(assembly)) + { + return; + } + + foreach (var type in exportedTypes) + { + // extension methods can only live in non-generic static classes + if (!type.IsAbstract || !type.IsSealed || type.IsGenericType || !type.IsDefined(typeof(ExtensionAttribute), false)) + { + continue; + } + + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (!IsExtensionMethod(method)) + { + continue; + } + + var extendedType = method.GetParameters()[0].ParameterType; + // The binder cannot infer open generic targets (this T, this IEnumerable) from the instance + if (extendedType.ContainsGenericParameters || extendedType.IsByRef) + { + continue; + } + + var methods = _methodsByExtendedType.GetOrAdd(extendedType, _ => new List()); + lock (methods) + { + methods.Add(method); + } + } + } + } + + /// + /// Gets the extension methods to bind to the given type: those extending the type itself and the + /// interfaces it introduces. Extensions of base classes are inherited through the Python type hierarchy. + /// + internal static List GetMethodsToBind(Type type) + { + var result = new List(); + AddMethods(type, result); + + if (!type.IsInterface) + { + var baseInterfaces = type.BaseType?.GetInterfaces() ?? Type.EmptyTypes; + foreach (var @interface in type.GetInterfaces().Except(baseInterfaces)) + { + AddMethods(@interface, result); + } + } + + return result; + } + + private static void AddMethods(Type extendedType, List result) + { + if (_methodsByExtendedType.TryGetValue(extendedType, out var methods)) + { + lock (methods) + { + result.AddRange(methods); + } + } + } + + private static bool IsFrameworkAssembly(Assembly assembly) + { + var name = assembly.GetName().Name ?? string.Empty; + return _frameworkAssemblyPrefixes.Any(prefix => name == prefix || name.StartsWith(prefix + ".", StringComparison.Ordinal)); + } + } +} diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 6deec74f8..803ee2f23 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -534,6 +534,17 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe pi = pi.Take(1).ToArray(); } + // An extension method called through an instance gets the instance as its first CLR argument + bool isExtension = inst != null && methodInformation.IsExtension; + if (isExtension) + { + pi = pi.Skip(1).ToArray(); + if (hasNamedArgs) + { + paramNames = paramNames.Skip(1).ToArray(); + } + } + // Must be done after IsOperator section int clrArgCount = pi.Length; @@ -775,6 +786,15 @@ internal Binding Bind(BorrowedReference inst, BorrowedReference args, BorrowedRe } } + if (isExtension) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + margs = new[] { co.inst }.Concat(margs).ToArray(); + } + else continue; + } + var match = new MatchedMethod(kwargsMatched, margs, outs, methodInformation, implicitConversions); if (implicitConversions > 0) { @@ -1238,9 +1258,12 @@ internal class MethodInformation { private ParameterInfo[] _parameterInfo; private string[] _parametersNames; + private bool? _isExtension; public MethodBase MethodBase { get; } + public bool IsExtension => _isExtension ??= ExtensionMethods.IsExtensionMethod(MethodBase); + public bool IsOriginal { get; set; } public ParameterInfo[] ParameterInfo diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 0a24781b8..dacadc64d 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.66")] -[assembly: AssemblyFileVersion("2.0.66")] +[assembly: AssemblyVersion("2.0.67")] +[assembly: AssemblyFileVersion("2.0.67")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 765a0dd2c..e290143d9 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.66 + 2.0.67 false LICENSE https://github.com/pythonnet/pythonnet