Give traced frames their caller's frame as f_back

src/core/IronPython/Runtime/Exceptions/TraceBack.cs

@@ -105,14 +105,24 @@ internal TraceBackFrame(CodeContext/*!*/ context, PythonDictionary globals, obje
             _back = back;
         }
 
-        internal TraceBackFrame(PythonTracebackListener traceAdapter, FunctionCode code, TraceBackFrame back, PythonDebuggingPayload debugProperties, Func<IDictionary<object, object>> scopeCallback) {
+        internal TraceBackFrame(PythonTracebackListener traceAdapter, FunctionCode code, TraceBackFrame back, PythonDebuggingPayload debugProperties, Func<IDictionary<object, object>> scopeCallback, bool pushedByCaller) {
             _traceAdapter = traceAdapter;
             _code = code;
             _back = back;
             _debugProperties = debugProperties;
             _scopeCallback = scopeCallback;
+            PushedByCaller = pushedByCaller;
         }
 
+        /// <summary>
+        ///   True if the frame stack entry of this frame was pushed, and is popped, by the caller of the code rather than the trace listener.
+        /// </summary>
+        /// <remarks>
+        ///   This is the case for module code run by <c>exec</c> or a script runner, which push the entry for it since module code has
+        ///   no frame push of its own (see <see cref="PythonTracebackListener"/>).
+        /// </remarks>
+        internal bool PushedByCaller { get; }
+
         [SpecialName, PropertyMethod]
         public object Getf_trace() {
             if (_traceAdapter != null) {

src/core/IronPython/Runtime/PythonTracebackListener.cs

@@ -101,16 +101,25 @@ public void OnTraceEvent(Debugging.TraceEventKind kind, string name, string sour
 
                     var properties = (PythonDebuggingPayload)customPayload;
 
-                    // push the new frame
+                    // Module code has no frame push of its own: when exec or a script runner runs it, the caller has just pushed an entry
+                    // for it. That entry gets the new frame, rather than a second entry being pushed for the same code.
+                    bool pushedByCaller = IsPushedByCaller(thread, properties.Code);
+
                     pyFrame = new TraceBackFrame(
                         this,
                         properties.Code,
-                        thread.Count == 0 ? null : thread[thread.Count - 1].Frame,
+                        GetFrame(thread, pushedByCaller ? thread.Count - 2 : thread.Count - 1),
                         properties,
-                        scopeCallback
+                        scopeCallback,
+                        pushedByCaller
                     );
 
-                    thread.Add(new FunctionStack(pyFrame));
+                    if (pushedByCaller) {
+                        FunctionStack entry = thread[thread.Count - 1];
+                        thread[thread.Count - 1] = new FunctionStack(entry.Context, entry.Code, pyFrame);
+                    } else {
+                        thread.Add(new FunctionStack(pyFrame));
+                    }
 
                     if (traceDispatchObject == null) {
                         return;
@@ -140,8 +149,9 @@ public void OnTraceEvent(Debugging.TraceEventKind kind, string name, string sour
                 }
             } finally {
                 if (kind == Debugging.TraceEventKind.FrameExit && thread.Count > 0) {
-                    // don't pop frames we didn't push
-                    if (thread[thread.Count - 1].Code == ((PythonDebuggingPayload)customPayload).Code) {
+                    // don't pop frames we didn't push, nor an entry that the caller of module code pushed (the caller pops it)
+                    FunctionStack top = thread[thread.Count - 1];
+                    if (top.Code == ((PythonDebuggingPayload)customPayload).Code && top.Frame is not { PushedByCaller: true }) {
                         thread.RemoveAt(thread.Count - 1);
                     }
                 }
@@ -150,6 +160,39 @@ public void OnTraceEvent(Debugging.TraceEventKind kind, string name, string sour
 
         #endregion
 
+
+        /// <summary>
+        ///   Determines whether the entry on top of the frame stack was pushed for <paramref name="code"/> by the caller of that code.
+        /// </summary>
+        /// <remarks>
+        ///   Module code has no frame push of its own, so <c>exec</c> and the script runners push an entry for it just before running it.
+        ///   No frame object has been created for that entry yet, since no code has run in between.
+        /// </remarks>
+        private static bool IsPushedByCaller(List<FunctionStack> thread, FunctionCode code) {
+            if (!code.IsModule || thread.Count == 0) {
+                return false;
+            }
+
+            FunctionStack top = thread[thread.Count - 1];
+            return top.Code == code && top.Frame is null;
+        }
+
+
+        /// <summary>
+        ///   Gets the frame object of the frame stack entry at <paramref name="index"/>, creating it if needed.
+        /// </summary>
+        /// <remarks>
+        ///   An entry pushed by code that is not traced has no frame object until one is asked for.
+        ///   Creating it here gives the frame of traced code called from there its <see cref="TraceBackFrame.f_back"/>.
+        /// </remarks>
+        private static TraceBackFrame GetFrame(List<FunctionStack> thread, int index) {
+            if (index < 0) {
+                return null;
+            }
+
+            return thread[index].Frame ?? SysModule._getframeImpl(thread[index].Context, thread.Count - 1 - index, thread);
+        }
+
 
         private void DispatchTrace(List<FunctionStack> thread, Debugging.TraceEventKind kind, object payload, TracebackDelegate traceDispatch, object traceDispatchObject, TraceBackFrame pyFrame) {
             object args = null;

tests/suite/modules/system_related/test_sys.py

@@ -249,7 +249,7 @@ def f():
 sys.settrace(None)
 print(('call', 'f') in events)
 """
-        # Before the fix, defining f under -X:Debug gave it its normal (untraced) delegate, even with tracing on.
+        # Before PR #2118, defining f under -X:Debug gave it its normal (untraced) delegate, even with tracing on.
         out = subprocess.check_output([sys.executable, '-X:Debug', '-c', code])
         self.assertEqual(out.strip(), b'True')
 
@@ -299,6 +299,36 @@ def other():
             thread.join()
         self.assertEqual(events, [('call', 'f'), ('line', 'f'), ('call', 'stop'), ('line', 'stop')])
 
+    def test_settrace_f_back(self):
+        # The frame of a traced function called from code that is not traced has the frame of that code as f_back
+        def g():
+            return sys._getframe(0).f_back
+
+        sys.settrace(lambda *args: None)
+        try:
+            back = g()
+        finally:
+            sys.settrace(None)
+        self.assertIs(back, sys._getframe(0))
+
+    def test_settrace_exec_f_back(self):
+        # Code run by exec while tracing is on has a single frame, whose f_back is the frame that called exec
+        ns = {'sys': sys}
+        sys.settrace(lambda *args: None)
+        try:
+            exec("back = sys._getframe(0).f_back", ns)
+        finally:
+            sys.settrace(None)
+        self.assertIs(ns['back'], sys._getframe(0))
+
+    @skipUnlessIronPython()
+    def test_settrace_Xtracing_module_frame(self):
+        # With -X:Tracing, the main module has a single frame
+        import subprocess
+        code = "import traceback\ndef g():\n    return [entry[2] for entry in traceback.extract_stack()]\nprint(g())\n"
+        out = subprocess.check_output([sys.executable, '-X:Tracing', '-c', code])
+        self.assertEqual(out.strip(), b"['<module>', 'g']")
+
     @skipUnlessIronPython()
     @unittest.expectedFailure # dummy getrefcount removed in https://github.com/IronLanguages/ironpython3/pull/577
     def test_getrefcount(self):