diff --git a/CHANGELOG.md b/CHANGELOG.md index 87738d67..e0533822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,29 @@ hashes are given where a change is easiest to understand by reading the commit. ### JIT v2 +- **FPU integer conversions are compiled inline.** `CVT.W/L.S/D`, + `ROUND/TRUNC/CEIL/FLOOR.W/L.S/D` and the reverse `CVT.S/D.W/L` each emitted + a `call_indirect` into the interpreter's own conversion helper, so a guest + conversion cost 17-19 ns against 3.1 ns for the already-inline `cvt.d.s` + widening (and 8.9 ns for `cvt.s.d`, which still calls out). The common + case now compiles to a rounding step, the saturation cast and one Cause + store: RN-even rounding for the two dynamic-rounding instructions is an + add/subtract of a format-sized power of two in the source's sign direction + (only under `FCSR.RM == 0`, which is the only case where host MXCSR is + known to hold nearest-even), the fixed-mode instructions keep the existing + mode-independent bit manipulation. NaN, infinities, out-of-range values, + every other rounding mode and every Invalid/Inexact trap still branch to + the unchanged callout, so the exception model is the interpreter's own. + Per compiled instruction on an R4400 with `--cache shadow`: `cvt.w.s` + 17.5 -> 6.4 ns, `cvt.l.d` 18.5 -> 6.7, `cvt.w.d` 18.6 -> 7.0, + `cvt.l.s` 18.6 -> 6.1, `trunc.w.s` 17.1 -> 8.9, `cvt.s.w` 8.6 -> 5.0, + `cvt.d.w` 7.2 -> 4.7, `cvt.s.l` 8.2 -> 5.1, `cvt.d.l` 7.7 -> 4.9; + the untouched `cvt.s.d` control is unchanged at 8.7-8.9. The equivalence + suite pins the whole core snapshot, `fpu_fcsr` included, across the range + edges, all four rounding modes, both FR modes and the enable-bit/trap + paths. (`src/cpu/jitv2/codegen.rs`) (Motivation and the A/B kernel method: + ranked recommendation 5 of + [Jupiter Belic's irix7/iris performance study](https://github.com/irix7/iris/blob/510adc5/clanker/perf/mem-and-fpu-hot-paths.md).) - **FP arithmetic skips the exception model for ordinary operands.** A guest `fadd.s` in a jitv2 build emitted about 106 x86 instructions for the model around one `vaddss`. Two ordinary finite operands — a biased exponent that diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index e74b198c..a31d4ff3 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -8229,24 +8229,211 @@ fn emit_round_to_int_mode(builder: &mut FunctionBuilder, x: Value, rm: Value) -> #[derive(Clone, Copy)] enum RoundMode { Fixed(i64), Dynamic } +/// Clear FCSR's Cause field, skipping the store entirely when it is already +/// clear — the same load/test/cold-store shape `emit_fpu_update_fcsr_no_flags` +/// uses, and observationally identical to the interpreter's unconditional +/// `fpu_fcsr &= !FCSR_CM` (writing back the value already in memory has no +/// guest-visible effect). Leaves `builder` in a fresh sealed block. +fn emit_fpu_clear_cause_if_set(ctx: &mut EmitCtx) { + let mem = MemFlagsData::trusted(); + let i32t = ir::types::I32; + const FCSR_CM: i64 = 0x0001_f000; + + let fcsr_off = ir::immediates::Offset32::new(core_offset_of_fpu_fcsr()); + let fcsr = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let cause = ctx.builder.ins().band_imm_s(fcsr, FCSR_CM); + let cause_set = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, cause, 0); + + let clear_block = ctx.builder.create_block(); + let continue_block = ctx.builder.create_block(); + ctx.builder.ins().brif(cause_set, clear_block, &[], continue_block, &[]); + + ctx.builder.switch_to_block(clear_block); + ctx.builder.set_cold_block(clear_block); + ctx.builder.seal_block(clear_block); + let cleared = ctx.builder.ins().band_imm_s(fcsr, !FCSR_CM); + ctx.builder.ins().store(mem, cleared, ctx.core_ptr, fcsr_off); + ctx.builder.ins().jump(continue_block, &[]); + + ctx.builder.switch_to_block(continue_block); + ctx.builder.seal_block(continue_block); +} + +/// The exact inclusive/exclusive bounds `cvt_f32/64_to_int_by_value` tests an +/// already-rounded value against for a W or L destination: `>= MIN` and +/// `<= MAX` for W (both bounds exact as f32/f64 — note `i32::MAX as f32` +/// rounds *up* to 2^31, so 2^31 counts as in range for an F32 source and the +/// saturating cast below reproduces the interpreter's i32::MAX there), and +/// `>= -2^63 && < 2^63` for L (2^63 is exact in both formats and is the +/// first value that does not fit). Returned as `(min, max, max_inclusive)`. +fn cvt_int_bounds(ctx: &mut EmitCtx, src_f64: bool, dst_i64: bool) -> (Value, Value, bool) { + if src_f64 { + if dst_i64 { + let min = ctx.builder.ins().f64const(-(9223372036854775808.0f64)); + let max = ctx.builder.ins().f64const(9223372036854775808.0f64); + (min, max, false) + } else { + let min = ctx.builder.ins().f64const(i32::MIN as f64); + let max = ctx.builder.ins().f64const(i32::MAX as f64); + (min, max, true) + } + } else if dst_i64 { + let min = ctx.builder.ins().f32const(-(9223372036854775808.0f32)); + let max = ctx.builder.ins().f32const(9223372036854775808.0f32); + (min, max, false) + } else { + let min = ctx.builder.ins().f32const(i32::MIN as f32); + let max = ctx.builder.ins().f32const(i32::MAX as f32); + (min, max, true) + } +} + /// Shared body for every float-source rounding conversion (ROUND/TRUNC/ -/// CEIL/FLOOR/CVT.W/CVT.L, source format S or D, dest width W(i32) or -/// L(i64)). Unlike the old Cranelift-IR-side rounding/saturation, this is a -/// single `call_indirect` into `core.fpu_cvt_to_int_fn` -/// (`mips_exec.rs::jit_cvt_to_int`, which calls the same -/// `cvt_to_int_and_commit` the interpreter's `fpu_cvt_to_int` calls) — no -/// Cranelift-side rounding math, no host MXCSR/FPSR read at all. See -/// `cvt_to_int_and_commit`'s doc comment for the full rationale: reading -/// host FP status after a hardware conversion instruction races the flag -/// write against the read on out-of-order hardware (found live via this -/// file's own equivalence tests), and this also fixes the result itself to -/// real MIPS saturation (always the largest-magnitude representable -/// integer, e.g. 0x7FFFFFFF for a negative overflow too) instead of -/// Rust/Cranelift's own saturate-toward-sign convention. The function reads/ -/// writes `core.fpr`/`core.fpu_fcsr` directly, so nothing computed here -/// needs to survive across the call as a cached `Value` — only the plain -/// scalar register indices and mode flags are passed in. +/// CEIL/FLOOR/CVT.W/CVT.L, source format S or D, dest width W(i32) or L(i64)). +/// +/// **Inlined fast path.** For a finite source that rounds to a value inside +/// the destination's range, the interpreter's `cvt_to_int_and_commit` reduces +/// to "cast the rounded value to int, clear Cause, write the destination": +/// +/// * Saturation is unreachable: the range test is the interpreter's own, so +/// the cast operates on a value that fits — with the single deliberate +/// exception of an F32 source at exactly +2^31, which +/// `cvt_f32_to_int_by_value` also admits and both sides answer with +/// i32::MAX (`fcvt_to_sint_sat`'s saturating cast, same as Rust's `as`). +/// * Inexact is unreachable in range. `x` is integral after rounding (either +/// the dynamic-mode magic-number pair or `emit_round_to_int_mode` below), +/// hence exactly representable in its own format, so the interpreter's +/// round-trip test `(result as f32/f64) != x` can never fire (including the +/// ±2^31 edge, where the *integer* round trip `x as i32 as i64` and the +/// RN-even float cast back agree). +/// +/// Everything else — NaN, ±infinity, out-of-range finite values, and every +/// Invalid/Cause/Flag/trap consequence — is *not* proven here and branches to +/// the original `core.fpu_cvt_to_int_fn` callout, unchanged: `jit_cvt_to_int` +/// runs the very `cvt_to_int_and_commit` the interpreter calls, so the cold +/// path's model is identical by construction rather than by reimplementation. +/// The callout is entered before either path has written any guest state. +/// +/// `fcvt_to_sint_sat` (not a bare `fcvt_to_sint`) because the value is +/// integral and in range: the truncating saturation semantics are exact +/// there, and — unlike a hardware conversion followed by a host MXCSR status +/// read, the hazard `cvt_to_int_and_commit`'s doc comment describes — no +/// host FP status is read. +/// +/// **Rounding.** The two dynamic-RM converts (plain CVT.W/CVT.L) get a +/// cheap RN-even rounding when FCSR.RM is 0: add and subtract a +/// format-appropriate power of two in the source's sign direction +/// (`x + copysign(2^23|2^52, x) - copysign(...)`), which forces the sum's +/// ulp to 1 and therefore rounds `x` to the nearest integral value, ties to +/// even — the exact operation `round_f32/64_to_int_mode` performs for +/// RM_NEAREST_EVEN, for every finite magnitude including the `[0.5, 1)` +/// ties and values already >= 2^mantissa_bits. It relies on host FP add/sub +/// rounding to nearest-even, which is exactly what MXCSR holds whenever +/// FCSR.RM is 0 (`emit_fpu_set_mode` syncs MXCSR from FCSR), so — as in +/// `emit_fcvt_from_int` — every other rounding mode takes the callout. The +/// fixed ROUND/TRUNC/CEIL/FLOOR variants keep the mode-independent +/// `emit_round_to_int_mode` bit manipulation instead: their mode is a +/// compile-time constant, so it is (measured) cheaper than a gate plus the +/// magic-number pair, and they are never emitted by compiled C. fn emit_fcvt_to_int(ctx: &mut EmitCtx, fr_mode: FrMode, src_f64: bool, dst_i64: bool, mode: RoundMode) { + use cranelift_codegen::ir::condcodes::FloatCC; + let fs = field_rd(ctx.raw); + let fd = field_sa(ctx.raw); + let mem = MemFlagsData::trusted(); + let i32t = ir::types::I32; + let float_ty = if src_f64 { ir::types::F64 } else { ir::types::F32 }; + let int_ty = if dst_i64 { ir::types::I64 } else { i32t }; + let fcsr_off = ir::immediates::Offset32::new(core_offset_of_fpu_fcsr()); + + let src_bits = if src_f64 { + emit_read_fpr_l(ctx, fs, fr_mode) + } else { + emit_read_fpr_w(ctx, fs, fr_mode) + }; + let x = ctx.builder.ins().bitcast(float_ty, MemFlagsData::new(), src_bits); + + let fast_block = ctx.builder.create_block(); + let invalid_block = ctx.builder.create_block(); + let continue_block = ctx.builder.create_block(); + + let rounded = match mode { + RoundMode::Fixed(rm) => { + let rm = ctx.builder.ins().iconst(ir::types::I8, rm); + emit_round_to_int_mode(ctx.builder, x, rm) + } + RoundMode::Dynamic => { + let fcsr = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let rm = ctx.builder.ins().band_imm_s(fcsr, 0x3); + let rn_even = ctx.builder.ins().icmp_imm_s(IntCC::Equal, rm, 0); + let round_block = ctx.builder.create_block(); + ctx.builder.ins().brif(rn_even, round_block, &[], invalid_block, &[]); + + ctx.builder.switch_to_block(round_block); + ctx.builder.seal_block(round_block); + // |x| < 2^mantissa_bits: force the sum's ulp to 1 and round to + // nearest-even. At or above it every value is already integral, + // so `x` itself is the answer — and using the trick there would + // be wrong, because the biased sum crosses into the next binade + // where the ulp is 2 (found by `cvt_to_int_range_edges_...`: + // 2^23+1 came back as 2^23). + let magic = if src_f64 { + ctx.builder.ins().f64const(4503599627370496.0) // 2^52 + } else { + ctx.builder.ins().f32const(8388608.0) // 2^23 + }; + let abs_x = ctx.builder.ins().fabs(x); + let small = ctx.builder.ins().fcmp(FloatCC::LessThan, abs_x, magic); + let signed_magic = ctx.builder.ins().fcopysign(magic, x); + let biased = ctx.builder.ins().fadd(x, signed_magic); + let rounded_small = ctx.builder.ins().fsub(biased, signed_magic); + ctx.builder.ins().select(small, rounded_small, x) + } + }; + + let (min, max, max_inclusive) = cvt_int_bounds(ctx, src_f64, dst_i64); + let ge_min = ctx.builder.ins().fcmp(FloatCC::GreaterThanOrEqual, rounded, min); + let le_max = if max_inclusive { + ctx.builder.ins().fcmp(FloatCC::LessThanOrEqual, rounded, max) + } else { + ctx.builder.ins().fcmp(FloatCC::LessThan, rounded, max) + }; + // NaN fails both comparisons, so it takes the callout path with every + // other not-provably-in-range value. + let in_range = ctx.builder.ins().band(ge_min, le_max); + ctx.builder.ins().brif(in_range, fast_block, &[], invalid_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + let result = ctx.builder.ins().fcvt_to_sint_sat(int_ty, rounded); + emit_fpu_clear_cause_if_set(ctx); + if dst_i64 { + emit_write_fpr_l(ctx, fd, result, fr_mode); + } else { + emit_write_fpr_w(ctx, fd, result, fr_mode); + } + ctx.builder.ins().jump(continue_block, &[]); + + // Cold: NaN / infinity / out-of-range, plus (for the dynamic modes) every + // rounding mode other than RN-even. Not proven inline — see this + // function's doc comment. + ctx.builder.switch_to_block(invalid_block); + ctx.builder.set_cold_block(invalid_block); + ctx.builder.seal_block(invalid_block); + emit_fcvt_to_int_callout(ctx, fr_mode, src_f64, dst_i64, mode); + ctx.builder.ins().jump(continue_block, &[]); + + ctx.builder.switch_to_block(continue_block); + ctx.builder.seal_block(continue_block); +} + +/// The original callout form of [`emit_fcvt_to_int`]: one `call_indirect` +/// into `core.fpu_cvt_to_int_fn` (`mips_exec.rs::jit_cvt_to_int`, which calls +/// the same `cvt_to_int_and_commit` the interpreter's `fpu_cvt_to_int` calls) +/// and `EXC_FPE` if it reports a trap. Still every not-provably-inline case — +/// NaN, ±infinity and out-of-range for all twenty conversion instructions, +/// and every non-RN-even rounding mode for the two dynamic ones — so those +/// consequences are the interpreter's own model by construction. +fn emit_fcvt_to_int_callout(ctx: &mut EmitCtx, fr_mode: FrMode, src_f64: bool, dst_i64: bool, mode: RoundMode) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let mem = MemFlagsData::trusted(); @@ -8297,16 +8484,121 @@ fn emit_fcvt_to_int(ctx: &mut EmitCtx, fr_mode: FrMode, src_f64: bool, dst_i64: emit_trap_if_nonzero(ctx, trapped); } -/// Shared body for int-source-to-float conversions (CVT.S.W/D.W/S.L/D.L): -/// a single `call_indirect` into `core.fpu_cvt_int_to_float_fn` -/// (`mips_exec.rs::jit_cvt_int_to_float`, calling the same -/// `cvt_int_to_float_and_commit` the interpreter's `fpu_cvt_int_to_float` -/// calls) — no Cranelift-side `fcvt_from_sint` + host MXCSR/FPSR read, same -/// race-avoidance shape as `emit_fcvt_to_int`/`cvt_to_int_and_commit`. +/// Shared body for int-source-to-float conversions (CVT.S.W/D.W/S.L/D.L). +/// +/// **Inlined fast path, gated on host rounding mode.** `cvt_int_to_float_and_commit` +/// is small by value — convert, write the destination, then report Inexact +/// iff converting back changes the value — but its `src as f32/f64` is a +/// round-to-nearest-even conversion in the *interpreter*, while the JIT's +/// `fcvt_from_sint` lowers to `cvtsi2ss`/`cvtsi2sd`, which round per the +/// host MXCSR. The JIT syncs MXCSR from FCSR.RM on every FCSR write +/// (`emit_fpu_set_mode`), so the host mode is only known to be RN-even when +/// `FCSR.RM == 0`; every other mode takes the callout, which computes by +/// value and is therefore unconditionally correct. Under RM == 0 the two +/// agree exactly: both round to nearest, ties to even. +/// +/// The Inexact test is the interpreter's own `(result as i64) != src`, +/// reproduced as a truncating saturating cast back to i64 (exact for the +/// integral result, and saturating at ±2^63 exactly as Rust's `as` does). +/// Note the ordering this preserves from `cvt_int_to_float_and_commit`: the +/// destination is written *before* the exception check, so a trapping +/// conversion still leaves the converted value in the FPR. fn emit_fcvt_from_int(ctx: &mut EmitCtx, fr_mode: FrMode, src_i64: bool, dst_f64: bool) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let mem = MemFlagsData::trusted(); + let i32t = ir::types::I32; + let i64t = ir::types::I64; + let float_ty = if dst_f64 { ir::types::F64 } else { ir::types::F32 }; + let bits_ty = if dst_f64 { ir::types::I64 } else { i32t }; + const FCSR_CM: i64 = 0x0001_f000; + const FCSR_CI: i64 = 0x0000_1000; + const FCSR_IM: i64 = 0x0000_0080; + const FCSR_FI: i64 = 0x0000_0004; + + let fcsr_off = ir::immediates::Offset32::new(core_offset_of_fpu_fcsr()); + let fcsr0 = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let rm = ctx.builder.ins().band_imm_s(fcsr0, 0x3); + let rn_even = ctx.builder.ins().icmp_imm_s(IntCC::Equal, rm, 0); + + let inline_block = ctx.builder.create_block(); + let callout_block = ctx.builder.create_block(); + let continue_block = ctx.builder.create_block(); + ctx.builder.ins().brif(rn_even, inline_block, &[], callout_block, &[]); + + ctx.builder.switch_to_block(inline_block); + ctx.builder.seal_block(inline_block); + // src: W sign-extends from 32 bits, exactly like `read_fpr_w(..) as i32 + // as i64` in cvt_int_to_float_and_commit. + let src = if src_i64 { + emit_read_fpr_l(ctx, fs, fr_mode) + } else { + let w = emit_read_fpr_w(ctx, fs, fr_mode); + ctx.builder.ins().sextend(i64t, w) + }; + let result = ctx.builder.ins().fcvt_from_sint(float_ty, src); + let bits = ctx.builder.ins().bitcast(bits_ty, MemFlagsData::new(), result); + // Destination write happens first — including on the trapping path below. + if dst_f64 { + emit_write_fpr_l(ctx, fd, bits, fr_mode); + } else { + emit_write_fpr_w(ctx, fd, bits, fr_mode); + } + let back = ctx.builder.ins().fcvt_to_sint_sat(i64t, result); + let inexact = ctx.builder.ins().icmp(IntCC::NotEqual, back, src); + + let inexact_block = ctx.builder.create_block(); + let exact_block = ctx.builder.create_block(); + ctx.builder.ins().brif(inexact, inexact_block, &[], exact_block, &[]); + + // Exact: only Cause has to be rewritten (to zero). + ctx.builder.switch_to_block(exact_block); + ctx.builder.seal_block(exact_block); + emit_fpu_clear_cause_if_set(ctx); + ctx.builder.ins().jump(continue_block, &[]); + + // Cold: the conversion lost precision. + ctx.builder.switch_to_block(inexact_block); + ctx.builder.set_cold_block(inexact_block); + ctx.builder.seal_block(inexact_block); + let fcsr1 = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let cleared = ctx.builder.ins().band_imm_s(fcsr1, !FCSR_CM); + let with_cause = ctx.builder.ins().bor_imm_s(cleared, FCSR_CI); + ctx.builder.ins().store(mem, with_cause, ctx.core_ptr, fcsr_off); + let im_set = ctx.builder.ins().band_imm_s(with_cause, FCSR_IM); + let im_set = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, im_set, 0); + let raise_block = ctx.builder.create_block(); + let flag_block = ctx.builder.create_block(); + ctx.builder.ins().brif(im_set, raise_block, &[], flag_block, &[]); + + ctx.builder.switch_to_block(raise_block); + ctx.builder.seal_block(raise_block); + let status = crate::cpu::mips_exec::exec_exception_const(crate::cpu::mips_exec::EXC_FPE); + let status_val = ctx.builder.ins().iconst(i32t, status as i64); + emit_exception_exit(ctx, status_val); + + ctx.builder.switch_to_block(flag_block); + ctx.builder.seal_block(flag_block); + let with_flag = ctx.builder.ins().bor_imm_s(with_cause, FCSR_FI); + ctx.builder.ins().store(mem, with_flag, ctx.core_ptr, fcsr_off); + ctx.builder.ins().jump(continue_block, &[]); + + ctx.builder.switch_to_block(callout_block); + ctx.builder.set_cold_block(callout_block); + ctx.builder.seal_block(callout_block); + emit_fcvt_from_int_callout(ctx, fr_mode, src_i64, dst_f64); + ctx.builder.ins().jump(continue_block, &[]); + + ctx.builder.switch_to_block(continue_block); + ctx.builder.seal_block(continue_block); +} + +/// The original callout form of [`emit_fcvt_from_int`] — still the whole +/// implementation for every non-RN-even rounding mode; see that function's +/// doc comment for why the host-mode gate exists. +fn emit_fcvt_from_int_callout(ctx: &mut EmitCtx, fr_mode: FrMode, src_i64: bool, dst_f64: bool) { + let fs = field_rd(ctx.raw); + let fd = field_sa(ctx.raw); let ptr_ty = ctx.module.target_config().pointer_type(); let i32t = ir::types::I32; @@ -8317,7 +8609,7 @@ fn emit_fcvt_from_int(ctx: &mut EmitCtx, fr_mode: FrMode, src_i64: bool, dst_f64 let dst_f64_val = ctx.builder.ins().iconst(i32t, dst_f64 as i64); // arg 0 is `ctx.core_ptr`; the helper recovers its executor via - // `exec_from_core`. See `emit_fcvt_to_int`'s matching comment. + // `exec_from_core`. See `emit_fcvt_to_int_callout`'s matching comment. let callee = emit_hook_callee(ctx, core_offset_of_fpu_cvt_int_to_float_fn()); let mut sig = ctx.module.make_signature(); sig.params.push(AbiParam::new(ptr_ty)); diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 1c958612..d846b9e9 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -6220,6 +6220,189 @@ mod tests { cvt_from_int_case(crate::cpu::mips_isa::FUNCT_FCVT_D, crate::cpu::mips_isa::RS_L, -123456789); } + // ---- CP1 conversions: inline fast-path edge and flag coverage ------ + // + // The conversions used to be a callout into the interpreter's own + // `cvt_to_int_and_commit`/`cvt_int_to_float_and_commit`, so agreement was + // structural. `emit_fcvt_to_int`/`emit_fcvt_from_int` now inline the + // common path and keep the callout for what they cannot prove; these + // tests pin the *whole* core snapshot — `fpu_fcsr` included, with no + // flag mask — against the interpreter across the fast path, the callout + // fallbacks, both FR modes, all four FCSR rounding modes, and the + // enable-bit/trap paths. + // + // fs is rd=2 and fd is sa=4 in every encoding below, which addresses + // fpr[2]/fpr[4] identically in FR=0 (even slot, low word for W) and FR=1. + + /// Values that bracket every branch of the to-int path: in range, the + /// four boundary values the identical range test turns on (including + /// +2^31, which `i32::MAX as f32` rounds up to and which the interpreter + /// therefore admits), values just outside them, subnormals, NaN and both + /// infinities. + const CVT_TO_INT_EDGES: &[f64] = &[ + 0.0, -0.0, 0.25, -0.25, 0.5, -0.5, 0.999_999_9, -0.999_999_9, + 1.0, -1.0, 1.5, -1.5, 2.5, -2.5, 3.5, -3.5, 3.7, -3.7, + 8_388_608.0, 8_388_609.0, 16_777_216.0, 16_777_217.0, + 2_147_483_520.0, 2_147_483_647.0, 2_147_483_648.0, + -2_147_483_648.0, -2_147_483_649.0, + 1.0e20, -1.0e20, 9_223_372_036_854_775_808.0, -9_223_372_036_854_775_808.0, + 9_223_372_036_854_774_784.0, -9_223_372_036_854_774_784.0, + 1.4e-45, -1.4e-45, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, + ]; + + fn cvt_to_int_exact_case(funct: u32, fmt: u32, fs_val: f64, fr1: bool, fcsr: Option) { + let mut fpr = [0u64; 32]; + fpr[2] = if fmt == crate::cpu::mips_isa::RS_S { + (fs_val as f32).to_bits() as u64 + } else { + fs_val.to_bits() + }; + // Pre-existing destination: a trapping conversion must leave it + // untouched, so it must not look like a computed result. + fpr[4] = 0xDEAD_BEEF_DEAD_BEEF; + let instr = make_r(crate::cpu::mips_isa::OP_COP1, fmt, 0, 2, 4, funct); + assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, fr1, fcsr); + } + + #[test] + fn cvt_to_int_range_edges_match_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FCVT_L, FUNCT_FCVT_W, RS_D, RS_S}; + for &v in CVT_TO_INT_EDGES { + for fr1 in [false, true] { + for fmt in [RS_S, RS_D] { + cvt_to_int_exact_case(FUNCT_FCVT_W, fmt, v, fr1, None); + cvt_to_int_exact_case(FUNCT_FCVT_L, fmt, v, fr1, None); + } + } + } + } + + /// ROUND/TRUNC/CEIL/FLOOR pass a fixed rounding mode, so they share the + /// inline body but not the FCSR-dynamic mode selection; a half-way value + /// distinguishes all four. + #[test] + fn round_trunc_ceil_floor_edges_match_interpreter() { + use crate::cpu::mips_isa::*; + for &v in &[0.5, -0.5, 1.5, -1.5, 2.5, -2.5, 3.7, -3.7, 0.0, -0.0, 1.0e20, -1.0e20, f64::NAN] { + for fr1 in [false, true] { + for fmt in [RS_S, RS_D] { + for funct in [FUNCT_FROUND_W, FUNCT_FTRUNC_W, FUNCT_FCEIL_W, FUNCT_FFLOOR_W] { + cvt_to_int_exact_case(funct, fmt, v, fr1, None); + } + for funct in [FUNCT_FROUND_L, FUNCT_FTRUNC_L, FUNCT_FCEIL_L, FUNCT_FFLOOR_L] { + cvt_to_int_exact_case(funct, fmt, v, fr1, None); + } + } + } + } + } + + /// FCSR.RM drives CVT.W/CVT.L directly; all four modes must produce the + /// interpreter's result and Cause/Flag bits from the inline rounding. + #[test] + fn cvt_to_int_all_rounding_modes_match_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FCVT_L, FUNCT_FCVT_W, RS_D, RS_S}; + for rm in 0u32..4 { + for &v in &[0.5, -0.5, 1.5, -1.5, 2.5, -2.5, 3.7, -3.7, -0.7, 1.0e10, -1.0e10] { + for fr1 in [false, true] { + for fmt in [RS_S, RS_D] { + cvt_to_int_exact_case(FUNCT_FCVT_W, fmt, v, fr1, Some(rm)); + cvt_to_int_exact_case(FUNCT_FCVT_L, fmt, v, fr1, Some(rm)); + } + } + } + } + } + + /// Invalid with the destination pre-set: Flag.V when EV is clear, a trap + /// with the destination untouched when EV is set. The latter is the + /// callout path, the former is too (NaN/inf/out-of-range never inline). + #[test] + fn cvt_to_int_invalid_enable_bits_match_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FCVT_L, FUNCT_FCVT_W, RS_D, RS_S}; + for &v in &[f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 1.0e20, -1.0e20] { + for &fcsr in &[0x0000_0000u32, 0x0000_0800, 0x0000_0880, 0x0000_0004] { + for fmt in [RS_S, RS_D] { + cvt_to_int_exact_case(FUNCT_FCVT_W, fmt, v, true, Some(fcsr)); + cvt_to_int_exact_case(FUNCT_FCVT_L, fmt, v, true, Some(fcsr)); + } + } + } + } + + /// The same snapshot comparison for the to-int fast path with a dirty + /// FCSR: a stale sticky Flag set and a stale Cause set must be handled + /// (Flag preserved, Cause rewritten) exactly as the interpreter does. + #[test] + fn cvt_to_int_dirty_fcsr_matches_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FCVT_L, FUNCT_FCVT_W, RS_D, RS_S}; + for &fcsr in &[0x0000_1004u32, 0x0000_007cu32, 0x0000_1000u32] { + for &v in &[1.5, -1.5, 42.0, -42.0] { + for fmt in [RS_S, RS_D] { + cvt_to_int_exact_case(FUNCT_FCVT_W, fmt, v, true, Some(fcsr)); + cvt_to_int_exact_case(FUNCT_FCVT_L, fmt, v, true, Some(fcsr)); + } + } + } + } + + fn cvt_from_int_exact_case(funct: u32, fmt: u32, val: i64, fr1: bool, fcsr: Option) { + let mut fpr = [0u64; 32]; + fpr[2] = if fmt == crate::cpu::mips_isa::RS_W { + (val as i32 as u32) as u64 + } else { + val as u64 + }; + fpr[4] = 0xDEAD_BEEF_DEAD_BEEF; + let instr = make_r(crate::cpu::mips_isa::OP_COP1, fmt, 0, 2, 4, funct); + assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, fr1, fcsr); + } + + /// Int→float: exact and inexact values in both widths, both FR modes, and + /// every FCSR rounding mode (non-RN takes the callout; RN-even is the + /// inline `fcvt_from_sint` path whose host `cvtsi2ss`/`cvtsi2sd` rounding + /// is only equivalent under MXCSR's RN-even mode). + #[test] + fn cvt_from_int_edges_match_interpreter() { + use crate::cpu::mips_isa::*; + let vals: &[i64] = &[ + 0, 1, -1, 42, -42, 16_777_215, 16_777_216, 16_777_217, + 2_147_483_647, -2_147_483_648, 2_147_483_648, + 9_007_199_254_740_991, 9_007_199_254_740_993, + i64::MAX, i64::MIN, 123_456_789, -123_456_789, + ]; + for &v in vals { + for fr1 in [false, true] { + for rm in [0u32, 1, 2, 3] { + for (funct, fmt) in [ + (FUNCT_FCVT_S, RS_W), (FUNCT_FCVT_D, RS_W), + (FUNCT_FCVT_S, RS_L), (FUNCT_FCVT_D, RS_L), + ] { + cvt_from_int_exact_case(funct, fmt, v, fr1, Some(rm)); + } + } + } + } + } + + /// Inexact with Enable.I set must trap with the destination already + /// written (unlike the to-int direction), and with it clear must set the + /// sticky Flag.I; a dirty Flag must be preserved either way. + #[test] + fn cvt_from_int_inexact_enable_bits_match_interpreter() { + use crate::cpu::mips_isa::*; + for &fcsr in &[0x0000_0000u32, 0x0000_0080, 0x0000_0100, 0x0000_1004, 0x0000_007c] { + for &v in &[9_007_199_254_740_993i64, i64::MAX, 2_147_483_647, -2_147_483_648] { + for fr1 in [false, true] { + cvt_from_int_exact_case(FUNCT_FCVT_S, RS_W, v, fr1, Some(fcsr)); + cvt_from_int_exact_case(FUNCT_FCVT_D, RS_W, v, fr1, Some(fcsr)); + cvt_from_int_exact_case(FUNCT_FCVT_S, RS_L, v, fr1, Some(fcsr)); + cvt_from_int_exact_case(FUNCT_FCVT_D, RS_L, v, fr1, Some(fcsr)); + } + } + } + } + // ---- CP1: batch F4 compares — C.cond.fmt ------ fn fcc_case(funct: u32, fmt: u32, cc: u32, fs_val: f64, ft_val: f64) { @@ -7610,4 +7793,90 @@ mod tests { // bne zero, zero never taken: lands on word 2 after the slot. busy_slot_case(make_i(crate::cpu::mips_isa::OP_BNE, 0, 0, 0x40), [0u64; 32], 0xFFFF_FFFF_8000_0008); } + + // ---- CP1 conversion cost probe (diagnostic, not a correctness gate) ---- + // + // Page layout and the compiled-region shape make the guest benchmark + // kernels noisy for one-instruction families, so the per-instruction cost + // is measured here instead: one region of `N` conversions (destination + // registers rotated so the chain is not a single store-forward), called + // `REPS` times through the real `JitFn`. The region-entry/exit cost is + // common to both builds and identical for every instruction below, so the + // A/B ratio is the instruction's own cost. `cargo test --release -p iris + // --features lightning,jitv2 --lib cvt_perf_probe -- --ignored --nocapture` + #[test] + #[ignore] + fn cvt_perf_probe() { + use crate::cpu::mips_isa::*; + const N: usize = 128; + const REPS: usize = 20_000; + + fn probe(instrs: &[u32], fr1: bool) -> f64 { + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[..instrs.len()].copy_from_slice(instrs); + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, 0, 0, ENTRIES_PER_PAGE as usize); + assert!(non_empty, "probe region must not be empty"); + let mut region = *walked; + let mut codegen = Codegen::new(); + let jit_fn: JitFn = codegen + .compile_region(&mut region, 0, fr1, false) + .expect("probe instruction must be compilable"); + + let mut fpr = [0u64; 32]; + for (i, slot) in fpr.iter_mut().enumerate() { + // Small integer bit patterns: as an L/W source they convert + // exactly (the common int->float case), and as an S/D source + // they are subnormals that still round to an in-range integer, + // so either direction stays on the fast path. + *slot = (i as u64) * 3; + } + let (mut exec, _mem) = fpu_seeded_executor([0u64; 32], fpr, 0xFFFF_FFFF_8000_1000, fr1); + exec.core.fpu_fcsr = 0; + exec.install_jit_hooks(); + let core_ptr = &mut exec.core as *mut MipsCore; + + // Warm up (first call faults in the code page). + for _ in 0..1_000 { unsafe { jit_fn(core_ptr) }; } + + let t0 = std::time::Instant::now(); + for _ in 0..REPS { unsafe { jit_fn(core_ptr) }; } + let ns = t0.elapsed().as_secs_f64() * 1e9 / (REPS * instrs.len()) as f64; + std::mem::forget(codegen); + ns + } + + fn fill(make: impl Fn(usize) -> u32) -> Vec { + (0..N).map(make).collect() + } + + // fd rotated over four registers; fs fixed (to-int) or rotated too + // (from-int, so the source values differ). + let fds = [4u32, 6, 8, 10]; + let cases: &[(&str, Vec)] = &[ + ("cvt.s.d", fill(|i| make_r(OP_COP1, RS_D, 0, 2, fds[i % 4], FUNCT_FCVT_S))), + ("cvt.d.s", fill(|i| make_r(OP_COP1, RS_S, 0, 2, fds[i % 4], FUNCT_FCVT_D))), + ("cvt.w.s", fill(|i| make_r(OP_COP1, RS_S, 0, 2, fds[i % 4], FUNCT_FCVT_W))), + ("cvt.l.d", fill(|i| make_r(OP_COP1, RS_D, 0, 2, fds[i % 4], FUNCT_FCVT_L))), + ("cvt.w.d", fill(|i| make_r(OP_COP1, RS_D, 0, 2, fds[i % 4], FUNCT_FCVT_W))), + ("cvt.l.s", fill(|i| make_r(OP_COP1, RS_S, 0, 2, fds[i % 4], FUNCT_FCVT_L))), + ("trunc.w.s", fill(|i| make_r(OP_COP1, RS_S, 0, 2, fds[i % 4], FUNCT_FTRUNC_W))), + ("cvt.s.w", fill(|i| make_r(OP_COP1, RS_W, 0, (2 + 2 * (i % 4)) as u32, 10, FUNCT_FCVT_S))), + ("cvt.d.w", fill(|i| make_r(OP_COP1, RS_W, 0, (2 + 2 * (i % 4)) as u32, 10, FUNCT_FCVT_D))), + ("cvt.s.l", fill(|i| make_r(OP_COP1, RS_L, 0, (2 + 2 * (i % 4)) as u32, 10, FUNCT_FCVT_S))), + ("cvt.d.l", fill(|i| make_r(OP_COP1, RS_L, 0, (2 + 2 * (i % 4)) as u32, 10, FUNCT_FCVT_D))), + ]; + + let mut best = std::collections::BTreeMap::new(); + for _ in 0..3 { + for (name, instrs) in cases { + let ns = probe(instrs, true); + let e = best.entry(*name).or_insert(f64::MAX); + if ns < *e { *e = ns; } + } + } + for (name, ns) in &best { + println!("{:<12} {:8.3} ns/instruction", name, ns); + } + } }