From 5020c04ac193a746126ad09064f66873c3f6a8e0 Mon Sep 17 00:00:00 2001 From: Jupiter Belic Date: Sun, 25 Oct 2026 20:56:35 +1000 Subject: [PATCH] jitv2: skip the FP exception model for ordinary operands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A guest `fadd.s` in a jitv2 build emitted about 106 x86 instructions around one `vaddss`. The cost was the model, not the arithmetic: two per-operand denormal/quiet-NaN checks, the by-value signalling-NaN flags, the result-denormal rule, and an FCSR update that stored Cause on every instruction. Classify the operands once instead. If both are ordinary finite numbers — biased exponent neither zero nor all ones — then no operand check can trap, no signalling NaN can be present, and (for DIV) the divisor cannot be zero, so the instruction provably raises no flag. That arm computes the result, applies the unchanged result-denormal rule, and clears Cause only when it is set: rewriting a Cause field that is already clear writes back the value already in memory, which the guest cannot observe, and skipping it also removes a store-to-load dependency through `core.fpu_fcsr` between consecutive FP instructions. Everything else — zero, denormal, infinity, NaN, and a negative square root — takes the previous body verbatim, out of line, so the exception behaviour is the same code as before. The interpreter is untouched. Measured with the isolated kernels from the study below (single-kernel images, R4400, `--cache shadow`, five interleaved runs per cell, median, spread under 2% except where noted): kernel before after gain fadd.s indep 8.16 4.25 1.92x fadd.s dep 7.25 4.84 1.50x fadd.d indep 8.21 4.08 2.01x fadd.d dep 8.16 4.27 1.91x fmul.s indep 8.18 4.41 1.86x fmul.s dep 7.73 4.83 1.60x fmul.d indep 8.16 3.99 2.04x fdiv.s indep 8.52 4.20 2.03x fdiv.s dep 8.45 7.22 1.17x fdiv.d indep 7.71 4.29 1.80x fsqrt.s indep 5.87 4.02 1.46x fsqrt.d indep 6.08 3.45 1.76x The pure-conversion kernels, which this change does not touch, stay put (cvt.s.d 4.04 -> 3.84, cvt.w.s 10.48 -> 11.08, cvt.l.d 11.29 -> 11.57), which is the control. Semantics: the jitv2 equivalence suite (JIT vs interpreter snapshots, FCSR included, plus a new test that seeds Cause and Flag and checks that ordinary operand arithmetic clears Cause and keeps the sticky flags), the FPU/FCSR cpu-tests under jitv2 (identical pass/fail set before and after: 2620 checks passed, 126 failed, same 29 test identities), and `cargo test --workspace`. Credit: Jupiter Belic and irix7/iris — "Guest memory and FPU hot paths: what is actually slow", https://github.com/irix7/iris/blob/510adc512dc59395ba3ba2a42426bda09b11b9e4/clanker/perf/mem-and-fpu-hot-paths.md ; the measurements, the ranked recommendation this implements and the isolated-kernel method are that fork's work. --- CHANGELOG.md | 16 ++ src/cpu/jitv2/codegen.rs | 308 ++++++++++++++++++++++++++++++------ src/cpu/jitv2/equiv_test.rs | 64 +++++++- 3 files changed, 340 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dab0d0d4..87738d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,22 @@ hashes are given where a change is easiest to understand by reading the commit. ## October 2026 +### JIT v2 + +- **FP arithmetic skips the exception model for ordinary operands.** A guest + `fadd.s` in a jitv2 build emitted about 106 x86 instructions for the model + around one `vaddss`. Two ordinary finite operands — a biased exponent that + is neither zero nor all ones — cannot be denormal or NaN, cannot be a + signalling NaN and cannot make DIV divide by zero, so the instruction + provably raises no flag: the two operand checks, the by-value flags and the + per-instruction Cause store are skipped, and Cause is still cleared + whenever it is set. Every special operand (zero, denormal, infinity, NaN, + and a negative square root) runs the previous code unchanged, out of line; + the interpreter is untouched. Isolated kernels on an R4400 with + `--cache shadow`: `fadd.s` 8.2 -> 4.3 ns/op, `fmul.s` 8.2 -> 4.4, + `fdiv.s` 8.5 -> 4.2, `fsqrt.d` 6.1 -> 3.5; conversion kernels, which this + change does not touch, are unchanged. (`src/cpu/jitv2/codegen.rs`) + ### CI and automation - **`iris-ci screenshot` produces the guest screen in `--ci`.** A `--ci` diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index 89f8ba05..e74b198c 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -5178,6 +5178,46 @@ fn emit_fpu_update_fcsr(ctx: &mut EmitCtx, flags: Value, write_result: impl Fn(& ctx.builder.seal_block(continue_block); } +/// `emit_fpu_update_fcsr` for an instruction whose caller has proved `flags` +/// is zero (see `emit_is_normal_s`/`_d`: ordinary finite operands cannot +/// raise Invalid or Divide-by-Zero, and this model raises no other flag for +/// ADD/SUB/MUL/DIV/SQRT). Cause must still end up clear — it reflects only +/// the last instruction — but the store is skipped when Cause is already +/// clear: writing back the value that is already in memory is not +/// observable to the guest, and skipping it keeps consecutive FP +/// instructions from serialising on a store-to-load dependency through +/// `core.fpu_fcsr`. Cause is clear in the steady state (every arithmetic +/// instruction either clears it here or set it from an instruction that has +/// already been followed by a clear), so the common path is one load, one +/// test and the destination write. +fn emit_fpu_update_fcsr_no_flags(ctx: &mut EmitCtx, write_result: impl Fn(&mut EmitCtx)) { + let mem = MemFlagsData::trusted(); + let i32t = ir::types::I32; + const FCSR_CM: i64 = 0x0001_f000; + + let fcsr_off = ir::immediates::Offset32::new(core_offset_of_fpu_fcsr()); + let fcsr = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let cause = ctx.builder.ins().band_imm_s(fcsr, FCSR_CM); + let cause_set = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, cause, 0); + + let clear_block = ctx.builder.create_block(); + let continue_block = ctx.builder.create_block(); + ctx.builder.ins().brif(cause_set, clear_block, &[], continue_block, &[]); + + // Cold: only the first arithmetic instruction after one that left Cause + // set comes here; every instruction that follows has Cause already clear. + ctx.builder.switch_to_block(clear_block); + ctx.builder.set_cold_block(clear_block); + ctx.builder.seal_block(clear_block); + let cleared = ctx.builder.ins().band_imm_s(fcsr, !FCSR_CM); + ctx.builder.ins().store(mem, cleared, ctx.core_ptr, fcsr_off); + ctx.builder.ins().jump(continue_block, &[]); + + ctx.builder.switch_to_block(continue_block); + ctx.builder.seal_block(continue_block); + write_result(ctx); +} + /// Unimplemented Operation (Cause.E): R4000/VR5000 manuals — no Enable or /// Flag bit, always traps, destination register untouched. Mirrors /// `MipsExecutor::fpu_unimplemented`. Terminates the current block; caller @@ -5385,6 +5425,25 @@ fn emit_is_subnormal_or_qnan_d(ctx: &mut EmitCtx, bits: Value) -> Value { b.ins().bor(is_subnormal, is_qnan) } +/// `bits` is an ordinary finite number of the given format: biased exponent +/// neither zero (zero or subnormal) nor all ones (infinity or NaN). Exactly +/// the operand class for which a computational instruction provably takes no +/// Unimplemented Operation (`emit_check_denorm_operand` is false for both +/// operand checks), has no signalling-NaN operand (`fpu_arith_flags_*` is +/// zero), and — for DIV — cannot divide by zero (the divisor is nonzero). +/// One unsigned range test on the exponent field: `e - 1 < 0xFE` in exponent +/// units, i.e. `(bits & EXP_MASK) - BIAS < MAX_EXP_BITS`. +fn emit_is_normal_s(ctx: &mut EmitCtx, bits: Value) -> Value { + let e = ctx.builder.ins().band_imm_s(bits, 0x7F80_0000); + let biased = ctx.builder.ins().iadd_imm_s(e, -(0x0080_0000i64)); + ctx.builder.ins().icmp_imm_s(IntCC::UnsignedLessThan, biased, 0x7F00_0000) +} +fn emit_is_normal_d(ctx: &mut EmitCtx, bits: Value) -> Value { + let e = ctx.builder.ins().band_imm_s(bits, 0x7FF0_0000_0000_0000u64 as i64); + let biased = ctx.builder.ins().iadd_imm_s(e, -(0x0010_0000_0000_0000i64)); + ctx.builder.ins().icmp_imm_s(IntCC::UnsignedLessThan, biased, 0x7FE0_0000_0000_0000u64 as i64) +} + /// Result-only half: subnormal-and-nonzero, no NaN check (a computed /// arithmetic result's denormal-ness is what finding 7's Table 9-1 gates on; /// zero itself, exponent==0 mantissa==0, is explicitly not denormal). @@ -5417,9 +5476,16 @@ fn emit_is_subnormal_d(ctx: &mut EmitCtx, bits: Value) -> Value { /// `write_result`/`write_zero` are each called on exactly one path — the /// real result when not flushing, a signed zero of the given width when /// flushing. +/// +/// `flags` is `Some` for the general case and `None` when the caller has +/// proved the instruction raises no flag (the fast path of +/// `emit_fbinop_s`/`_d` and `emit_fsqrt_s`/`_d`, whose operands passed +/// `emit_is_normal_s`/`_d`). `None` selects `emit_fpu_update_fcsr_no_flags` +/// for the not-denormal case; the denormal-result rule below is identical +/// either way because it ignores `flags` entirely. fn emit_fpu_update_fcsr_arith( ctx: &mut EmitCtx, - flags: Value, + flags: Option, result_is_denorm: Value, result_is_negative: Value, write_result: impl Fn(&mut EmitCtx), @@ -5471,7 +5537,10 @@ fn emit_fpu_update_fcsr_arith( ctx.builder.switch_to_block(normal_block); ctx.builder.seal_block(normal_block); - emit_fpu_update_fcsr(ctx, flags, &write_result); + match flags { + Some(flags) => emit_fpu_update_fcsr(ctx, flags, &write_result), + None => emit_fpu_update_fcsr_no_flags(ctx, &write_result), + } ctx.builder.ins().jump(continue_block, &[]); ctx.builder.switch_to_block(continue_block); @@ -7495,6 +7564,58 @@ type SemanticsEmitter = fn(&mut EmitCtx); // bindings for readability. type Cp1Emitter = fn(&mut EmitCtx, fr_mode: FrMode); +/// Result of a binary FP arithmetic op, in the current block: the raw result +/// bits plus the two properties `emit_fpu_update_fcsr_arith` needs (whether +/// the result is a denormalized nonzero, and whether its sign bit is set). +/// Shared by both arms of `emit_fbinop_s`/`_d`'s operand split, so the +/// arithmetic and its result checks cannot drift between them. +fn emit_fbinop_result_s( + ctx: &mut EmitCtx, + op: fn(&mut FunctionBuilder, Value, Value) -> Value, + fs_bits: Value, + ft_bits: Value, +) -> (Value, Value, Value) { + let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); + let ft_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), ft_bits); + let result = op(ctx.builder, fs_val, ft_val); + let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); + let is_denorm = emit_is_subnormal_s(ctx, result_bits); + let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + (result_bits, is_denorm, is_negative) +} +fn emit_fbinop_result_d( + ctx: &mut EmitCtx, + op: fn(&mut FunctionBuilder, Value, Value) -> Value, + fs_bits: Value, + ft_bits: Value, +) -> (Value, Value, Value) { + let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); + let ft_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), ft_bits); + let result = op(ctx.builder, fs_val, ft_val); + let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); + let is_denorm = emit_is_subnormal_d(ctx, result_bits); + let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + (result_bits, is_denorm, is_negative) +} + +/// Write the signed zero the Table 9-1/7-1 underflow-flush path delivers: +/// `negative` selects the sign, the magnitude is always zero. Split out +/// because both the operand-class arms of an emitter pass the same closure. +fn emit_write_signed_zero_s(ctx: &mut EmitCtx, fd: u32, negative: Value, fr_mode: FrMode) { + let zero = ctx.builder.ins().iconst(ir::types::I32, 0); + let neg32 = ctx.builder.ins().uextend(ir::types::I32, negative); + let sign = ctx.builder.ins().ishl_imm_s(neg32, 31); + let signed_zero = ctx.builder.ins().bor(zero, sign); + emit_write_fpr_w(ctx, fd, signed_zero, fr_mode) +} +fn emit_write_signed_zero_d(ctx: &mut EmitCtx, fd: u32, negative: Value, fr_mode: FrMode) { + let zero = ctx.builder.ins().iconst(ir::types::I64, 0); + let neg64 = ctx.builder.ins().uextend(ir::types::I64, negative); + let sign = ctx.builder.ins().ishl_imm_s(neg64, 63); + let signed_zero = ctx.builder.ins().bor(zero, sign); + emit_write_fpr_l(ctx, fd, signed_zero, fr_mode) +} + /// Binary FP arithmetic op shape shared by ADD/SUB/MUL/DIV, S or D format: /// read fs/ft, apply `op`, write fd, then `emit_fpu_update_fcsr`. Mirrors /// `exec_fadd_s`/`exec_fsub_d`/etc.'s common structure exactly — CU1 is @@ -7502,6 +7623,18 @@ type Cp1Emitter = fn(&mut EmitCtx, fr_mode: FrMode); /// instruction. `flags_fn` computes the [6:2] flags `Value` from the raw /// operand bits (`emit_fpu_arith_flags_snan_only_s/d` for ADD/SUB/MUL, /// `emit_fpu_arith_flags_div_s/d` for DIV) — never read from the host FPU. +/// +/// The operands are classified once, before anything else: if both are +/// ordinary finite numbers (`emit_is_normal_s`/`_d`), no operand check can +/// trap, no signalling NaN can be present and the divisor cannot be zero, so +/// this instruction provably raises no flag and the fast arm only has to +/// clear Cause (and even that store is skipped when Cause is already clear — +/// see `emit_fpu_update_fcsr_no_flags`). Everything else — zero, denormal, +/// infinity, NaN — takes the cold arm, which is the full previous body +/// verbatim: per-operand Unimplemented checks, the by-value flags, the +/// result-denormal rule and the full FCSR update. The result-denormal check +/// stays on both arms because an underflowing result is reachable from +/// ordinary finite operands. fn emit_fbinop_s( ctx: &mut EmitCtx, fr_mode: FrMode, @@ -7515,26 +7648,41 @@ fn emit_fbinop_s( let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); let ft_bits = emit_read_fpr_w(ctx, ft, fr_mode); + + let fs_normal = emit_is_normal_s(ctx, fs_bits); + let ft_normal = emit_is_normal_s(ctx, ft_bits); + let operands_normal = ctx.builder.ins().band(fs_normal, ft_normal); + let fast_block = ctx.builder.create_block(); + let slow_block = ctx.builder.create_block(); + let done_block = ctx.builder.create_block(); + ctx.builder.ins().brif(operands_normal, fast_block, &[], slow_block, &[]); + + ctx.builder.switch_to_block(slow_block); + ctx.builder.set_cold_block(slow_block); + ctx.builder.seal_block(slow_block); emit_check_denorm_operand(ctx, fs_bits, false); emit_check_denorm_operand(ctx, ft_bits, false); let flags = flags_fn(ctx, fs_bits, ft_bits); - let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); - let ft_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), ft_bits); - let result = op(ctx.builder, fs_val, ft_val); - let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); - let is_denorm = emit_is_subnormal_s(ctx, result_bits); - let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + let (result_bits, is_denorm, is_negative) = emit_fbinop_result_s(ctx, op, fs_bits, ft_bits); emit_fpu_update_fcsr_arith( - ctx, flags, is_denorm, is_negative, + ctx, Some(flags), is_denorm, is_negative, |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), - |ctx, neg| { - let zero = ctx.builder.ins().iconst(ir::types::I32, 0); - let neg32 = ctx.builder.ins().uextend(ir::types::I32, neg); - let sign = ctx.builder.ins().ishl_imm_s(neg32, 31); - let signed_zero = ctx.builder.ins().bor(zero, sign); - emit_write_fpr_w(ctx, fd, signed_zero, fr_mode) - }, + |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), + ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + let (result_bits, is_denorm, is_negative) = emit_fbinop_result_s(ctx, op, fs_bits, ft_bits); + emit_fpu_update_fcsr_arith( + ctx, None, is_denorm, is_negative, + |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), + |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(done_block); + ctx.builder.seal_block(done_block); } fn emit_fbinop_d( ctx: &mut EmitCtx, @@ -7549,26 +7697,41 @@ fn emit_fbinop_d( let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); let ft_bits = emit_read_fpr_l(ctx, ft, fr_mode); + + let fs_normal = emit_is_normal_d(ctx, fs_bits); + let ft_normal = emit_is_normal_d(ctx, ft_bits); + let operands_normal = ctx.builder.ins().band(fs_normal, ft_normal); + let fast_block = ctx.builder.create_block(); + let slow_block = ctx.builder.create_block(); + let done_block = ctx.builder.create_block(); + ctx.builder.ins().brif(operands_normal, fast_block, &[], slow_block, &[]); + + ctx.builder.switch_to_block(slow_block); + ctx.builder.set_cold_block(slow_block); + ctx.builder.seal_block(slow_block); emit_check_denorm_operand(ctx, fs_bits, true); emit_check_denorm_operand(ctx, ft_bits, true); let flags = flags_fn(ctx, fs_bits, ft_bits); - let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); - let ft_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), ft_bits); - let result = op(ctx.builder, fs_val, ft_val); - let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); - let is_denorm = emit_is_subnormal_d(ctx, result_bits); - let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + let (result_bits, is_denorm, is_negative) = emit_fbinop_result_d(ctx, op, fs_bits, ft_bits); emit_fpu_update_fcsr_arith( - ctx, flags, is_denorm, is_negative, + ctx, Some(flags), is_denorm, is_negative, |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), - |ctx, neg| { - let zero = ctx.builder.ins().iconst(ir::types::I64, 0); - let neg64 = ctx.builder.ins().uextend(ir::types::I64, neg); - let sign = ctx.builder.ins().ishl_imm_s(neg64, 63); - let signed_zero = ctx.builder.ins().bor(zero, sign); - emit_write_fpr_l(ctx, fd, signed_zero, fr_mode) - }, + |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), + ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + let (result_bits, is_denorm, is_negative) = emit_fbinop_result_d(ctx, op, fs_bits, ft_bits); + emit_fpu_update_fcsr_arith( + ctx, None, is_denorm, is_negative, + |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), + |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(done_block); + ctx.builder.seal_block(done_block); } fn fop_add(builder: &mut FunctionBuilder, a: Value, b: Value) -> Value { builder.ins().fadd(a, b) } @@ -7588,10 +7751,27 @@ fn emit_fdiv_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, /// SQRT.S/D fd, fs: fd = sqrt(fs). Unlike the binary ops, only one operand /// (`fs`; `ft`/`d.rt` is unused/ignored, matching the interpreter). Still /// goes through `emit_fpu_update_fcsr` (sqrt of a negative raises Invalid). +/// +/// The fast arm requires a normal *and non-negative* operand: a negative +/// nonzero operand raises Invalid, and every other special value (zero, +/// denormal, infinity, NaN) is outside the class `emit_is_normal_s`/`_d` +/// proves flag-free. The cold arm is the previous body verbatim. fn emit_fsqrt_s(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); + + let fs_normal = emit_is_normal_s(ctx, fs_bits); + let fs_nonneg = ctx.builder.ins().icmp_imm_s(IntCC::SignedGreaterThanOrEqual, fs_bits, 0); + let flag_free = ctx.builder.ins().band(fs_normal, fs_nonneg); + let fast_block = ctx.builder.create_block(); + let slow_block = ctx.builder.create_block(); + let done_block = ctx.builder.create_block(); + ctx.builder.ins().brif(flag_free, fast_block, &[], slow_block, &[]); + + ctx.builder.switch_to_block(slow_block); + ctx.builder.set_cold_block(slow_block); + ctx.builder.seal_block(slow_block); emit_check_denorm_operand(ctx, fs_bits, false); let flags = emit_fpu_arith_flags_sqrt_s(ctx, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); @@ -7600,21 +7780,45 @@ fn emit_fsqrt_s(ctx: &mut EmitCtx, fr_mode: FrMode) { let is_denorm = emit_is_subnormal_s(ctx, result_bits); let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); emit_fpu_update_fcsr_arith( - ctx, flags, is_denorm, is_negative, + ctx, Some(flags), is_denorm, is_negative, |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), - |ctx, neg| { - let zero = ctx.builder.ins().iconst(ir::types::I32, 0); - let neg32 = ctx.builder.ins().uextend(ir::types::I32, neg); - let sign = ctx.builder.ins().ishl_imm_s(neg32, 31); - let signed_zero = ctx.builder.ins().bor(zero, sign); - emit_write_fpr_w(ctx, fd, signed_zero, fr_mode) - }, + |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); + let result = ctx.builder.ins().sqrt(fs_val); + let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); + let is_denorm = emit_is_subnormal_s(ctx, result_bits); + let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + emit_fpu_update_fcsr_arith( + ctx, None, is_denorm, is_negative, + |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), + |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), + ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(done_block); + ctx.builder.seal_block(done_block); } fn emit_fsqrt_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); + + let fs_normal = emit_is_normal_d(ctx, fs_bits); + let fs_nonneg = ctx.builder.ins().icmp_imm_s(IntCC::SignedGreaterThanOrEqual, fs_bits, 0); + let flag_free = ctx.builder.ins().band(fs_normal, fs_nonneg); + let fast_block = ctx.builder.create_block(); + let slow_block = ctx.builder.create_block(); + let done_block = ctx.builder.create_block(); + ctx.builder.ins().brif(flag_free, fast_block, &[], slow_block, &[]); + + ctx.builder.switch_to_block(slow_block); + ctx.builder.set_cold_block(slow_block); + ctx.builder.seal_block(slow_block); emit_check_denorm_operand(ctx, fs_bits, true); let flags = emit_fpu_arith_flags_sqrt_d(ctx, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); @@ -7623,16 +7827,28 @@ fn emit_fsqrt_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let is_denorm = emit_is_subnormal_d(ctx, result_bits); let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); emit_fpu_update_fcsr_arith( - ctx, flags, is_denorm, is_negative, + ctx, Some(flags), is_denorm, is_negative, |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), - |ctx, neg| { - let zero = ctx.builder.ins().iconst(ir::types::I64, 0); - let neg64 = ctx.builder.ins().uextend(ir::types::I64, neg); - let sign = ctx.builder.ins().ishl_imm_s(neg64, 63); - let signed_zero = ctx.builder.ins().bor(zero, sign); - emit_write_fpr_l(ctx, fd, signed_zero, fr_mode) - }, + |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), + ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); + let result = ctx.builder.ins().sqrt(fs_val); + let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); + let is_denorm = emit_is_subnormal_d(ctx, result_bits); + let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); + emit_fpu_update_fcsr_arith( + ctx, None, is_denorm, is_negative, + |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), + |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), ); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(done_block); + ctx.builder.seal_block(done_block); } /// ABS.S/D, NEG.S/D, MOV.S/D fd, fs: unlike ADD/SUB/MUL/DIV/SQRT, these diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index b207852d..1c958612 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -1264,12 +1264,26 @@ mod tests { } fn run_interpreter_fpu(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, fr1: bool) -> CoreSnapshot { + run_interpreter_fpu_fcsr(instr, gpr, fpr, pc, fr1, None) + } + + /// As above, with `core.fpu_fcsr` seeded to `fcsr` before the instruction + /// runs (or left at its initial value when `None`). Needed to test the + /// instruction's handling of pre-existing FCSR state — e.g. a Cause field + /// left set by an earlier instruction. + fn run_interpreter_fpu_fcsr(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, fr1: bool, fcsr: Option) -> CoreSnapshot { let (mut exec, _mem) = fpu_seeded_executor(gpr, fpr, pc, fr1); + if let Some(fcsr) = fcsr { exec.core.fpu_fcsr = fcsr; } exec.exec(instr); CoreSnapshot::capture(&exec.core) } fn run_jit_fpu(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, word_offset: u16, fr1: bool) -> Option { + run_jit_fpu_fcsr(instr, gpr, fpr, pc, word_offset, fr1, None) + } + + /// `run_interpreter_fpu_fcsr`'s JIT counterpart. + fn run_jit_fpu_fcsr(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, word_offset: u16, fr1: bool, fcsr: Option) -> Option { let mut page = [0u32; ENTRIES_PER_PAGE]; page[word_offset as usize] = instr; @@ -1284,6 +1298,7 @@ mod tests { let (exec, _mem) = fpu_seeded_executor(gpr, fpr, pc, fr1); let mut exec = Box::new(exec); exec.install_jit_hooks(); + if let Some(fcsr) = fcsr { exec.core.fpu_fcsr = fcsr; } unsafe { jit_fn(&mut exec.core as *mut MipsCore) }; std::mem::forget(codegen); @@ -1292,12 +1307,20 @@ mod tests { } fn assert_fpu_matches_interpreter(instr: u32, gpr: [u64; 32], fpr: [u64; 32], fr1: bool) { + assert_fpu_matches_interpreter_fcsr(instr, gpr, fpr, fr1, None); + } + + /// `assert_fpu_matches_interpreter` with `core.fpu_fcsr` seeded before both + /// engines run, returning the JIT's snapshot so a test can also pin the + /// absolute result rather than only JIT/interpreter agreement. + fn assert_fpu_matches_interpreter_fcsr(instr: u32, gpr: [u64; 32], fpr: [u64; 32], fr1: bool, fcsr: Option) -> CoreSnapshot { let pc = 0xFFFF_FFFF_8000_1000u64; let word_offset = (pc as u16 / 4) & 0x3FF; - let interp = run_interpreter_fpu(instr, gpr, fpr, pc, fr1); - let jit = run_jit_fpu(instr, gpr, fpr, pc, word_offset, fr1) + let interp = run_interpreter_fpu_fcsr(instr, gpr, fpr, pc, fr1, fcsr); + let jit = run_jit_fpu_fcsr(instr, gpr, fpr, pc, word_offset, fr1, fcsr) .expect("FPU instruction must be compilable for this test to be meaningful"); assert_eq!(jit, interp, "JIT and interpreter diverged for instr=0x{:08x} fr1={}", instr, fr1); + jit } /// `run_interpreter_fpu`/`run_jit_fpu`'s counterpart for LWC1/LDC1 — @@ -5436,6 +5459,43 @@ mod tests { fbinop_case(crate::cpu::mips_isa::FUNCT_FDIV, crate::cpu::mips_isa::RS_D, 1.0, 0.0); } + /// An ordinary-operand arithmetic instruction must rewrite Cause to zero + /// even when it raises nothing itself ("the results of only one + /// instruction"), and must not disturb the sticky Flag field. Seeded with + /// Cause.V|Cause.Z and Flag.V|Flag.I already set and every Enable clear, + /// so the JIT takes its "Cause is already set, clear it" arm rather than + /// the skip-the-store one, and both engines must agree and deliver the + /// flags unchanged. + #[test] + fn arithmetic_clears_preexisting_cause_and_keeps_sticky_flags() { + const FCSR_CAUSE_VZ: u32 = 0x0001_8000; + const FCSR_FLAG_VI: u32 = 0x0000_0044; + const SEED: u32 = FCSR_CAUSE_VZ | FCSR_FLAG_VI; + for fmt in [crate::cpu::mips_isa::RS_S, crate::cpu::mips_isa::RS_D] { + for funct in [ + crate::cpu::mips_isa::FUNCT_FADD, + crate::cpu::mips_isa::FUNCT_FSUB, + crate::cpu::mips_isa::FUNCT_FMUL, + crate::cpu::mips_isa::FUNCT_FDIV, + ] { + let mut fpr = [0u64; 32]; + if fmt == crate::cpu::mips_isa::RS_S { + fpr[1] = (1.5f32).to_bits() as u64; + fpr[2] = (2.5f32).to_bits() as u64; + } else { + fpr[1] = (1.5f64).to_bits(); + fpr[2] = (2.5f64).to_bits(); + } + let instr = make_r(crate::cpu::mips_isa::OP_COP1, fmt, 2, 1, 3, funct); + let jit = assert_fpu_matches_interpreter_fcsr( + instr, [0u64; 32], fpr, true, Some(SEED)); + assert_eq!(jit.fpu_fcsr, FCSR_FLAG_VI, + "ordinary-operand arithmetic must clear Cause and keep sticky Flags \ + (instr=0x{:08x} fmt={} funct={})", instr, fmt, funct); + } + } + } + fn funop_case(funct: u32, fmt: u32, fs_val: f64) { let mut fpr = [0u64; 32]; if fmt == crate::cpu::mips_isa::RS_S {