Implement explicit state switch betwee X87 and MMX

Fixes #3850

FEXCore/Scripts/json_ir_generator.py

@@ -699,8 +699,10 @@ def print_ir_allocator_helpers():
 
             # We gather the "has x87?" flag as we go. This saves the user from
             # having to keep track of whether they emitted any x87.
+            # Also changes the mmx state to X87.
             if op.LoweredX87:
                 output_file.write("\t\tRecordX87Use();\n")
+                output_file.write("\t\tMMXState = MMXState_X87;\n")
 
             output_file.write("\t\tauto _Op = AllocateOp<IROp_{}, IROps::OP_{}>();\n".format(op.Name, op.Name.upper()))
 
@@ -826,4 +828,3 @@ def print_ir_dispatcher_dispatch():
 print_ir_dispatcher_dispatch()
 
 output_dispatch_file.close()
-

FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -4413,8 +4413,12 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl
       LOGMAN_THROW_A_FMT(Class == FPRClass, "MMX is floaty");
 
       // Partial store into bottom 64-bits, leave the upper bits unaffected.
-      // XXX: We actually should set the upper bits to all-1s?
-      StoreContextPartial(MM0Index + gpr - FEXCore::X86State::REG_MM_0, Src);
+      if (MMXState == MMXState_X87) {
+        ChgStateX87_MMX();
+      }
+      uint8_t Index = MM0Index + gpr - FEXCore::X86State::REG_MM_0;
+      StoreContext(Index, Src);
+      RegCache.Partial |= (1ull << (uint64_t)Index);
     } else if (gpr >= FEXCore::X86State::REG_XMM_0) {
       const auto gprIndex = gpr - X86State::REG_XMM_0;
       const auto VectorSize = GetGuestVectorLength();

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -1242,6 +1242,13 @@ class OpDispatchBuilder final : public IREmitter {
           _StoreContextPair(Size, Class, ValueNext, Value, Offset - Size);
           Bits &= ~NextBit;
         } else {
+          // If Partial and MMX register, then we need to store all 1s in bits 64-80
+          if (Partial && Index >= MM0Index && Index <= MM7Index) {
+            _Print(Value);
+            Value = _VInsGPR(i64Bit, 8, 1, Value, _Constant(0xFFFFUL));
+            Size = 16;
+          }
+          _Print(Value);
           _StoreContext(Size, Class, Value, Offset);
         }
       }
@@ -1964,12 +1971,6 @@ class OpDispatchBuilder final : public IREmitter {
     RegCache.Written |= Bit;
   }
 
-  void StoreContextPartial(uint8_t Index, Ref Value) {
-    StoreContext(Index, Value);
-
-    RegCache.Partial |= (1ull << (uint64_t)Index);
-  }
-
   void StoreRegister(uint8_t Reg, bool FPR, Ref Value) {
     StoreContext(Reg + (FPR ? FPR0Index : GPR0Index), Value);
   }
@@ -2333,7 +2334,6 @@ class OpDispatchBuilder final : public IREmitter {
     }
   }
 
-  /**  @} */
   /**  @} */
 
   Ref GetX87Top();
@@ -2342,6 +2342,14 @@ class OpDispatchBuilder final : public IREmitter {
   Ref GetX87FTW_Helper();
   void SetX87Top(Ref Value);
 
+  void ChgStateX87_MMX() override {
+    LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");
+    _StackForceSlow();
+    SetX87Top(_Constant(0));                             // top reset to zero
+    StoreContext(AbridgedFTWIndex, _Constant(0xFFFFUL)); // all valid
+    MMXState = MMXState_MMX;
+  }
+
   bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) const {
     return DestIsMem(Op) && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
   }

FEXCore/Source/Interface/IR/IREmitter.h

@@ -343,8 +343,12 @@ class IREmitter {
     return Ptr;
   }
 
+  // MMX State can be either MMX (for 64bit) or x87 FPU (for 80bit)
+  enum { MMXState_MMX, MMXState_X87 } MMXState = MMXState_MMX;
+
   // Overriden by dispatcher, stubbed for IR tests
   virtual void RecordX87Use() {}
+  virtual void ChgStateX87_MMX() {}
   virtual void SaveNZCV(IROps Op) {}
 
   Ref CurrentWriteCursor = nullptr;

unittests/ASM/X87/X87MMXInteraction.asm

@@ -0,0 +1,56 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x0",
+    "RBX": "0x0",
+    "RCX": "0x8000000000000000",
+    "RDX": "0x3FFF",
+    "R8": "0xc90fdaa22168c235",
+    "R9": "0x4000",
+    "R10": "0xc90fdaa22168c235",
+    "R11": "0xFFFF"
+  },
+  "MemoryRegions": {
+    "0x100000000": "4096"
+  }
+}
+%endif
+
+section .bss
+  x87env: resb 108
+
+section .text 
+global _start
+; Checks that after moving from X87 to MMX States, the
+; values are correct and that MMX register writes, puts the top 16 bits as
+; all 1s.
+_start:
+finit ; enters x87 state 
+
+fldpi ; goes in mm7
+fld1  ; goes in mm6
+
+movq mm5, mm7 ; enters mmx state, so 1 is now in st6 and pi in st7, while st5 has a broken pi. 
+o32 fnsave [rel x87env]
+
+; Top into eax
+mov eax, dword [rel x87env + 4]
+and eax, 0x3800
+shr eax, 11 ; top in eax
+
+; Tag into ebx
+mov bx, word [rel x87env + 8]
+
+; st6 is 1
+mov rcx, qword [rel x87env + 88]
+mov dx, word [rel x87env + 96]
+
+; st7 is pi
+mov r8, qword [rel x87env + 98]
+mov r9w, word [rel x87env + 106]
+
+; st5 
+mov r10, qword [rel x87env + 78]
+mov r11w, word [rel x87env + 86]
+
+hlt