difftest: support SMP difftest (#786)

* difftest: revise coreid assignment

* nemuproxy: compatible for smp difftest

* difftest: fix goldenMem initialization problem

* difftest: goldenMem update works

* difftest: api compatible for modified nemu

* difftest: support load check for smp difftest
* verification is required later

* misc: remove unused xstrap wiring

* Remove unused code

* difftest: add latch for difftest-loadevent

* misc: update inclusivecache

* difftest: reset resp for sbuffer & atomic-unit to avoid duplicate update
of goldenMem

* difftest: dump coreid when difftest failed

* difftest: dump corresponding memory of another core when smp difftest failed
* Only works for dual-core

* difftest: fix interrupt handler

* difftest: cleanup code

* roq: remove legacy signal for difftest

block-inclusivecache-sifive @ 2dd39823

-Subproject commit 1566b4be1693f0e0d076172f70055693c8cea503
+Subproject commit 2dd39823479ab3fb3101196cf46eaf9437378096

src/main/scala/xiangshan/backend/FloatBlock.scala

@@ -224,7 +224,7 @@ class FloatBlock
     }
     val difftest = Module(new DifftestArchFpRegState)
     difftest.io.clock  := clock
-    difftest.io.coreid := 0.U
+    difftest.io.coreid := hardId.U
     difftest.io.fpr    := VecInit(fpRf.io.debug_rports.map(p => ieee(p.data)))
   }
 

src/main/scala/xiangshan/backend/IntegerBlock.scala

@@ -271,7 +271,7 @@ class IntegerBlock
     }
     val difftest = Module(new DifftestArchIntRegState)
     difftest.io.clock  := clock
-    difftest.io.coreid := 0.U
+    difftest.io.coreid := hardId.U
     difftest.io.gpr    := VecInit(intRf.io.debug_rports.map(_.data))
   }
 

src/main/scala/xiangshan/backend/fu/CSR.scala

@@ -870,7 +870,7 @@ class CSR(implicit p: Parameters) extends FunctionUnit with HasCSRConst
   if (!env.FPGAPlatform) {
     val difftest = Module(new DifftestArchEvent)
     difftest.io.clock := clock
-    difftest.io.coreid := 0.U
+    difftest.io.coreid := hardId.U
     difftest.io.intrNO := RegNext(difftestIntrNO)
     difftest.io.cause := RegNext(Mux(csrio.exception.valid, causeNO, 0.U))
     difftest.io.exceptionPC := RegNext(SignExt(csrio.exception.bits.uop.cf.pc, XLEN))
@@ -879,7 +879,7 @@ class CSR(implicit p: Parameters) extends FunctionUnit with HasCSRConst
   if (!env.FPGAPlatform) {
     val difftest = Module(new DifftestCSRState)
     difftest.io.clock := clock
-    difftest.io.coreid := 0.U
+    difftest.io.coreid := hardId.U
     difftest.io.priviledgeMode := priviledgeMode
     difftest.io.mstatus := mstatus
     difftest.io.sstatus := mstatus & sstatusRmask

src/main/scala/xiangshan/backend/roq/Roq.scala

@@ -9,14 +9,6 @@ import utils._
 import xiangshan.backend.ftq.FtqPtr
 import difftest._
 
-object roqDebugId extends Function0[Integer] {
-  var x = 0
-  def apply(): Integer = {
-    x = x + 1
-    return x
-  }
-}
-
 class RoqPtr(implicit p: Parameters) extends CircularQueuePtr[RoqPtr](
   p => p(XSCoreParamsKey).RoqSize
 ) with HasCircularQueuePtrHelper {
@@ -817,64 +809,30 @@ class Roq(numWbPorts: Int)(implicit p: Parameters) extends XSModule with HasCirc
   //difftest signals
   val firstValidCommit = (deqPtr + PriorityMux(io.commits.valid, VecInit(List.tabulate(CommitWidth)(_.U)))).value
 
-  val skip = Wire(Vec(CommitWidth, Bool()))
-  val wen = Wire(Vec(CommitWidth, Bool()))
   val wdata = Wire(Vec(CommitWidth, UInt(XLEN.W)))
-  val lpaddr = Wire(Vec(CommitWidth, UInt(PAddrBits.W)))
-  val ltype = Wire(Vec(CommitWidth, UInt(32.W)))
-  val lfu = Wire(Vec(CommitWidth, UInt(4.W)))
-  val wdst = Wire(Vec(CommitWidth, UInt(32.W)))
-  val diffTestDebugLrScValid = Wire(Vec(CommitWidth, Bool()))
   val wpc = Wire(Vec(CommitWidth, UInt(XLEN.W)))
   val trapVec = Wire(Vec(CommitWidth, Bool()))
-  val isRVC = Wire(Vec(CommitWidth, Bool()))
   for(i <- 0 until CommitWidth) {
     // io.commits(i).valid
     val idx = deqPtrVec(i).value
     val uop = debug_microOp(idx)
-    val DifftestSkipSC = false
-    if(!DifftestSkipSC){
-      skip(i) := (debug_exuDebug(idx).isMMIO || debug_exuDebug(idx).isPerfCnt) && io.commits.valid(i)
-    }else{
-      skip(i) := (
-          debug_exuDebug(idx).isMMIO ||
-          debug_exuDebug(idx).isPerfCnt ||
-          uop.ctrl.fuType === FuType.mou && uop.ctrl.fuOpType === LSUOpType.sc_d ||
-          uop.ctrl.fuType === FuType.mou && uop.ctrl.fuOpType === LSUOpType.sc_w
-        ) && io.commits.valid(i)
-    }
-    wen(i) := io.commits.valid(i) && uop.ctrl.rfWen && uop.ctrl.ldest =/= 0.U
     wdata(i) := debug_exuData(idx)
-    lpaddr(i) := debug_exuDebug(idx).paddr
-    lfu(i) := uop.ctrl.fuType
-    ltype(i) := uop.ctrl.fuOpType
-    wdst(i) := uop.ctrl.ldest
-    diffTestDebugLrScValid(i) := uop.diffTestDebugLrScValid
     wpc(i) := SignExt(uop.cf.pc, XLEN)
     trapVec(i) := io.commits.valid(i) && (state===s_idle) && uop.ctrl.isXSTrap
-    isRVC(i) := uop.cf.pd.isRVC
   }
   val retireCounterFix = Mux(io.exception.valid, 1.U, retireCounter)
   val retirePCFix = SignExt(Mux(io.exception.valid, io.exception.bits.uop.cf.pc, debug_microOp(firstValidCommit).cf.pc), XLEN)
   val retireInstFix = Mux(io.exception.valid, io.exception.bits.uop.cf.instr, debug_microOp(firstValidCommit).cf.instr)
 
-  val scFailed = !diffTestDebugLrScValid(0) &&
-    debug_deqUop.ctrl.fuType === FuType.mou &&
-    (debug_deqUop.ctrl.fuOpType === LSUOpType.sc_d || debug_deqUop.ctrl.fuOpType === LSUOpType.sc_w)
-
   val hitTrap = trapVec.reduce(_||_)
   val trapCode = PriorityMux(wdata.zip(trapVec).map(x => x._2 -> x._1))
   val trapPC = SignExt(PriorityMux(wpc.zip(trapVec).map(x => x._2 ->x._1)), XLEN)
 
-  if (!env.FPGAPlatform) {
-    ExcitingUtils.addSource(hitTrap, "XSTRAP", ConnectionType.Debug)
-  }
-
   if (!env.FPGAPlatform) {
     for (i <- 0 until CommitWidth) {
       val difftest = Module(new DifftestInstrCommit)
       difftest.io.clock    := clock
-      difftest.io.coreid   := 0.U
+      difftest.io.coreid   := hardId.U
       difftest.io.index    := i.U
 
       val ptr = deqPtrVec(i).value
@@ -899,23 +857,23 @@ class Roq(numWbPorts: Int)(implicit p: Parameters) extends XSModule with HasCirc
     for (i <- 0 until CommitWidth) {
       val difftest = Module(new DifftestLoadEvent)
       difftest.io.clock  := clock
-      difftest.io.coreid := 0.U
+      difftest.io.coreid := hardId.U
       difftest.io.index  := i.U
 
       val ptr = deqPtrVec(i).value
       val uop = debug_microOp(ptr)
       val exuOut = debug_exuDebug(ptr)
-      difftest.io.valid  := io.commits.valid(i) && !io.commits.isWalk
-      difftest.io.paddr  := exuOut.paddr
-      difftest.io.opType := uop.ctrl.fuOpType
-      difftest.io.fuType := uop.ctrl.fuType
+      difftest.io.valid  := RegNext(io.commits.valid(i) && !io.commits.isWalk)
+      difftest.io.paddr  := RegNext(exuOut.paddr)
+      difftest.io.opType := RegNext(uop.ctrl.fuOpType)
+      difftest.io.fuType := RegNext(uop.ctrl.fuType)
     }
   }
 
   if (!env.FPGAPlatform) {
     val difftest = Module(new DifftestTrapEvent)
     difftest.io.clock    := clock
-    difftest.io.coreid   := 0.U
+    difftest.io.coreid   := hardId.U
     difftest.io.valid    := hitTrap
     difftest.io.code     := trapCode
     difftest.io.pc       := trapPC

src/main/scala/xiangshan/mem/lsqueue/StoreQueue.scala

@@ -384,7 +384,7 @@ class StoreQueue(implicit p: Parameters) extends XSModule with HasDCacheParamete
 
       val difftest = Module(new DifftestStoreEvent)
       difftest.io.clock       := clock
-      difftest.io.coreid      := 0.U
+      difftest.io.coreid      := hardId.U
       difftest.io.index       := i.U
       difftest.io.valid       := storeCommit
       difftest.io.storeAddr   := waddr

src/main/scala/xiangshan/mem/pipeline/AtomicsUnit.scala

@@ -250,7 +250,7 @@ class AtomicsUnit(implicit p: Parameters) extends XSModule with MemoryOpConstant
   if (!env.FPGAPlatform) {
     val difftest = Module(new DifftestAtomicEvent)
     difftest.io.clock      := clock
-    difftest.io.coreid     := 0.U
+    difftest.io.coreid     := hardId.U
     difftest.io.atomicResp := io.dcache.resp.fire()
     difftest.io.atomicAddr := paddr_reg
     difftest.io.atomicData := data_reg

src/main/scala/xiangshan/mem/sbuffer/NewSbuffer.scala

@@ -356,7 +356,7 @@ class NewSbuffer(implicit p: Parameters) extends XSModule with HasSbufferConst {
   if (!env.FPGAPlatform) {
     val difftest = Module(new DifftestSbufferEvent)
     difftest.io.clock := clock
-    difftest.io.coreid := 0.U
+    difftest.io.coreid := hardId.U
     difftest.io.sbufferResp := io.dcache.resp.fire()
     difftest.io.sbufferAddr := getAddr(tag(respId))
     difftest.io.sbufferData := data(respId).asTypeOf(Vec(CacheLineBytes, UInt(8.W)))

src/test/csrc/difftest/difftest.cpp

@@ -83,7 +83,7 @@ int Difftest::step() {
   if (do_store_check()) {
     return 1;
   }
-  if (do_golden_memory_check()) {
+  if (do_golden_memory_update()) {
     return 1;
   }
 
@@ -94,6 +94,7 @@ int Difftest::step() {
   num_commit = 0; // reset num_commit this cycle to 0
   // interrupt has the highest priority
   if (dut.event.interrupt) {
+    dut.csr.this_pc = dut.event.exceptionPC;
     do_interrupt();
   } else if(dut.event.exception) { 
     // We ignored instrAddrMisaligned exception (0) for better debug interface
@@ -141,7 +142,7 @@ int Difftest::step() {
 
 void Difftest::do_interrupt() {
   state->record_abnormal_inst(dut.commit[0].pc, dut.commit[0].inst, RET_INT, dut.event.interrupt);
-  proxy->raise_intr(dut.event.interrupt);
+  proxy->raise_intr(dut.event.interrupt | (1ULL << 63));
   progress = true;
 }
 
@@ -198,64 +199,66 @@ void Difftest::do_instr_commit(int i) {
   // ds.mtval = dut.csr.mtval;
   // ds.stval = dut.csr.stval;
   // proxy->disambiguate_exec(&ds);
-  // } else {
-
-    // // Load instruction
-    // if (dut.load[i].fuType == 0xC || dut.load[i].fuType == 0xF) {
-    //   proxy->get_regs(ref_regs_ptr);
-    //   if (dut.commit[i].wen && ref_regs_ptr[dut.commit[i].wdest] != dut.commit[i].wdata) {
-    //     // printf("---[DIFF Core%d] This load instruction gets rectified!\n", coreid);
-    //     // printf("---    ltype: 0x%x paddr: 0x%lx wen: 0x%x wdst: 0x%x wdata: 0x%lx pc: 0x%lx\n", s->ltype[i], s->lpaddr[i], selectBit(s->wen, i), s->wdst[i], s->wdata[i], s->wpc[i]);
-    //     uint64_t golden;
-    //     int len = 0;
-    //     if (dut.load[i].fuType == 0xC) {
-    //       switch (dut.load[i].opType) {
-    //         case 0: len = 1; break;
-    //         case 1: len = 2; break;
-    //         case 2: len = 4; break;
-    //         case 3: len = 8; break;
-    //         case 4: len = 1; break;
-    //         case 5: len = 2; break;
-    //         case 6: len = 4; break;
-    //         default:
-    //           printf("Unknown fuOpType: 0x%x\n", dut.load[i].opType);
-    //       }
-    //     } else if (dut.load[i].fuType == 0xF) {
-    //       if (dut.load[i].opType % 2 == 0) {
-    //         len = 4;
-    //       } else if (dut.load[i].opType % 2 == 1) {
-    //         len = 8;
-    //       }
-    //     }
-    //     read_goldenmem(dut.load[i].paddr, &golden, len);
-    //     if (dut.load[i].fuType == 0xC) {
-    //       switch (dut.load[i].opType) {
-    //         case 0: golden = (int64_t)(int8_t)golden; break;
-    //         case 1: golden = (int64_t)(int16_t)golden; break;
-    //         case 2: golden = (int64_t)(int32_t)golden; break;
-    //       }
-    //     }
-    //     // printf("---    golden: 0x%lx  original: 0x%lx\n", golden, ref_r[dut.commit[i].wdest]);
-    //     if (golden == dut.commit[i].wdata) {
-    //       // ref_difftest_memcpy_from_dut(0x80000000, get_img_start(), get_img_size(), i);
-    //       proxy->memcpy_from_dut(dut.load[i].paddr, &golden, len);
-    //       if (dut.commit[i].wdest != 0) {
-    //         ref_regs_ptr[dut.commit[i].wdest] = dut.commit[i].wdata;
-    //         proxy->set_regs(ref_regs_ptr);
-    //       }
-    //     } else if (dut.load[i].fuType == 0xF) {
-    //       proxy->memcpy_from_dut(dut.load[i].paddr, &golden, len);
-    //       if (dut.commit[i].wdest != 0) {
-    //         ref_regs_ptr[dut.commit[i].wdest] = dut.commit[i].wdata;
-    //         proxy->set_regs(ref_regs_ptr);
-    //       }
-    //       // printf("---    atomic instr carefully handled\n");
-    //     } else {
-    //       // printf("---    goldenmem check failed as well\n");
-    //     }
-      // }
-    // }
   // }
+
+  // Handle load instruction carefully for SMP
+  if (dut.load[i].fuType == 0xC || dut.load[i].fuType == 0xF) {
+    proxy->get_regs(ref_regs_ptr);
+    if (dut.commit[i].wen && ref_regs_ptr[dut.commit[i].wdest] != dut.commit[i].wdata) {
+      // printf("---[DIFF Core%d] This load instruction gets rectified!\n", this->id);
+      // printf("---    ltype: 0x%x paddr: 0x%lx wen: 0x%x wdst: 0x%x wdata: 0x%lx pc: 0x%lx\n", dut.load[i].opType, dut.load[i].paddr, dut.commit[i].wen, dut.commit[i].wdest, dut.commit[i].wdata, dut.commit[i].pc);
+      uint64_t golden;
+      int len = 0;
+      if (dut.load[i].fuType == 0xC) {
+        switch (dut.load[i].opType) {
+          case 0: len = 1; break;
+          case 1: len = 2; break;
+          case 2: len = 4; break;
+          case 3: len = 8; break;
+          case 4: len = 1; break;
+          case 5: len = 2; break;
+          case 6: len = 4; break;
+          default:
+            printf("Unknown fuOpType: 0x%x\n", dut.load[i].opType);
+        }
+      } else {  // dut.load[i].fuType == 0xF
+        if (dut.load[i].opType % 2 == 0) {
+          len = 4;
+        } else {  // dut.load[i].opType % 2 == 1
+          len = 8;
+        }
+      }
+      read_goldenmem(dut.load[i].paddr, &golden, len);
+      if (dut.load[i].fuType == 0xC) {
+        switch (dut.load[i].opType) {
+          case 0: golden = (int64_t)(int8_t)golden; break;
+          case 1: golden = (int64_t)(int16_t)golden; break;
+          case 2: golden = (int64_t)(int32_t)golden; break;
+        }
+      }
+      // printf("---    golden: 0x%lx  original: 0x%lx\n", golden, ref_regs_ptr[dut.commit[i].wdest]);
+      if (golden == dut.commit[i].wdata) {
+        proxy->memcpy_from_dut(dut.load[i].paddr, &golden, len);
+        if (dut.commit[i].wdest != 0) {
+          ref_regs_ptr[dut.commit[i].wdest] = dut.commit[i].wdata;
+          proxy->set_regs(ref_regs_ptr);
+        }
+      } else if (dut.load[i].fuType == 0xF) {  //  atomic instr carefully handled
+        proxy->memcpy_from_dut(dut.load[i].paddr, &golden, len);
+        if (dut.commit[i].wdest != 0) {
+          ref_regs_ptr[dut.commit[i].wdest] = dut.commit[i].wdata;
+          proxy->set_regs(ref_regs_ptr);
+        }
+      } else {
+        // goldenmem check failed as well, raise error
+        printf("---  SMP difftest mismatch!\n");
+        printf("---  Trying to probe local data of another core\n");
+        uint64_t buf;
+        difftest[(EMU_CORES-1) - this->id]->proxy->memcpy_from_ref(&buf, dut.load[i].paddr, len);
+        printf("---    content: %lx\n", buf);
+      }
+    }
+  }
 }
 
 void Difftest::do_first_instr_commit() {
@@ -294,21 +297,22 @@ int Difftest::do_store_check() {
   return 0;
 }
 
-inline void handle_atomic(uint64_t atomicAddr, uint64_t atomicData, uint64_t atomicMask, uint8_t atomicFuop, uint64_t atomicOut) {
+inline int handle_atomic(int coreid, uint64_t atomicAddr, uint64_t atomicData, uint64_t atomicMask, uint8_t atomicFuop, uint64_t atomicOut) {
+  // We need to do atmoic operations here so as to update goldenMem
   if (!(atomicMask == 0xf || atomicMask == 0xf0 || atomicMask == 0xff)) {
-    printf("Mask f**ked: %lx\n", atomicMask);
+    printf("Unrecognized mask: %lx\n", atomicMask);
+    return 1;
   }
-  assert(atomicMask == 0xf || atomicMask == 0xf0 || atomicMask == 0xff);
 
   if (atomicMask == 0xff) {
     uint64_t rs = atomicData;  // rs2
-    uint64_t t  = atomicOut;
+    uint64_t t  = atomicOut;   // original value
     uint64_t ret;
     uint64_t mem;
     read_goldenmem(atomicAddr, &mem, 8);
-    if (mem != t && atomicFuop != 007 && atomicFuop != 003) {
-      printf("Atomic instr f**ked up, mem: 0x%lx, t: 0x%lx, op: 0x%x, addr: 0x%lx\n", mem, t, atomicFuop, atomicAddr);
-      // assert(0);
+    if (mem != t && atomicFuop != 007 && atomicFuop != 003) {  // ignore sc_d & lr_d
+      printf("Core %d atomic instr mismatch goldenMem, mem: 0x%lx, t: 0x%lx, op: 0x%x, addr: 0x%lx\n", coreid, mem, t, atomicFuop, atomicAddr);
+      return 1;
     }
     switch (atomicFuop) {
       case 002: case 003: ret = t; break;
@@ -326,24 +330,25 @@ inline void handle_atomic(uint64_t atomicAddr, uint64_t atomicData, uint64_t ato
     }
     update_goldenmem(atomicAddr, &ret, atomicMask, 8);
   }
+
   if (atomicMask == 0xf || atomicMask == 0xf0) {
     uint32_t rs = (uint32_t)atomicData;  // rs2
-    uint32_t t  = (uint32_t)atomicOut;
+    uint32_t t  = (uint32_t)atomicOut;   // original value
     uint32_t ret;
     uint32_t mem;
-    uint64_t mem_temp;
-    uint64_t ret_temp;
+    uint64_t mem_raw;
+    uint64_t ret_sel;
     atomicAddr = (atomicAddr & 0xfffffffffffffff8);
-    read_goldenmem(atomicAddr, &mem_temp, 8);
+    read_goldenmem(atomicAddr, &mem_raw, 8);
 
     if (atomicMask == 0xf)
-      mem = (uint32_t)mem_temp;
+      mem = (uint32_t)mem_raw;
     else
-      mem = (uint32_t)(mem_temp >> 32);
+      mem = (uint32_t)(mem_raw >> 32);
 
-    if (mem != t && atomicFuop != 006 && atomicFuop != 002) {
-      printf("Atomic instr f**ked up, rawmem: 0x%lx mem: 0x%x, t: 0x%x, op: 0x%x, addr: 0x%lx\n", mem_temp, mem, t, atomicFuop, atomicAddr);
-      // assert(0);
+    if (mem != t && atomicFuop != 006 && atomicFuop != 002) {  // ignore sc_w & lr_w
+      printf("Core %d atomic instr mismatch goldenMem, rawmem: 0x%lx mem: 0x%x, t: 0x%x, op: 0x%x, addr: 0x%lx\n", coreid, mem_raw, mem, t, atomicFuop, atomicAddr);
+      return 1;
     }
     switch (atomicFuop) {
       case 002: case 003: ret = t; break;
@@ -359,25 +364,25 @@ inline void handle_atomic(uint64_t atomicAddr, uint64_t atomicData, uint64_t ato
       case 052: case 053: ret = (t > rs) ? t : rs; break;
       default: printf("Unknown atomic fuOpType: 0x%x\n", atomicFuop);
     }
-    ret_temp = ret;
+    ret_sel = ret;
     if (atomicMask == 0xf0)
-      ret_temp = (ret_temp << 32);
-    update_goldenmem(atomicAddr, &ret_temp, atomicMask, 8);
+      ret_sel = (ret_sel << 32);
+    update_goldenmem(atomicAddr, &ret_sel, atomicMask, 8);
   }
-
+  return 0;
 }
 
-int Difftest::do_golden_memory_check() {
+int Difftest::do_golden_memory_update() {
   // Update Golden Memory info
-  // if (dut.sbuffer.resp) {
-  //   update_goldenmem(dut.sbuffer.addr, dut.sbuffer.data, dut.sbuffer.mask, 64);
-  // }
-
-  // if (dut.atomic.resp) {
-  //   handle_atomic(dut.atomic.addr, dut.atomic.data, dut.atomic.mask, dut.atomic.fuop, dut.atomic.out);
-  // }
-
-  // TODO
+  if (dut.sbuffer.resp) {
+    dut.sbuffer.resp = 0;
+    update_goldenmem(dut.sbuffer.addr, dut.sbuffer.data, dut.sbuffer.mask, 64);
+  }
+  if (dut.atomic.resp) {
+    dut.atomic.resp = 0;
+    int ret = handle_atomic(id, dut.atomic.addr, dut.atomic.data, dut.atomic.mask, dut.atomic.fuop, dut.atomic.out);
+    if (ret) return ret;
+  }
   return 0;
 }
 
@@ -428,7 +433,7 @@ void Difftest::clear_step() {
 }
 
 void Difftest::display() {
-  state->display();
+  state->display(this->id);
 
   printf("\n==============  REF Regs  ==============\n");
   fflush(stdout);
@@ -436,8 +441,8 @@ void Difftest::display() {
   printf("priviledgeMode: %lu\n", dut.csr.priviledgeMode);
 }
 
-void DiffState::display() {
-  printf("\n============== Commit Group Trace ==============\n");
+void DiffState::display(int coreid) {
+  printf("\n============== Commit Group Trace (Core %d) ==============\n", coreid);
   for (int j = 0; j < DEBUG_GROUP_TRACE_SIZE; j++) {
     printf("commit group [%x]: pc %010lx cmtcnt %d %s\n",
         j, retire_group_pc_queue[j], retire_group_cnt_queue[j],

src/test/csrc/difftest/difftest.h

@@ -153,7 +153,7 @@ public:
     retire_inst_type_queue[retire_inst_pointer] = abnormal_type;
     retire_inst_pointer = (retire_inst_pointer + 1) % DEBUG_INST_TRACE_SIZE;
   };
-  void display();
+  void display(int coreid);
 
 private:
   const static size_t DEBUG_GROUP_TRACE_SIZE = 16;
@@ -178,6 +178,7 @@ public:
   // Its backend should be cross-platform (NEMU, Spike, ...)
   // Initialize difftest environments
   Difftest(int coreid);
+  DIFF_PROXY *proxy;
   uint32_t num_commit = 0; // # of commits if made progress
   // Trigger a difftest checking procdure
   int step();
@@ -229,7 +230,6 @@ private:
   int id;
   difftest_core_state_t dut;
   difftest_core_state_t ref;
-  DIFF_PROXY *proxy;
   uint64_t *ref_regs_ptr = (uint64_t*)&ref.regs;
   uint64_t *dut_regs_ptr = (uint64_t*)&dut.regs;
 
@@ -248,7 +248,7 @@ private:
   void do_exception();
   void do_instr_commit(int index);
   int do_store_check();
-  int do_golden_memory_check();
+  int do_golden_memory_update();
   // inline uint64_t *ref_regs_ptr() { return (uint64_t*)&ref.regs; }
   // inline uint64_t *dut_regs_ptr() { return (uint64_t*)&dut.regs; }
 

src/test/csrc/difftest/nemuproxy.cpp

@@ -66,11 +66,12 @@ NemuProxy::NemuProxy(int coreid) {
   assert(isa_reg_display);
 
   auto nemu_difftest_set_mhartid = (void (*)(int))dlsym(handle, "difftest_set_mhartid");
-  // assert(nemu_difftest_set_mhartid);
   auto nemu_misc_put_gmaddr = (void (*)(void*))dlsym(handle, "misc_put_gmaddr");
-  // assert(nemu_misc_put_gmaddr);
-  auto nemu_init = (void (*)(void))dlsym(handle, "difftest_init");
-  assert(nemu_init);
+
+  if (EMU_CORES > 1) {
+    assert(nemu_difftest_set_mhartid);
+    assert(nemu_misc_put_gmaddr);
+  }
 
   if (nemu_difftest_set_mhartid) {
     nemu_difftest_set_mhartid(coreid);
@@ -78,6 +79,10 @@ NemuProxy::NemuProxy(int coreid) {
   if (nemu_misc_put_gmaddr) {
     nemu_misc_put_gmaddr(goldenMem);
   }
+
+  auto nemu_init = (void (*)(void))dlsym(handle, "difftest_init");
+  assert(nemu_init);
+
   nemu_init();
 }
 

src/test/csrc/verilator/emu.cpp

@@ -271,7 +271,7 @@ uint64_t Emulator::execute(uint64_t max_cycle, uint64_t max_instr) {
     dut_ptr->io_perfInfo_clean = 0;
     dut_ptr->io_perfInfo_dump = 0;
 
-    // Naive instr cnt, dual core is not supported
+    // Naive instr cnt per core
     for (int i = 0; i < EMU_CORES; i++) {
       // update instr_cnt
       uint64_t commit_count = (core_max_instr[i] >= difftest[i]->num_commit) ? difftest[i]->num_commit : core_max_instr[i];

src/test/csrc/verilator/main.cpp

@@ -25,9 +25,9 @@ int main(int argc, const char** argv) {
   }
 
   auto emu = new Emulator(argc, argv);
+  init_goldenmem();
   difftest_init();
 
-  init_goldenmem();
   // init device
   init_device();