Merge pull request #348 from RISCVERS/coverage

Coverage

scripts/coverage/coverage.py

@@ -11,6 +11,8 @@ if __name__ == "__main__":
     lines = []
     line_count = 0
     synthesis_nest_level = 0
+    reg_init_nest_level = 0
+    mem_init_nest_level = 0
     with open(input_file) as f:
         for line in f:
             line_count += 1
@@ -18,30 +20,72 @@ if __name__ == "__main__":
             ifdef = re.compile('`ifdef')
             ifndef = re.compile('`ifndef')
             endif = re.compile('`endif')
+            # remove the line coverage results of not synthesizable code(mostly assert and fwrite)
             synthesis = re.compile('`ifndef SYNTHESIS')
-            line_coverage = re.compile('^\s*([%]?\d+)\s+if')
+            # remove the coverage results of random init variables
+            reg_init = re.compile('`ifdef RANDOMIZE_REG_INIT')
+            mem_init = re.compile('`ifdef RANDOMIZE_MEM_INIT')
+            coverage = re.compile('^\s*(%?\d+)\s+')
+
 
             ifdef_match = ifdef.search(line)
             ifndef_match = ifndef.search(line)
             endif_match = endif.search(line)
             synthesis_match = synthesis.search(line)
-            line_coverage_match = line_coverage.search(line)
+            reg_init_match = reg_init.search(line)
+            mem_init_match = mem_init.search(line)
+            coverage_match = coverage.search(line)
             
             # enter synthesis block
             if synthesis_match:
                 assert synthesis_nest_level == 0, "Should not nest SYNTHESIS macro"
                 synthesis_nest_level = 1
 
-            if ifdef_match or (ifndef_match and not synthesis_match):
-                synthesis_nest_level += 1
-            if endif_match:
-                synthesis_nest_level -= 1
-                assert synthesis_nest_level >= 0, "Macro nest level should be >= 0"
+            if synthesis_nest_level > 0:
+                if ifdef_match or (ifndef_match and not synthesis_match):
+                    synthesis_nest_level += 1
+                if endif_match:
+                    synthesis_nest_level -= 1
+                    assert synthesis_nest_level >= 0, "Macro nest level should be >= 0"
+
+                # remove line coverage results in systhesis block
+                if coverage_match:
+                    coverage_stat = coverage_match.group(1)
+                    line = line.replace(coverage_match.group(1), " " * len(coverage_stat))
+
+            # enter reg_init block
+            if reg_init_match:
+                assert reg_init_nest_level == 0, "Should not nest reg_init macro"
+                reg_init_nest_level = 1
+
+            if reg_init_nest_level > 0:
+                if (ifdef_match and not reg_init_match) or ifndef_match:
+                    reg_init_nest_level += 1
+                if endif_match:
+                    reg_init_nest_level -= 1
+                    assert reg_init_nest_level >= 0, "Macro nest level should be >= 0"
+
+                # remove line coverage results in systhesis block
+                if coverage_match:
+                    coverage_stat = coverage_match.group(1)
+                    line = line.replace(coverage_match.group(1), " " * len(coverage_stat))
+
+            # enter mem_init block
+            if mem_init_match:
+                assert mem_init_nest_level == 0, "Should not nest mem_init macro"
+                mem_init_nest_level = 1
+
+            if mem_init_nest_level > 0:
+                if (ifdef_match and not mem_init_match) or ifndef_match:
+                    mem_init_nest_level += 1
+                if endif_match:
+                    mem_init_nest_level -= 1
+                    assert mem_init_nest_level >= 0, "Macro nest level should be >= 0"
 
-            # remove line coverage results in systhesis block
-            if synthesis_nest_level > 0 and line_coverage_match:
-                coverage_stat = line_coverage_match.group(1)
-                line = line.replace(line_coverage_match.group(1), " " * len(coverage_stat))
+                # remove line coverage results in systhesis block
+                if coverage_match:
+                    coverage_stat = coverage_match.group(1)
+                    line = line.replace(coverage_match.group(1), " " * len(coverage_stat))
 
             lines += line
 

scripts/coverage/statistics.py

@@ -5,19 +5,31 @@ import re
 import copy
 import pprint
 
-COVERRED = "COVERRED"
-NOT_COVERRED = "NOT_COVERRED"
+LINE_COVERRED = "LINE_COVERRED"
+NOT_LINE_COVERRED = "NOT_LINE_COVERRED"
+TOGGLE_COVERRED = "TOGGLE_COVERRED"
+NOT_TOGGLE_COVERRED = "NOT_TOGGLE_COVERRED"
 DONTCARE = "DONTCARE"
+
 BEGIN = "BEGIN"
 END = "END"
 CHILDREN = "CHILDREN"
 MODULE = "MODULE"
 INSTANCE = "INSTANCE"
-TYPE="TYPE"
-ROOT="ROOT"
-NODE="NODE"
-SELFCOVERAGE="SELFCOVERAGE"
-TREECOVERAGE="TREECOVERAGE"
+TYPE = "TYPE"
+ROOT = "ROOT"
+NODE = "NODE"
+SELFCOVERAGE = "SELFCOVERAGE"
+TREECOVERAGE = "TREECOVERAGE"
+LINECOVERAGE = 0 
+TOGGLECOVERAGE = 1
+
+def check_one_hot(l):
+    cnt = 0
+    for e in l:
+        if e:
+            cnt += 1
+    return cnt <= 1
 
 def get_lines(input_file):
     lines = []
@@ -31,41 +43,85 @@ def get_line_annotation(lines):
     # pattern_1: 040192     if(array_0_MPORT_en & array_0_MPORT_mask) begin
     # pattern_2: 2218110        end else if (_T_30) begin // @[Conditional.scala 40:58]
     # pattern_2: 000417     end else begin
-    coverred_pattern_1 = re.compile('^\s*(\d+)\s+if')
-    coverred_pattern_2 = re.compile('^\s*(\d+)\s+end else')
-    not_coverred_pattern_1 = re.compile('^\s*(%0+)\s+if')
-    not_coverred_pattern_2 = re.compile('^\s*(%0+)\s+end else')
+    line_coverred_pattern_1 = re.compile('^\s*(\d+)\s+if')
+    line_coverred_pattern_2 = re.compile('^\s*(\d+)\s+end else')
+    not_line_coverred_pattern_1 = re.compile('^\s*(%0+)\s+if')
+    not_line_coverred_pattern_2 = re.compile('^\s*(%0+)\s+end else')
+
+    toggle_coverred_pattern_1 = re.compile('^\s*(\d+)\s+reg')
+    toggle_coverred_pattern_2 = re.compile('^\s*(\d+)\s+wire')
+    toggle_coverred_pattern_3 = re.compile('^\s*(\d+)\s+input')
+    toggle_coverred_pattern_4 = re.compile('^\s*(\d+)\s+output')
+
+    not_toggle_coverred_pattern_1 = re.compile('^\s*(%0+)\s+reg')
+    not_toggle_coverred_pattern_2 = re.compile('^\s*(%0+)\s+wire')
+    not_toggle_coverred_pattern_3 = re.compile('^\s*(%0+)\s+input')
+    not_toggle_coverred_pattern_4 = re.compile('^\s*(%0+)\s+output')
+
+    line_cnt = 0
 
     for line in lines:
-        coverred_match = coverred_pattern_1.search(line) or coverred_pattern_2.search(line)
-        not_coverred_match = not_coverred_pattern_1.search(line) or not_coverred_pattern_2.search(line)
+        line_coverred_match = line_coverred_pattern_1.search(line) or line_coverred_pattern_2.search(line)
+        not_line_coverred_match = not_line_coverred_pattern_1.search(line) or not_line_coverred_pattern_2.search(line)
+
+        assert not (line_coverred_match and not_line_coverred_match)
 
-        assert not (coverred_match and not_coverred_match)
+        toggle_coverred_match = toggle_coverred_pattern_1.search(line) or toggle_coverred_pattern_2.search(line) or \
+                toggle_coverred_pattern_3.search(line) or toggle_coverred_pattern_4.search(line)
+        not_toggle_coverred_match = not_toggle_coverred_pattern_1.search(line) or not_toggle_coverred_pattern_2.search(line) or \
+                not_toggle_coverred_pattern_3.search(line) or not_toggle_coverred_pattern_4.search(line)
+
+        assert not (toggle_coverred_match and not_toggle_coverred_match)
         
-        if coverred_match:
-            line_annotations.append(COVERRED)
-        elif not_coverred_match:
-            line_annotations.append(NOT_COVERRED)
+        all_match = (line_coverred_match, not_line_coverred_match,
+                toggle_coverred_match, not_toggle_coverred_match)
+        if not check_one_hot(all_match):
+            print("not_one_hot")
+            print(line_cnt)
+            print(all_match)
+            assert False, "This line matches multiple patterns"
+        if line_coverred_match:
+            line_annotations.append(LINE_COVERRED)
+        elif not_line_coverred_match:
+            line_annotations.append(NOT_LINE_COVERRED)
+        elif toggle_coverred_match:
+            line_annotations.append(TOGGLE_COVERRED)
+        elif not_toggle_coverred_match:
+            line_annotations.append(NOT_TOGGLE_COVERRED)
         else:
             line_annotations.append(DONTCARE)
+        line_cnt += 1
     return line_annotations
 
 # get the line coverage statistics in line range [start, end)
 def get_coverage_statistics(line_annotations, start, end):
-    coverred = 0
-    not_coverred = 0
+    line_coverred = 0
+    not_line_coverred = 0
+    toggle_coverred = 0
+    not_toggle_coverred = 0
     for i in range(start, end):
-        if line_annotations[i] == COVERRED:
-            coverred += 1
+        if line_annotations[i] == LINE_COVERRED:
+            line_coverred += 1
+
+        if line_annotations[i] == NOT_LINE_COVERRED:
+            not_line_coverred += 1
+
+        if line_annotations[i] == TOGGLE_COVERRED:
+            toggle_coverred += 1
 
-        if line_annotations[i] == NOT_COVERRED:
-            not_coverred += 1
+        if line_annotations[i] == NOT_TOGGLE_COVERRED:
+            not_toggle_coverred += 1
 
     # deal with divide by zero
-    coverage = 1.0
-    if coverred + not_coverred != 0:
-        coverage = float(coverred) / (coverred + not_coverred)
-    return (coverred, not_coverred, coverage)
+    line_coverage = 1.0
+    if line_coverred + not_line_coverred != 0:
+        line_coverage = float(line_coverred) / (line_coverred + not_line_coverred)
+
+    toggle_coverage = 1.0
+    if toggle_coverred + not_toggle_coverred != 0:
+        toggle_coverage = float(toggle_coverred) / (toggle_coverred + not_toggle_coverred)
+    return ((line_coverred, not_line_coverred, line_coverage),
+            (toggle_coverred, not_toggle_coverred, toggle_coverage))
 
 # get modules and all it's submodules
 def get_modules(lines):
@@ -140,18 +196,26 @@ def get_tree_coverage(modules, coverage):
             if CHILDREN not in modules[module]:
                 modules[module][TREECOVERAGE] = self_coverage
             else:
-                coverred = self_coverage[0]
-                not_coverred = self_coverage[1]
+                line_coverred = self_coverage[LINECOVERAGE][0]
+                not_line_coverred = self_coverage[LINECOVERAGE][1]
+                toggle_coverred = self_coverage[TOGGLECOVERAGE][0]
+                not_toggle_coverred = self_coverage[TOGGLECOVERAGE][1]
                 # the dfs part
                 for child in modules[module][CHILDREN]:
                     child_coverage = dfs(child[MODULE])
-                    coverred += child_coverage[0]
-                    not_coverred += child_coverage[1]
+                    line_coverred += child_coverage[LINECOVERAGE][0]
+                    not_line_coverred += child_coverage[LINECOVERAGE][1]
+                    toggle_coverred += child_coverage[TOGGLECOVERAGE][0]
+                    not_toggle_coverred += child_coverage[TOGGLECOVERAGE][1]
                 # deal with divide by zero
-                coverage = 1.0
-                if coverred + not_coverred != 0:
-                    coverage = float(coverred) / (coverred + not_coverred)
-                modules[module][TREECOVERAGE] = (coverred, not_coverred, coverage)
+                line_coverage = 1.0
+                if line_coverred + not_line_coverred != 0:
+                    line_coverage = float(line_coverred) / (line_coverred + not_line_coverred)
+                toggle_coverage = 1.0
+                if toggle_coverred + not_toggle_coverred != 0:
+                    toggle_coverage = float(toggle_coverred) / (toggle_coverred + not_toggle_coverred)
+                modules[module][TREECOVERAGE] = ((line_coverred, not_line_coverred, line_coverage),
+                        (toggle_coverred, not_toggle_coverred, toggle_coverage))
         return modules[module][TREECOVERAGE]
 
     for module in modules:
@@ -163,8 +227,8 @@ def get_tree_coverage(modules, coverage):
 
 # arg1: tree coverage results
 # arg2: coverage type
-def sort_coverage(coverage, coverage_type):
-    l = [(module, coverage[module][coverage_type])for module in coverage]
+def sort_coverage(coverage, self_or_tree, coverage_type):
+    l = [(module, coverage[module][self_or_tree][coverage_type])for module in coverage]
     l.sort(key=lambda x:x[1][2])
     return l
 
@@ -174,10 +238,15 @@ def print_tree_coverage(tree_coverage):
         tree = tree_coverage[module][TREECOVERAGE]
         self = tree_coverage[module][SELFCOVERAGE]
         print("  " * level + "- " + module)
-        print("  " * level + "  tree", end="")
-        print("(%d, %d, %.2f)" % (tree[0], tree[1], tree[2] * 100.0))
-        print("  " * level + "  self", end="")
-        print("(%d, %d, %.2f)" % (self[0], self[1], self[2] * 100.0))
+        print("  " * level + "  tree_line", end="")
+        print("(%d, %d, %.2f)" % (tree[LINECOVERAGE][0], tree[LINECOVERAGE][1], tree[LINECOVERAGE][2] * 100.0))
+        print("  " * level + "  self_line", end="")
+        print("(%d, %d, %.2f)" % (self[LINECOVERAGE][0], self[LINECOVERAGE][1], self[LINECOVERAGE][2] * 100.0))
+
+        print("  " * level + "  tree_toggle", end="")
+        print("(%d, %d, %.2f)" % (tree[TOGGLECOVERAGE][0], tree[TOGGLECOVERAGE][1], tree[TOGGLECOVERAGE][2] * 100.0))
+        print("  " * level + "  self_toggle", end="")
+        print("(%d, %d, %.2f)" % (self[TOGGLECOVERAGE][0], self[TOGGLECOVERAGE][1], self[TOGGLECOVERAGE][2] * 100.0))
 
         # print children nodes
         if CHILDREN in modules[module]:
@@ -215,11 +284,15 @@ if __name__ == "__main__":
     # print("tree_coverage:")
     # pp.pprint(tree_coverage)
 
-    print("SelfCoverage:")
-    pp.pprint(sort_coverage(tree_coverage, SELFCOVERAGE))
+    print("LineSelfCoverage:")
+    pp.pprint(sort_coverage(tree_coverage, SELFCOVERAGE, LINECOVERAGE))
+    print("LineTreeCoverage:")
+    pp.pprint(sort_coverage(tree_coverage, TREECOVERAGE, LINECOVERAGE))
 
-    print("TreeCoverage:")
-    pp.pprint(sort_coverage(tree_coverage, TREECOVERAGE))
+    print("ToggleSelfCoverage:")
+    pp.pprint(sort_coverage(tree_coverage, SELFCOVERAGE, TOGGLECOVERAGE))
+    print("ToggleTreeCoverage:")
+    pp.pprint(sort_coverage(tree_coverage, TREECOVERAGE, TOGGLECOVERAGE))
 
     print("AllCoverage:")
     print_tree_coverage(tree_coverage)

src/test/csrc/emu.cpp

@@ -2,6 +2,8 @@
 #include "sdcard.h"
 #include "difftest.h"
 #include <getopt.h>
+#include<signal.h>
+#include<unistd.h>
 #include "ram.h"
 #include "zlib.h"
 #include "compress.h"
@@ -244,6 +246,16 @@ inline void Emulator::single_cycle() {
   cycles ++;
 }
 
+#if VM_COVERAGE == 1
+uint64_t *max_cycle_ptr = NULL;
+// when interrupted, we set max_cycle to zero
+// so that the emulator will stop gracefully
+void sig_handler(int signo) {
+  if (signo == SIGINT)
+    *max_cycle_ptr = 0;
+}
+#endif
+
 uint64_t Emulator::execute(uint64_t max_cycle, uint64_t max_instr) {
   extern void poll_event(void);
   extern uint32_t uptime(void);
@@ -268,6 +280,9 @@ uint64_t Emulator::execute(uint64_t max_cycle, uint64_t max_instr) {
   // since we are not sure when an emu will stop
   // we distinguish multiple dat files by emu start time
   time_t start_time = time(NULL);
+  max_cycle_ptr = &max_cycle;
+  if (signal(SIGINT, sig_handler) == SIG_ERR)
+    printf("\ncan't catch SIGINT\n");
 #endif
 
   while (!Verilated::gotFinish() && trapCode == STATE_RUNNING) {

src/test/scala/cache/L2CacheTest.scala

@@ -5,6 +5,7 @@ import chisel3._
 import chisel3.util._
 import chiseltest.experimental.TestOptionBuilder._
 import chiseltest.internal.{VerilatorBackendAnnotation, LineCoverageAnnotation, ToggleCoverageAnnotation, UserCoverageAnnotation, StructuralCoverageAnnotation}
+import chiseltest.legacy.backends.verilator.VerilatorFlags
 import chiseltest._
 import chisel3.experimental.BundleLiterals._
 import firrtl.stage.RunFirrtlTransformAnnotation
@@ -262,6 +263,7 @@ class L2CacheTest extends AnyFlatSpec with ChiselScalatestTester with Matchers{
     ToggleCoverageAnnotation,
     UserCoverageAnnotation,
     StructuralCoverageAnnotation,
+    VerilatorFlags(Seq("--output-split 5000", "--output-split-cfuncs 5000")),
     RunFirrtlTransformAnnotation(new PrintModuleName)
   )