[DRAMsys] DRAM analyze support for MemPool

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "toolchain/riscv-opcodes"]
 	path = toolchain/riscv-opcodes
 	url = https://github.com/pulp-platform/riscv-opcodes.git
+[submodule "hardware/dramsys/DRAMSys"]
+	path = hardware/dramsys/DRAMSys
+	url = https://github.com/tukl-msd/DRAMSys.git

Bender.yml

@@ -63,6 +63,16 @@ sources:
       # Level 2
       - hardware/tb/mempool_tb_verilator.sv
 
+  - target: mempool_dramsys
+    files:
+      # Level 1
+      - hardware/tb/axi_uart.sv
+      - hardware/tb/traffic_generator.sv
+      - hardware/dramsys/src/mempool_system_to_dram.sv
+      - hardware/dramsys/src/simulation_ctrl.sv
+      # Level 2
+      - hardware/dramsys/src/mempool_dramsys_tb.sv
+
   - target: fpga
     files:
       # Level 1

Makefile

@@ -22,6 +22,7 @@ LLVM_INSTALL_DIR      ?= ${INSTALL_DIR}/llvm
 HALIDE_INSTALL_DIR    ?= ${INSTALL_DIR}/halide
 BENDER_INSTALL_DIR    ?= ${INSTALL_DIR}/bender
 VERILATOR_INSTALL_DIR ?= ${INSTALL_DIR}/verilator
+VERILATOR_DRAMSYS_INSTALL_DIR ?= ${INSTALL_DIR}/verilator_dramsys
 RISCV_TESTS_DIR       ?= ${ROOT_DIR}/${SOFTWARE_DIR}/riscv-tests
 
 CMAKE ?= cmake
@@ -146,6 +147,16 @@ $(VERILATOR_INSTALL_DIR)/bin/verilator: toolchain/verilator Makefile
 	autoconf && CC=$(CLANG_CC) CXX=$(CLANG_CXX) CXXFLAGS=$(CLANG_CXXFLAGS) LDFLAGS=$(CLANG_LDFLAGS) ./configure --prefix=$(VERILATOR_INSTALL_DIR) $(VERILATOR_CI) && \
 	make -j4 && make install
 
+verilator_dramsys:$(VERILATOR_DRAMSYS_INSTALL_DIR)/bin/verilator
+$(VERILATOR_DRAMSYS_INSTALL_DIR)/bin/verilator: toolchain/verilator Makefile
+	cd $<;\
+	autoconf && CC=$(CLANG_CC) CXX=$(CLANG_CXX) CXXFLAGS=$(CLANG_CXXFLAGS) LDFLAGS=$(CLANG_LDFLAGS) ./configure --prefix=$(VERILATOR_DRAMSYS_INSTALL_DIR) && \
+	make -j4 && make install
+	sed -i "/CXX =/c\CXX = g++" $(VERILATOR_DRAMSYS_INSTALL_DIR)/share/verilator/include/verilated.mk
+	sed -i "/LINK =/c\LINK = g++" $(VERILATOR_DRAMSYS_INSTALL_DIR)/share/verilator/include/verilated.mk
+	sed -i "/CFG_CXXFLAGS_STD_NEWEST =/c\CFG_CXXFLAGS_STD_NEWEST = -std=gnu++17" $(VERILATOR_DRAMSYS_INSTALL_DIR)/share/verilator/include/verilated.mk
+	sed -i "/CFG_CXXFLAGS_NO_UNUSED =/c\CFG_CXXFLAGS_NO_UNUSED =  -faligned-new -fcf-protection=none -Wno-bool-operation -Wno-tautological-bitwise-compare -Wno-parentheses-equality -Wno-sign-compare -Wno-uninitialized -Wno-unused-parameter -Wno-unused-variable -Wno-shadow" $(VERILATOR_DRAMSYS_INSTALL_DIR)/share/verilator/include/verilated.mk
+
 # Update and patch hardware dependencies for MemPool
 # Previous changes will be stashed. Clear all the stashes with `git stash clear`
 .PHONY: update-deps

hardware/Makefile

@@ -46,7 +46,7 @@ verilator_top   ?= mempool_tb_verilator
 # Python
 python          ?= python3
 # Enable tracing
-snitch_trace    ?= 0
+snitch_trace    ?= 1
 
 # Check if the specified QuestaSim version exists
 ifeq (, $(shell which $(questa_cmd)))
@@ -65,7 +65,7 @@ ifdef preload
 	questa_args += +PRELOAD=$(preload)
 endif
 questa_args += -sv_lib $(dpi_library)/mempool_dpi -work $(library)
-questa_args += -suppress vsim-12070
+questa_args += -suppress vsim-12070 -suppress vsim-3009
 
 # VCS options
 vlogan_args += -assert svaext +v2k -override_timescale=1ns/1ps -kdb
@@ -271,6 +271,93 @@ verilate: $(VERILATOR_EXE) $(buildpath) Makefile
 	# Avoid capturing the return status when running the load-throughput analysis
 	if [ $(tg) -ne 1 ]; then ./scripts/return_status.sh $(buildpath)/transcript; fi
 
+#########################
+# DRAMsys Co-simulation #
+#########################
+
+DRAMSYS_GIT_ROOT := $(ROOT_DIR)/dramsys/DRAMSys
+DRAMSYS_SRC_ROOT := $(ROOT_DIR)/dramsys/src
+DRAMSYS_BUILD_ROOT := $(ROOT_DIR)/dramsys/build
+DRAMSYS_BUILD_LIB := $(ROOT_DIR)/dramsys/build/dramsys_library
+DRAMSYS_BUILD_MEMPOOL_LIB := $(ROOT_DIR)/dramsys/build/mempool_library
+DRAMSYS_BUILD_SIMULATION := $(ROOT_DIR)/dramsys/build/main_program
+DRAMSYS_BUILD_SIMULATION_EXE := $(ROOT_DIR)/dramsys/build/main_program/simulator/DRAMSys
+
+dramsys_init:
+	# initialize DRAMsys submodules
+	cd $(DRAMSYS_GIT_ROOT) && git submodule update --init --recursive
+	# build folder
+	rm -rf $(DRAMSYS_BUILD_ROOT); mkdir -p $(DRAMSYS_BUILD_ROOT)
+	# apply DRAMsys patch
+	cd $(DRAMSYS_GIT_ROOT) && git apply ../patch/dramsys.patch
+	# build dramsys library
+	mkdir -p $(DRAMSYS_BUILD_LIB) && cd $(DRAMSYS_BUILD_LIB) && cmake -DENABLE_PHASE_CALLBACKS_TRACING=OFF $(DRAMSYS_GIT_ROOT)/DRAMSys/library/ && make
+	# modify dramsys simulator
+	cp $(DRAMSYS_SRC_ROOT)/AXI4_to_TLM.h $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator
+	cp $(DRAMSYS_SRC_ROOT)/axi_sc_mem.h $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator
+	cp $(DRAMSYS_SRC_ROOT)/elfloader.cpp $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator
+	cp $(DRAMSYS_SRC_ROOT)/elfloader.h $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator
+	rm $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator/main.cpp
+	cp $(DRAMSYS_SRC_ROOT)/main.cpp $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator
+	# prepare the folder for dramsys-mempool co-simulation main program
+	rm -rf $(DRAMSYS_BUILD_SIMULATION); mkdir -p $(DRAMSYS_BUILD_SIMULATION)
+
+#systemc library path
+systemc_lib_path ?= $(abspath $(DRAMSYS_BUILD_LIB)/src/common/third_party/systemc/src)
+#systemc include path
+systemc_include_path ?= $(abspath $(DRAMSYS_GIT_ROOT)/DRAMSys/library/src/common/third_party/systemc/src)
+
+verilator_dramsys       ?= $(INSTALL_DIR)/verilator_dramsys/bin/verilator
+verilator_dramsys_build ?= $(DRAMSYS_BUILD_MEMPOOL_LIB)
+verilator_dramsys_files ?= $(verilator_dramsys_build)/files
+verilator_dramsys_top   ?= mempool_dramsys_tb
+
+VERILATOR_DRAMSYS_SRC   := $(DRAMSYS_SRC_ROOT)
+VERILATOR_DRAMSYS_EXE   := $(verilator_dramsys_build)/V$(verilator_dramsys_top)
+VERILATOR_DRAMSYS_MK    := $(VERILATOR_DRAMSYS_EXE).mk
+VERILATOR_DRAMSYS_WAIVE := $(shell find $(VERILATOR_DRAMSYS_SRC) -name "*.vlt" -print | sort)
+VERILATOR_DRAMSYS_CONF  := $(VERILATOR_DRAMSYS_SRC)/verilator.flags
+
+VERILATOR_DRAMSYS_FLAGS += -CFLAGS "-DTOPLEVEL_NAME=$(verilator_dramsys_top)"
+VERILATOR_DRAMSYS_FLAGS += --Mdir $(verilator_dramsys_build)
+VERILATOR_DRAMSYS_FLAGS += -f $(verilator_dramsys_files)
+VERILATOR_DRAMSYS_FLAGS += -f $(VERILATOR_DRAMSYS_CONF)
+VERILATOR_DRAMSYS_FLAGS += $(VERILATOR_DRAMSYS_WAIVE)
+
+VERILATOR_DRAMSYS_GROUP_MK    := $(verilator_dramsys_build)/Vmempool_group_a/Vmempool_group_a.mk
+VERILATOR_DRAMSYS_TILE_MK    	:= $(verilator_dramsys_build)/Vmempool_tile_f/Vmempool_tile_f.mk
+
+DRAMSYS_BUILD_MEMPOOL_LIB_FILE := $(DRAMSYS_BUILD_MEMPOOL_LIB)/V$(verilator_dramsys_top)__ALL.a
+
+$(VERILATOR_DRAMSYS_MK): $(VERILATOR_DRAMSYS_CONF) $(VERILATOR_DRAMSYS_WAIVE) $(MEMPOOL_DIR)/Bender.yml $(shell find $(VERILATOR_DRAMSYS_SRC) -type f) $(shell find {src,tb,deps} -type f) $(bender) $(config_mk) Makefile
+	rm -rf $(verilator_dramsys_build); mkdir -p $(verilator_dramsys_build)
+	# Create Bender script of all RTL files
+	$(bender) script verilator $(vlog_defs) -t rtl -t mempool_dramsys > $(verilator_dramsys_files)
+	# SystemC Envirnomnet adn Create Verilator Makefile
+	export SYSTEMC_INCLUDE=$(systemc_include_path) && \
+	export SYSTEMC_LIBDIR=$(systemc_lib_path) && \
+	$(verilator_dramsys) $(VERILATOR_DRAMSYS_FLAGS) --trace --top-module $(verilator_dramsys_top)
+	#modify .mk files
+	sed -i "/VERILATOR_ROOT =/c\VERILATOR_ROOT = $(INSTALL_DIR)/verilator_dramsys/share/verilator" $(VERILATOR_DRAMSYS_MK)
+	sed -i "/VERILATOR_ROOT =/c\VERILATOR_ROOT = $(INSTALL_DIR)/verilator_dramsys/share/verilator" $(VERILATOR_DRAMSYS_GROUP_MK)
+	sed -i "/VERILATOR_ROOT =/c\VERILATOR_ROOT = $(INSTALL_DIR)/verilator_dramsys/share/verilator" $(VERILATOR_DRAMSYS_TILE_MK)
+	#build library
+	cd $(verilator_dramsys_build) && make -f V$(verilator_dramsys_top).mk
+	#modify Cmakefile for dramsys simulator
+	sed -i "/__ALL.a/c\    $(DRAMSYS_BUILD_MEMPOOL_LIB_FILE)" $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator/CMakeLists.txt
+
+$(DRAMSYS_BUILD_SIMULATION_EXE): $(shell find $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator -type f) Makefile $(VERILATOR_DRAMSYS_MK)
+	cd $(DRAMSYS_BUILD_SIMULATION) && cmake -DENABLE_PHASE_CALLBACKS_TRACING=OFF $(DRAMSYS_GIT_ROOT)/DRAMSys && make
+
+dramsys_compile: $(DRAMSYS_BUILD_SIMULATION_EXE) $(shell find $(DRAMSYS_GIT_ROOT)/DRAMSys/simulator -type f) Makefile
+	echo "Mempool + DRAMSys co-simulation is well prepared!"
+
+DRAMSYS_SIMULATION_RESOURCES_PATH := $(DRAMSYS_GIT_ROOT)/DRAMSys/library/resources/simulations
+dram ?= ddr3
+
+dramsys_sim: dramsys_compile
+	cd $(DRAMSYS_BUILD_SIMULATION)/simulator && ./DRAMSys $(preload) $(DRAMSYS_SIMULATION_RESOURCES_PATH)/$(dram)-example.json
+
 #############
 # Lint      #
 #############

hardware/dramsys/README.md

@@ -0,0 +1,70 @@
+# DRAMSys co-simulation
+
+DRAMSys is a DRAM simulation and analyze tool writing in SystemC. Here we co-simulate MemPool with DRAMSys.
+
+## Structure
+
+- `DRAMSys` is the DRAMSys repo as a submodule to MemPool
+- `Patch` contains modification patch on DRAMSys
+- `Patch` contains python script to show DRAM analyze results
+- `src` contains important source files for co-simulation
+
+## Getting Start
+
+Make sure you have already:
+- initilazed MemPool repo
+- located at the root folder of MemPool repo.
+- compiled some applications from `software/app`
+
+### Initialize DRAMSys
+
+Start executing these commands below:
+
+```bash
+make verilator_dramsys
+cd hardware
+make dramsys_init && make dramsys_compile
+```
+
+### Run DRAMSys Co-simulation
+
+Supposing you're at the `hardware` folder, execute the commands below to run simulation
+
+```bash
+app=hello_world dram=ddr3 make dramsys_sim
+```
+
+Where `app` is the name of your compiled application, and `dram` states the type of DRAM. right now the supported dram types are:
+- ddr3
+- ddr4
+- lpddr4
+- hbm2
+
+### Show Simulation Results
+
+After simulation, the performance of DRAM will be shown on the console, including `bandwidth` and `power` (Noting that power analysis on works on `ddr3` and `ddr4`). Meanwhile, a database file `xxx.tdb` will be dumpped out under `dramsys/build/main_program/simulator` folder 
+
+Using our python script to show the DRAM `bandwidth` and `power` fluctuation with time
+```bash
+python3 dramsys/scripts/dramsys_plot.py  dramsys/build/main_program/simulator/xxx.tdb
+```
+
+## Modification 
+
+### MemPool System
+
+The new construction of MemPool System connecting to DRAM is `dramsys/src/mempool_system_to_dram.sv`
+The top-level for co-simulation is `dramsys/src/mempool_dramsys_tb.sv`
+
+If you want to modifiy the MemPool System level construction, please feel free to change `dramsys/src/mempool_system_to_dram.sv` while keeping its interface, especially the AXI4 port to DRAM, and do not modify the `dramsys/src/mempool_dramsys_tb.sv`. 
+
+### DRAMSys Simulation Parameters
+
+For each type of DRAM, DRAMSys defines their simulation parameters in json files under folder `dramsys/DRAMSys/DRAMSys/library/resources/config/simulator`. For example, you can modify the parameter `WindowSize` from 1000 to 100, and the final results will have higher sample rates to record DRAM metrics.
+
+window size = 1000
+![W=1000](./figures/1000window.png)
+
+window size = 100
+![W=100](./figures/100window.png)
+