refactor bencher to track storage changes

bencher/Cargo.toml

@@ -8,18 +8,21 @@ authors = ["Laminar Developers <[email protected]>"]
 edition = "2021"
 
 [dependencies]
-paste = "1.0"
+paste = "1.0.6"
 build-helper = { version = "0.1.1", optional = true }
-cargo_metadata = { version = "0.13.1", optional = true }
-tempfile = { version = "3.1.0", optional = true }
-toml = { version = "0.5.4", optional = true }
+cargo_metadata = { version = "0.14.1", optional = true }
+tempfile = { version = "3.2.0", optional = true }
+toml = { version = "0.5.8", optional = true }
 walkdir = { version = "2.3.1", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
 wasm-gc-api = { version = "0.1.11", optional = true }
 rand = {version = "0.8.3", optional = true }
 linregress = { version = "0.4.4", optional = true }
-serde = { version = "1.0.119", optional = true, features = ['derive'] }
-serde_json = {version = "1.0.64", optional = true }
+parking_lot = { version = "0.11.2", optional = true }
+serde = { version = "1.0.124", optional = true, features = ['derive'] }
+serde_json = {version = "1.0.68", optional = true }
+hash-db = { version = "0.15.2", default-features = false, optional = true }
+bencher-procedural = { path = "bencher-procedural", default-features = false }
 codec = { package = "parity-scale-codec", version = "2.3.1", features = ["derive"], default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false }
 sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false }
@@ -31,10 +34,13 @@ sc-executor-common = { git = "https://github.com/paritytech/substrate", branch =
 sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false, features = ["with-kvdb-rocksdb"], optional = true }
 sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false, optional = true }
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false }
+sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false }
+sp-storage = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.13", default-features = false, optional = true }
 
 [features]
 default = ["std"]
 std = [
+    "bencher-procedural/std",
     "build-helper",
     "cargo_metadata",
     "tempfile",
@@ -44,8 +50,10 @@ std = [
     "wasm-gc-api",
     "rand",
     "linregress",
+    "parking_lot",
     "serde/std",
     "serde_json/std",
+    "hash-db/std",
     "codec/std",
     "sp-core/std",
     "sp-std/std",
@@ -57,7 +65,9 @@ std = [
     "sc-client-db",
     "sp-maybe-compressed-blob",
     "frame-benchmarking/std",
+    "sp-externalities/std",
+    "sp-storage/std",
 ]
 bench = [
-    "sp-io/disable_panic_handler"
+    "sp-io/disable_panic_handler",
 ]

bencher/bencher-procedural/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "bencher-procedural"
+version = "0.1.0"
+license = "Apache-2.0"
+authors = ["Laminar Developers <[email protected]>"]
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1.0.6"
+quote = "1.0.3"
+syn = { version = "1.0.58", features = ["full"] }
+
+[features]
+default = ["std"]
+std = []

bencher/bencher-procedural/src/lib.rs

@@ -0,0 +1,18 @@
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn benchmarkable(_attr: TokenStream, item: TokenStream) -> TokenStream {
+	let syn::ItemFn { attrs, vis, sig, block } = syn::parse(item).unwrap();
+	(quote::quote! {
+		#(#attrs)*
+		#vis #sig {
+			#[cfg(not(feature = "std"))]
+			::orml_bencher::bench::before_block();
+			let result = #block;
+			#[cfg(not(feature = "std"))]
+			::orml_bencher::bench::after_block();
+			result
+		}
+	})
+	.into()
+}

bencher/src/bench_runner.rs

@@ -1,19 +1,32 @@
+use super::{
+	ext::BenchExt,
+	tracker::{BenchTracker, BenchTrackerExt},
+};
 use frame_benchmarking::frame_support::sp_runtime::traits::Block;
 use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutionMethod, WasmExecutor};
 use sc_executor_common::runtime_blob::RuntimeBlob;
+use sp_externalities::Extensions;
 use sp_state_machine::{Ext, OverlayedChanges, StorageTransactionCache};
+use sp_std::sync::Arc;
 
 /// Run benches
 pub fn run<B: Block>(wasm_code: Vec<u8>) -> std::result::Result<Vec<u8>, String> {
 	let mut overlay = OverlayedChanges::default();
 	let mut cache = StorageTransactionCache::default();
-	let state =
-		sc_client_db::BenchmarkingState::<B>::new(Default::default(), Default::default(), false, false).unwrap();
-	let mut ext = Ext::new(&mut overlay, &mut cache, &state, None);
+	let state = sc_client_db::BenchmarkingState::<B>::new(Default::default(), Default::default(), false, true).unwrap();
+
+	let tracker = Arc::new(BenchTracker::new());
+	let tracker_ext = BenchTrackerExt(Arc::clone(&tracker));
+
+	let mut extensions = Extensions::default();
+	extensions.register(tracker_ext);
+
+	let ext = Ext::<_, _>::new(&mut overlay, &mut cache, &state, Some(&mut extensions));
+	let mut bench_ext = BenchExt::new(ext, tracker);
 
 	let mut host_functions = sp_io::SubstrateHostFunctions::host_functions();
 	host_functions.append(&mut frame_benchmarking::benchmarking::HostFunctions::host_functions());
-	host_functions.append(&mut super::bencher::HostFunctions::host_functions());
+	host_functions.append(&mut super::bench::HostFunctions::host_functions());
 
 	let executor = WasmExecutor::new(
 		WasmExecutionMethod::Compiled,
@@ -25,5 +38,5 @@ pub fn run<B: Block>(wasm_code: Vec<u8>) -> std::result::Result<Vec<u8>, String>
 
 	let blob = RuntimeBlob::uncompress_if_needed(&wasm_code[..]).unwrap();
 
-	executor.uncached_call(blob, &mut ext, true, "run_benches", &[])
+	executor.uncached_call(blob, &mut bench_ext, false, "run_benches", &[])
 }

bencher/src/bencher.rs

@@ -0,0 +1,86 @@
+use codec::{Decode, Encode};
+use sp_std::prelude::Vec;
+
+#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
+pub struct BenchResult {
+	pub method: Vec<u8>,
+	pub elapses: Vec<u128>,
+	pub reads: u32,
+	pub writes: u32,
+	pub keys: Vec<u8>,
+}
+
+impl BenchResult {
+	pub fn with_name(name: &str) -> Self {
+		Self {
+			method: name.as_bytes().to_vec(),
+			..Default::default()
+		}
+	}
+}
+
+#[derive(Default)]
+pub struct Bencher {
+	pub current: BenchResult,
+	pub results: Vec<BenchResult>,
+}
+
+#[inline]
+fn black_box<T>(dummy: T) -> T {
+	let ret = unsafe { sp_std::ptr::read_volatile(&dummy) };
+	sp_std::mem::forget(dummy);
+	ret
+}
+
+#[allow(unused_variables, clippy::let_and_return)]
+impl Bencher {
+	pub fn whitelist(&mut self, key: Vec<u8>, read: bool, write: bool) {
+		#[cfg(not(feature = "std"))]
+		crate::bench::whitelist(key, read, write);
+	}
+
+	pub fn prepare(&self) {
+		#[cfg(not(feature = "std"))]
+		{
+			frame_benchmarking::benchmarking::commit_db();
+			frame_benchmarking::benchmarking::wipe_db();
+		}
+	}
+
+	pub fn bench<T, F>(&mut self, mut inner: F) -> T
+	where
+		F: FnMut() -> T,
+	{
+		#[cfg(not(feature = "std"))]
+		{
+			frame_benchmarking::benchmarking::commit_db();
+			frame_benchmarking::benchmarking::reset_read_write_count();
+			crate::bench::prepare();
+			crate::bench::instant();
+		}
+
+		let ret = black_box(inner());
+
+		#[cfg(not(feature = "std"))]
+		{
+			let elapsed = crate::bench::elapsed().saturating_sub(crate::bench::redundant_time());
+			self.current.elapses.push(elapsed);
+
+			frame_benchmarking::benchmarking::commit_db();
+			let (reads, _, written, _) = frame_benchmarking::benchmarking::read_write_count();
+
+			self.current.reads = reads;
+			self.current.writes = written;
+
+			// changed keys
+			self.current.keys = crate::bench::read_written_keys();
+		}
+
+		ret
+	}
+
+	pub fn print_warnings(&self, name: &str) {
+		#[cfg(not(feature = "std"))]
+		crate::bench::print_warnings(name.encode());
+	}
+}