Use a database and JSON file for persistent, multi-day, live metrics. (…

…#440)

* Use a database and JSON file for persistent, multi-day, live metrics.

* Metrics CI fixes

* Add init function for integration tests. Can init other things too.

* These changes should resolve any issues in IDEs also.

* Create default JSON metrics file

* Error handling.

.github/workflows/rust.yaml

@@ -106,8 +106,11 @@ jobs:
           MAIN_VAULT_ADDRESS: ${{ secrets.MAIN_VAULT_ADDRESS }}
           RUST_BACKTRACE: 1
 
+      - name: RGB Test Init
+        run: cargo test --locked --features server --test _init -- _init --nocapture --test-threads 1
+
       - name: RGB Tests
-        run: cargo test --locked --features server --test rgb  -- rgb --nocapture --test-threads 1
+        run: cargo test --locked --features server --test rgb -- rgb --nocapture --test-threads 1
         env:
           TEST_WALLET_SEED: ${{ secrets.TEST_WALLET_SEED }}
           RUST_BACKTRACE: 1

Cargo.toml

@@ -120,6 +120,7 @@ esplora_block = { version = "0.5.0", package = "esplora-client", default-feature
     "blocking",
 ] }
 inflate = "0.4.5"
+sled = "0.34.7"
 tower-http = { version = "0.4.4", features = ["cors"], optional = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
@@ -134,7 +135,8 @@ anyhow = "1.0.71"
 blake3 = "1.4.1"
 rgb-std = { version = "0.10.2" }
 serde = "1.0.189"
-toml = { version = "0.7.8", features = ["preserve_order"] }
+serde_json = "1.0.107"
+toml = { version = "0.8.0", features = ["preserve_order"] }
 
 [patch.crates-io]
 # Remove after merge and release https://github.com/BP-WG/bitcoin_foundation/pull/20

build.rs

@@ -1,4 +1,4 @@
-use std::{collections::BTreeMap, fs};
+use std::{collections::BTreeMap, env, fs, path};
 
 use anyhow::Result;
 use rgbstd::{
@@ -54,6 +54,30 @@ const MARKETPLACE_OFFERS: &str = "bitmask-marketplace_public_offers.c15";
 const MARKETPLACE_BIDS: &str = "bitmask-marketplace_public_bids.c15";
 const NETWORK: &str = "bitcoin"; // Only mainnet is tracked, no monetary incentive to upgrade testnet assets
 
+#[derive(Serialize, Deserialize, Default)]
+pub struct MetricsData {
+    bytes: u64,
+    bytes_by_day: BTreeMap<String, u64>,
+    bitcoin_wallets_by_day: BTreeMap<String, usize>,
+    signet_wallets_by_day: BTreeMap<String, usize>,
+    testnet_wallets_by_day: BTreeMap<String, usize>,
+    regtest_wallets_by_day: BTreeMap<String, usize>,
+    wallets_by_network: BTreeMap<String, usize>,
+}
+
+pub fn init_fs() -> Result<()> {
+    let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned());
+    let dir = path::Path::new(&dir);
+
+    fs::create_dir_all(dir)?;
+    fs::write(
+        dir.join("metrics.json"),
+        serde_json::to_string_pretty(&MetricsData::default())?,
+    )?;
+
+    Ok(())
+}
+
 fn main() -> Result<()> {
     // lib ids
     const BMC_VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -158,5 +182,7 @@ fn main() -> Result<()> {
 
     fs::write(FILE_HASHES_FILE, toml)?;
 
+    init_fs()?;
+
     Ok(())
 }

src/bin/bitmaskd.rs

@@ -724,7 +724,8 @@ async fn send_coins(
 }
 
 async fn json_metrics() -> Result<impl IntoResponse, AppError> {
-    let metrics_json = metrics::json().await?;
+    let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned());
+    let metrics_json = fs::read_to_string(&format!("{dir}/metrics.json")).await?;
 
     Ok((
         StatusCode::OK,
@@ -734,14 +735,16 @@ async fn json_metrics() -> Result<impl IntoResponse, AppError> {
 }
 
 async fn csv_metrics() -> Result<impl IntoResponse, AppError> {
-    let metrics_csv = metrics::csv().await;
+    let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned());
+    let metrics_csv = fs::read_to_string(&format!("{dir}/metrics.csv")).await?;
 
     Ok((StatusCode::OK, [("content-type", "text/csv")], metrics_csv))
 }
 
 async fn init_metrics() -> Result<()> {
-    let path = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned());
-    let dir = path::Path::new(&path);
+    let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned());
+    let dir = path::Path::new(&dir);
+    fs::create_dir_all(dir).await?;
 
     info!("Starting metrics collection...");
     let duration = Instant::now();
@@ -819,14 +822,14 @@ async fn main() -> Result<()> {
         app = app
             .route("/regtest/block", get(new_block))
             .route("/regtest/send/:address/:amount", get(send_coins));
-    } else {
-        tokio::spawn(async {
-            if let Err(e) = init_metrics().await {
-                error!("Error in periodic metrics: {e}");
-            }
-        });
     }
 
+    tokio::spawn(async {
+        if let Err(e) = init_metrics().await {
+            error!("Error in init metrics: {e}");
+        }
+    });
+
     let app = app.layer(CorsLayer::permissive());
     let addr = SocketAddr::from(([0, 0, 0, 0], 7070));