From 5f1d06c9589cdbb8d43a436565e2b2cf38ae0dee Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Wed, 6 Nov 2024 16:17:36 +0100 Subject: [PATCH 1/2] db: retry db connection --- Cargo.lock | 14 ++++++++++++++ Cargo.toml | 2 +- block-index/Cargo.toml | 1 + block-index/src/appstate.rs | 33 +++++++++++++++++++++------------ block-index/src/main.rs | 2 +- chain/Cargo.toml | 1 + chain/src/appstate.rs | 30 +++++++++++++++++++++--------- chain/src/main.rs | 2 +- webserver/Cargo.toml | 1 + webserver/src/app.rs | 2 +- webserver/src/appstate.rs | 33 +++++++++++++++++++++------------ 11 files changed, 84 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef74ac1..456ef60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "tryhard", "vergen", "xorf", ] @@ -855,6 +856,7 @@ dependencies = [ "tokio-retry", "tracing", "tracing-subscriber", + "tryhard", "vergen", ] @@ -6719,6 +6721,17 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tryhard" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9f0a709784e86923586cff0d872dba54cd2d2e116b3bc57587d15737cfce9d" +dependencies = [ + "futures", + "pin-project-lite", + "tokio", +] + [[package]] name = "tungstenite" version = "0.20.1" @@ -7131,6 +7144,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "tryhard", "validator", "vergen", "xorf", diff --git a/Cargo.toml b/Cargo.toml index e13112c..150aa90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] resolver = "2" - members = ["block-index", "chain", "shared", "orm", "webserver"] [workspace.package] @@ -43,3 +42,4 @@ tracing-appender = "0.2.0" tracing-subscriber = { version = "0.3", features = [ "env-filter" ] } validator = { version = "0.16.0", features = ["derive"] } xorf = { version = "0.11.0", features = ["serde"]} +tryhard = { version = "0.5.1" } diff --git a/block-index/Cargo.toml b/block-index/Cargo.toml index bd5c8ed..41b00a6 100644 --- a/block-index/Cargo.toml +++ b/block-index/Cargo.toml @@ -27,6 +27,7 @@ tracing-appender.workspace = true tracing-subscriber.workspace = true tracing.workspace = true xorf.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/block-index/src/appstate.rs b/block-index/src/appstate.rs index 69a8492..c711bbd 100644 --- a/block-index/src/appstate.rs +++ b/block-index/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,22 +10,30 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::from_config( - db_url, - deadpool_diesel::Runtime::Tokio1, - deadpool_diesel::ManagerConfig { - recycling_method: deadpool_diesel::RecyclingMethod::Verified, - }, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(5) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) } diff --git a/block-index/src/main.rs b/block-index/src/main.rs index 05b23a9..aaaea2e 100644 --- a/block-index/src/main.rs +++ b/block-index/src/main.rs @@ -46,7 +46,7 @@ async fn main() -> Result<(), MainError> { tracing::info!(version = VERSION_STRING, "Started the block index builder"); let mut exit_handle = must_exit(); - let app_state = AppState::new(database_url).into_db_error()?; + let app_state = AppState::new(database_url).await.into_db_error()?; if wait_for_migrations(&mut exit_handle, &app_state) .await diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 9248a16..d3c8bcd 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -30,6 +30,7 @@ tokio-retry.workspace = true tokio.workspace = true tracing-subscriber.workspace = true tracing.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/chain/src/appstate.rs b/chain/src/appstate.rs index 8179fb3..c711bbd 100644 --- a/chain/src/appstate.rs +++ b/chain/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,19 +10,30 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::new( - db_url, - deadpool_diesel::Runtime::Tokio1, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(5) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) } diff --git a/chain/src/main.rs b/chain/src/main.rs index f5d12ad..bdc9eb2 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -45,7 +45,7 @@ async fn main() -> Result<(), MainError> { tracing::info!(version = VERSION_STRING, "Started the namada-masp-indexer"); let exit_handle = must_exit_handle(); - let app_state = AppState::new(database_url).into_db_error()?; + let app_state = AppState::new(database_url).await.into_db_error()?; run_migrations(&app_state).await?; diff --git a/webserver/Cargo.toml b/webserver/Cargo.toml index 9a5910d..43efb8c 100644 --- a/webserver/Cargo.toml +++ b/webserver/Cargo.toml @@ -41,6 +41,7 @@ tracing-subscriber.workspace = true tracing.workspace = true validator.workspace = true xorf.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/webserver/src/app.rs b/webserver/src/app.rs index 59e5c6e..5c2b0d0 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -34,7 +34,7 @@ impl ApplicationServer { let rps = config.rps.unwrap_or_else(|| *REQ_PER_SEC); let db_url = config.database_url.clone(); - let app_state = AppState::new(db_url)?; + let app_state = AppState::new(db_url).await?; let routes = { let common_state = CommonState::new(app_state.clone()); diff --git a/webserver/src/appstate.rs b/webserver/src/appstate.rs index 69a8492..c711bbd 100644 --- a/webserver/src/appstate.rs +++ b/webserver/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,22 +10,30 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::from_config( - db_url, - deadpool_diesel::Runtime::Tokio1, - deadpool_diesel::ManagerConfig { - recycling_method: deadpool_diesel::RecyclingMethod::Verified, - }, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(5) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) } From 079dab5f72a23e34f16376a6fa3a782284d4c2bd Mon Sep 17 00:00:00 2001 From: Gianmarco Fraccaroli Date: Wed, 6 Nov 2024 16:31:03 +0100 Subject: [PATCH 2/2] make max retry configurable via env --- block-index/src/appstate.rs | 7 ++++++- chain/src/appstate.rs | 7 ++++++- webserver/src/appstate.rs | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/block-index/src/appstate.rs b/block-index/src/appstate.rs index c711bbd..734f51d 100644 --- a/block-index/src/appstate.rs +++ b/block-index/src/appstate.rs @@ -16,6 +16,11 @@ impl AppState { .parse::() .unwrap_or(8_usize); + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + let pool = tryhard::retry_fn(|| async { let pool_manager = deadpool_diesel::Manager::from_config( db_url.clone(), @@ -30,7 +35,7 @@ impl AppState { .build() .context("Failed to build Postgres db pool") }) - .retries(5) + .retries(max_conn_retries) .exponential_backoff(Duration::from_millis(100)) .max_delay(Duration::from_secs(5)) .await?; diff --git a/chain/src/appstate.rs b/chain/src/appstate.rs index c711bbd..734f51d 100644 --- a/chain/src/appstate.rs +++ b/chain/src/appstate.rs @@ -16,6 +16,11 @@ impl AppState { .parse::() .unwrap_or(8_usize); + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + let pool = tryhard::retry_fn(|| async { let pool_manager = deadpool_diesel::Manager::from_config( db_url.clone(), @@ -30,7 +35,7 @@ impl AppState { .build() .context("Failed to build Postgres db pool") }) - .retries(5) + .retries(max_conn_retries) .exponential_backoff(Duration::from_millis(100)) .max_delay(Duration::from_secs(5)) .await?; diff --git a/webserver/src/appstate.rs b/webserver/src/appstate.rs index c711bbd..734f51d 100644 --- a/webserver/src/appstate.rs +++ b/webserver/src/appstate.rs @@ -16,6 +16,11 @@ impl AppState { .parse::() .unwrap_or(8_usize); + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + let pool = tryhard::retry_fn(|| async { let pool_manager = deadpool_diesel::Manager::from_config( db_url.clone(), @@ -30,7 +35,7 @@ impl AppState { .build() .context("Failed to build Postgres db pool") }) - .retries(5) + .retries(max_conn_retries) .exponential_backoff(Duration::from_millis(100)) .max_delay(Duration::from_secs(5)) .await?;