diff --git a/Cargo.lock b/Cargo.lock index ef74ac1..456ef60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "tryhard", "vergen", "xorf", ] @@ -855,6 +856,7 @@ dependencies = [ "tokio-retry", "tracing", "tracing-subscriber", + "tryhard", "vergen", ] @@ -6719,6 +6721,17 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tryhard" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9f0a709784e86923586cff0d872dba54cd2d2e116b3bc57587d15737cfce9d" +dependencies = [ + "futures", + "pin-project-lite", + "tokio", +] + [[package]] name = "tungstenite" version = "0.20.1" @@ -7131,6 +7144,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "tryhard", "validator", "vergen", "xorf", diff --git a/Cargo.toml b/Cargo.toml index e13112c..150aa90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] resolver = "2" - members = ["block-index", "chain", "shared", "orm", "webserver"] [workspace.package] @@ -43,3 +42,4 @@ tracing-appender = "0.2.0" tracing-subscriber = { version = "0.3", features = [ "env-filter" ] } validator = { version = "0.16.0", features = ["derive"] } xorf = { version = "0.11.0", features = ["serde"]} +tryhard = { version = "0.5.1" } diff --git a/block-index/Cargo.toml b/block-index/Cargo.toml index bd5c8ed..41b00a6 100644 --- a/block-index/Cargo.toml +++ b/block-index/Cargo.toml @@ -27,6 +27,7 @@ tracing-appender.workspace = true tracing-subscriber.workspace = true tracing.workspace = true xorf.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/block-index/src/appstate.rs b/block-index/src/appstate.rs index 69a8492..734f51d 100644 --- a/block-index/src/appstate.rs +++ b/block-index/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,22 +10,35 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::from_config( - db_url, - deadpool_diesel::Runtime::Tokio1, - deadpool_diesel::ManagerConfig { - recycling_method: deadpool_diesel::RecyclingMethod::Verified, - }, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(max_conn_retries) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) } diff --git a/block-index/src/main.rs b/block-index/src/main.rs index 05b23a9..aaaea2e 100644 --- a/block-index/src/main.rs +++ b/block-index/src/main.rs @@ -46,7 +46,7 @@ async fn main() -> Result<(), MainError> { tracing::info!(version = VERSION_STRING, "Started the block index builder"); let mut exit_handle = must_exit(); - let app_state = AppState::new(database_url).into_db_error()?; + let app_state = AppState::new(database_url).await.into_db_error()?; if wait_for_migrations(&mut exit_handle, &app_state) .await diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 9248a16..d3c8bcd 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -30,6 +30,7 @@ tokio-retry.workspace = true tokio.workspace = true tracing-subscriber.workspace = true tracing.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/chain/src/appstate.rs b/chain/src/appstate.rs index 8179fb3..734f51d 100644 --- a/chain/src/appstate.rs +++ b/chain/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,19 +10,35 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::new( - db_url, - deadpool_diesel::Runtime::Tokio1, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(max_conn_retries) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) } diff --git a/chain/src/main.rs b/chain/src/main.rs index f5d12ad..bdc9eb2 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -45,7 +45,7 @@ async fn main() -> Result<(), MainError> { tracing::info!(version = VERSION_STRING, "Started the namada-masp-indexer"); let exit_handle = must_exit_handle(); - let app_state = AppState::new(database_url).into_db_error()?; + let app_state = AppState::new(database_url).await.into_db_error()?; run_migrations(&app_state).await?; diff --git a/webserver/Cargo.toml b/webserver/Cargo.toml index 9a5910d..43efb8c 100644 --- a/webserver/Cargo.toml +++ b/webserver/Cargo.toml @@ -41,6 +41,7 @@ tracing-subscriber.workspace = true tracing.workspace = true validator.workspace = true xorf.workspace = true +tryhard.workspace = true [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/webserver/src/app.rs b/webserver/src/app.rs index 59e5c6e..5c2b0d0 100644 --- a/webserver/src/app.rs +++ b/webserver/src/app.rs @@ -34,7 +34,7 @@ impl ApplicationServer { let rps = config.rps.unwrap_or_else(|| *REQ_PER_SEC); let db_url = config.database_url.clone(); - let app_state = AppState::new(db_url)?; + let app_state = AppState::new(db_url).await?; let routes = { let common_state = CommonState::new(app_state.clone()); diff --git a/webserver/src/appstate.rs b/webserver/src/appstate.rs index 69a8492..734f51d 100644 --- a/webserver/src/appstate.rs +++ b/webserver/src/appstate.rs @@ -1,4 +1,5 @@ use std::env; +use std::time::Duration; use anyhow::Context; use deadpool_diesel::postgres::{Object, Pool as DbPool}; @@ -9,22 +10,35 @@ pub struct AppState { } impl AppState { - pub fn new(db_url: String) -> anyhow::Result { + pub async fn new(db_url: String) -> anyhow::Result { let max_pool_size = env::var("DATABASE_POOL_SIZE") .unwrap_or_else(|_| 8.to_string()) .parse::() .unwrap_or(8_usize); - let pool_manager = deadpool_diesel::Manager::from_config( - db_url, - deadpool_diesel::Runtime::Tokio1, - deadpool_diesel::ManagerConfig { - recycling_method: deadpool_diesel::RecyclingMethod::Verified, - }, - ); - let pool = DbPool::builder(pool_manager) - .max_size(max_pool_size) - .build() - .context("Failed to build Postgres db pool")?; + + let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5); + + let pool = tryhard::retry_fn(|| async { + let pool_manager = deadpool_diesel::Manager::from_config( + db_url.clone(), + deadpool_diesel::Runtime::Tokio1, + deadpool_diesel::ManagerConfig { + recycling_method: + deadpool_diesel::RecyclingMethod::Verified, + }, + ); + DbPool::builder(pool_manager) + .max_size(max_pool_size) + .build() + .context("Failed to build Postgres db pool") + }) + .retries(max_conn_retries) + .exponential_backoff(Duration::from_millis(100)) + .max_delay(Duration::from_secs(5)) + .await?; Ok(Self { db: pool }) }