Merge pull request #26 from anoma/fraccaman/retry-connection-error

db: retry db connection
anoma · Nov 6, 2024 · b50f5ab · b50f5ab
2 parents 4267d17 + 079dab5
commit b50f5ab
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 37 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,5 @@
 [workspace]
 resolver = "2"
-
 members = ["block-index", "chain", "shared", "orm", "webserver"]
 
 [workspace.package]
@@ -43,3 +42,4 @@ tracing-appender = "0.2.0"
 tracing-subscriber = { version = "0.3", features = [ "env-filter" ] }
 validator = { version = "0.16.0", features = ["derive"] }
 xorf = { version = "0.11.0", features = ["serde"]}
+tryhard = { version = "0.5.1" }
diff --git a/block-index/Cargo.toml b/block-index/Cargo.toml
@@ -27,6 +27,7 @@ tracing-appender.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
 xorf.workspace = true
+tryhard.workspace = true
 
 [build-dependencies]
 vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] }
diff --git a/block-index/src/appstate.rs b/block-index/src/appstate.rs
@@ -1,4 +1,5 @@
 use std::env;
+use std::time::Duration;
 
 use anyhow::Context;
 use deadpool_diesel::postgres::{Object, Pool as DbPool};
@@ -9,22 +10,35 @@ pub struct AppState {
 }
 
 impl AppState {
-    pub fn new(db_url: String) -> anyhow::Result<Self> {
+    pub async fn new(db_url: String) -> anyhow::Result<Self> {
         let max_pool_size = env::var("DATABASE_POOL_SIZE")
             .unwrap_or_else(|_| 8.to_string())
             .parse::<usize>()
             .unwrap_or(8_usize);
-        let pool_manager = deadpool_diesel::Manager::from_config(
-            db_url,
-            deadpool_diesel::Runtime::Tokio1,
-            deadpool_diesel::ManagerConfig {
-                recycling_method: deadpool_diesel::RecyclingMethod::Verified,
-            },
-        );
-        let pool = DbPool::builder(pool_manager)
-            .max_size(max_pool_size)
-            .build()
-            .context("Failed to build Postgres db pool")?;
+
+        let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES")
+            .unwrap_or_else(|_| 5.to_string())
+            .parse::<u32>()
+            .unwrap_or(5);
+
+        let pool = tryhard::retry_fn(|| async {
+            let pool_manager = deadpool_diesel::Manager::from_config(
+                db_url.clone(),
+                deadpool_diesel::Runtime::Tokio1,
+                deadpool_diesel::ManagerConfig {
+                    recycling_method:
+                        deadpool_diesel::RecyclingMethod::Verified,
+                },
+            );
+            DbPool::builder(pool_manager)
+                .max_size(max_pool_size)
+                .build()
+                .context("Failed to build Postgres db pool")
+        })
+        .retries(max_conn_retries)
+        .exponential_backoff(Duration::from_millis(100))
+        .max_delay(Duration::from_secs(5))
+        .await?;
 
         Ok(Self { db: pool })
     }

diff --git a/block-index/src/main.rs b/block-index/src/main.rs
@@ -46,7 +46,7 @@ async fn main() -> Result<(), MainError> {
     tracing::info!(version = VERSION_STRING, "Started the block index builder");
     let mut exit_handle = must_exit();
 
-    let app_state = AppState::new(database_url).into_db_error()?;
+    let app_state = AppState::new(database_url).await.into_db_error()?;
 
     if wait_for_migrations(&mut exit_handle, &app_state)
         .await

diff --git a/chain/Cargo.toml b/chain/Cargo.toml
@@ -30,6 +30,7 @@ tokio-retry.workspace = true
 tokio.workspace = true 
 tracing-subscriber.workspace = true
 tracing.workspace = true
+tryhard.workspace = true
 
 [build-dependencies]
 vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] }
diff --git a/chain/src/appstate.rs b/chain/src/appstate.rs
@@ -1,4 +1,5 @@
 use std::env;
+use std::time::Duration;
 
 use anyhow::Context;
 use deadpool_diesel::postgres::{Object, Pool as DbPool};
@@ -9,19 +10,35 @@ pub struct AppState {
 }
 
 impl AppState {
-    pub fn new(db_url: String) -> anyhow::Result<Self> {
+    pub async fn new(db_url: String) -> anyhow::Result<Self> {
         let max_pool_size = env::var("DATABASE_POOL_SIZE")
             .unwrap_or_else(|_| 8.to_string())
             .parse::<usize>()
             .unwrap_or(8_usize);
-        let pool_manager = deadpool_diesel::Manager::new(
-            db_url,
-            deadpool_diesel::Runtime::Tokio1,
-        );
-        let pool = DbPool::builder(pool_manager)
-            .max_size(max_pool_size)
-            .build()
-            .context("Failed to build Postgres db pool")?;
+
+        let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES")
+            .unwrap_or_else(|_| 5.to_string())
+            .parse::<u32>()
+            .unwrap_or(5);
+
+        let pool = tryhard::retry_fn(|| async {
+            let pool_manager = deadpool_diesel::Manager::from_config(
+                db_url.clone(),
+                deadpool_diesel::Runtime::Tokio1,
+                deadpool_diesel::ManagerConfig {
+                    recycling_method:
+                        deadpool_diesel::RecyclingMethod::Verified,
+                },
+            );
+            DbPool::builder(pool_manager)
+                .max_size(max_pool_size)
+                .build()
+                .context("Failed to build Postgres db pool")
+        })
+        .retries(max_conn_retries)
+        .exponential_backoff(Duration::from_millis(100))
+        .max_delay(Duration::from_secs(5))
+        .await?;
 
         Ok(Self { db: pool })
     }

diff --git a/chain/src/main.rs b/chain/src/main.rs
@@ -45,7 +45,7 @@ async fn main() -> Result<(), MainError> {
     tracing::info!(version = VERSION_STRING, "Started the namada-masp-indexer");
     let exit_handle = must_exit_handle();
 
-    let app_state = AppState::new(database_url).into_db_error()?;
+    let app_state = AppState::new(database_url).await.into_db_error()?;
 
     run_migrations(&app_state).await?;
 

diff --git a/webserver/Cargo.toml b/webserver/Cargo.toml
@@ -41,6 +41,7 @@ tracing-subscriber.workspace = true
 tracing.workspace = true
 validator.workspace = true
 xorf.workspace = true
+tryhard.workspace = true
 
 [build-dependencies]
 vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] }
diff --git a/webserver/src/app.rs b/webserver/src/app.rs
@@ -34,7 +34,7 @@ impl ApplicationServer {
         let rps = config.rps.unwrap_or_else(|| *REQ_PER_SEC);
         let db_url = config.database_url.clone();
 
-        let app_state = AppState::new(db_url)?;
+        let app_state = AppState::new(db_url).await?;
 
         let routes = {
             let common_state = CommonState::new(app_state.clone());

diff --git a/webserver/src/appstate.rs b/webserver/src/appstate.rs
@@ -1,4 +1,5 @@
 use std::env;
+use std::time::Duration;
 
 use anyhow::Context;
 use deadpool_diesel::postgres::{Object, Pool as DbPool};
@@ -9,22 +10,35 @@ pub struct AppState {
 }
 
 impl AppState {
-    pub fn new(db_url: String) -> anyhow::Result<Self> {
+    pub async fn new(db_url: String) -> anyhow::Result<Self> {
         let max_pool_size = env::var("DATABASE_POOL_SIZE")
             .unwrap_or_else(|_| 8.to_string())
             .parse::<usize>()
             .unwrap_or(8_usize);
-        let pool_manager = deadpool_diesel::Manager::from_config(
-            db_url,
-            deadpool_diesel::Runtime::Tokio1,
-            deadpool_diesel::ManagerConfig {
-                recycling_method: deadpool_diesel::RecyclingMethod::Verified,
-            },
-        );
-        let pool = DbPool::builder(pool_manager)
-            .max_size(max_pool_size)
-            .build()
-            .context("Failed to build Postgres db pool")?;
+
+        let max_conn_retries = env::var("DATABASE_MAX_CONN_RETRIES")
+            .unwrap_or_else(|_| 5.to_string())
+            .parse::<u32>()
+            .unwrap_or(5);
+
+        let pool = tryhard::retry_fn(|| async {
+            let pool_manager = deadpool_diesel::Manager::from_config(
+                db_url.clone(),
+                deadpool_diesel::Runtime::Tokio1,
+                deadpool_diesel::ManagerConfig {
+                    recycling_method:
+                        deadpool_diesel::RecyclingMethod::Verified,
+                },
+            );
+            DbPool::builder(pool_manager)
+                .max_size(max_pool_size)
+                .build()
+                .context("Failed to build Postgres db pool")
+        })
+        .retries(max_conn_retries)
+        .exponential_backoff(Duration::from_millis(100))
+        .max_delay(Duration::from_secs(5))
+        .await?;
 
         Ok(Self { db: pool })
     }