diff --git a/chain/src/main.rs b/chain/src/main.rs index bdc9eb2..eac8a16 100644 --- a/chain/src/main.rs +++ b/chain/src/main.rs @@ -4,8 +4,10 @@ pub mod entity; pub mod services; use std::collections::BTreeMap; +use std::env; use std::sync::atomic::{self, AtomicBool}; use std::sync::Arc; +use std::time::Duration; use anyhow::Context; use clap::Parser; @@ -16,6 +18,7 @@ use shared::transaction::Transaction; use shared::tx_index::{MaspTxIndex, TxIndex}; use tendermint_rpc::HttpClient; use tokio::signal; +use tokio::time::sleep; use tokio_retry::strategy::{jitter, FixedInterval}; use tokio_retry::RetryIf; @@ -106,12 +109,38 @@ fn must_exit_handle() -> Arc { } async fn run_migrations(app_state: &AppState) -> Result<(), MainError> { - db_service::run_migrations( - app_state.get_db_connection().await.into_db_error()?, - ) - .await - .context("Failed to run db migrations") - .into_db_error() + let mut max_retries = env::var("DATABASE_MAX_MIGRATION_RETRY") + .unwrap_or_else(|_| 5.to_string()) + .parse::() + .unwrap_or(5_u64); + loop { + let migration_res = db_service::run_migrations( + app_state.get_db_connection().await.into_db_error()?, + ) + .await; + + match &migration_res { + Ok(_) => { + return migration_res + .context("Failed to run db migrations") + .into_db_error(); + } + Err(e) => { + tracing::debug!( + "Failed runnign migrations: {} ({}/5)", + e.to_string(), + max_retries + ); + if max_retries == 0 { + return migration_res + .context("Failed to run db migrations") + .into_db_error(); + } + max_retries -= 1; + sleep(Duration::from_secs(3)).await; + } + } + } } async fn load_committed_state( diff --git a/chain/src/services/db.rs b/chain/src/services/db.rs index c3e66ac..1a596d8 100644 --- a/chain/src/services/db.rs +++ b/chain/src/services/db.rs @@ -36,7 +36,9 @@ pub async fn run_migrations(conn: Object) -> anyhow::Result<()> { conn.interact(|transaction_conn| { transaction_conn .run_pending_migrations(MIGRATIONS) - .map_err(|_| anyhow!("Failed to run db migrations"))?; + .map_err(|e| { + anyhow!("Failed to run db migrations: {}", e.to_string()) + })?; anyhow::Ok(()) }) .await