feat: remove validators metadata and improve chain initial query loop (…

…#33)

README.md

@@ -7,20 +7,21 @@ A set of microservices that crawler data from a namada node, store them in a pos
 ## Architecture
 
 The indexer is composed of a set microservices and a webserver, each one of these lives in his own crate. Each microservice is responsible of indexing some data from the chain and store them in the postgres database. Right now, there are 4 microservices:
+
 - `chain`: goes block by block and fetches information from transactions (e.g balances)
 - `pos`: fetches the validator set each new epoch
 - `rewards`: fetches PoS rewards each new epoch
 - `governance`: fetches new proposal and the corresponding votes
 
-The `webserver` is responsible to serve the data via a REST API, which are described in the `swaller.yml` file in the project root.
+The `webserver` is responsible to serve the data via a REST API, which are described in the `swagger.yml` file in the project root.
 
 ![Namada indexer architecture](docs/architecture.png "Architecture")
 
 ## How to run
 
 - Get a Namada RPC url
-    - [Either create a local chain ](https://github.com/anoma/namada/blob/main/scripts/gen_localnet.py)
-    - Or use a Public RPC 
+  - [Either create a local chain ](https://github.com/anoma/namada/blob/main/scripts/gen_localnet.py)
+  - Or use a Public RPC
 - Change `CHAIN_ID` and `CHECKSUMS_FILE` env variable and file
 - Install [just](https://github.com/casey/just)
 - Run `just docker-run`

chain/run.sh

@@ -1,2 +1,2 @@
 
-cargo run -- --tendermint-url http://127.0.0.1:27657 --chain-id local.988b6b47aadd23b12b8a9a03 --database-url postgres://postgres:[email protected]:5435/namada-indexer
+cargo run -- --tendermint-url http://127.0.0.1:27657 --initial-query-retry-time 5 --database-url postgres://postgres:[email protected]:5435/namada-indexer

chain/src/config.rs

@@ -24,7 +24,7 @@ pub struct AppConfig {
     pub database_url: String,
 
     #[clap(long, env)]
-    pub chain_id: String,
+    pub initial_query_retry_time: u64,
 
     #[command(flatten)]
     pub verbosity: Verbosity<InfoLevel>,

chain/src/main.rs

@@ -70,7 +70,7 @@ async fn main() -> Result<(), MainError> {
         .context_db_interact_error()
         .into_db_error()?;
 
-    initial_query(&client, &conn).await?;
+    initial_query(&client, &conn, config.initial_query_retry_time).await?;
 
     let last_block_height = db_service::get_last_synched_block(&conn)
         .await
@@ -224,28 +224,33 @@ async fn crawling_fn(
 async fn initial_query(
     client: &HttpClient,
     conn: &Object,
+    initial_query_retry_time: u64,
 ) -> Result<(), MainError> {
     tracing::info!("Querying initial data...");
     let block_height =
         query_last_block_height(client).await.into_rpc_error()?;
-    let epoch = namada_service::get_epoch_at_block_height(client, block_height)
-        .await
-        .into_rpc_error()?;
+    let mut epoch =
+        namada_service::get_epoch_at_block_height(client, block_height)
+            .await
+            .into_rpc_error()?;
 
     loop {
         let pos_crawler_epoch =
             get_pos_crawler_state(conn).await.into_db_error();
 
         match pos_crawler_epoch {
-            Ok(pos_crawler_epoch) if pos_crawler_epoch.epoch == epoch => {
+            // >= in case epochs are really short
+            Ok(pos_crawler_epoch) if pos_crawler_epoch.epoch >= epoch => {
+                // We assign pos crawler epoch as epoch to process
+                epoch = pos_crawler_epoch.epoch;
                 break;
             }
             _ => {}
         }
 
         tracing::info!("Waiting for PoS service update...");
 
-        sleep(Duration::from_secs(1)).await;
+        sleep(Duration::from_secs(initial_query_retry_time)).await;
     }
 
     let balances = query_all_balances(client).await.into_rpc_error()?;

orm/migrations/2024-04-30-081808_init_validators/up.sql

@@ -6,7 +6,7 @@ CREATE TABLE validators (
     voting_power INT NOT NULL,
     max_commission VARCHAR NOT NULL,
     commission VARCHAR NOT NULL,
-    email VARCHAR NOT NULL,
+    email VARCHAR,
     website VARCHAR,
     description VARCHAR,
     discord_handle VARCHAR,

orm/src/schema.rs

@@ -1,27 +1,15 @@
 // @generated automatically by Diesel CLI.
 
 pub mod sql_types {
-    #[derive(
-        diesel::query_builder::QueryId,
-        std::fmt::Debug,
-        diesel::sql_types::SqlType,
-    )]
+    #[derive(diesel::query_builder::QueryId, std::fmt::Debug, diesel::sql_types::SqlType)]
     #[diesel(postgres_type(name = "governance_kind"))]
     pub struct GovernanceKind;
 
-    #[derive(
-        diesel::query_builder::QueryId,
-        std::fmt::Debug,
-        diesel::sql_types::SqlType,
-    )]
+    #[derive(diesel::query_builder::QueryId, std::fmt::Debug, diesel::sql_types::SqlType)]
     #[diesel(postgres_type(name = "governance_result"))]
     pub struct GovernanceResult;
 
-    #[derive(
-        diesel::query_builder::QueryId,
-        std::fmt::Debug,
-        diesel::sql_types::SqlType,
-    )]
+    #[derive(diesel::query_builder::QueryId, std::fmt::Debug, diesel::sql_types::SqlType)]
     #[diesel(postgres_type(name = "vote_kind"))]
     pub struct VoteKind;
 }
@@ -49,7 +37,6 @@ diesel::table! {
         address -> Varchar,
         validator_id -> Int4,
         raw_amount -> Varchar,
-        epoch -> Int4,
     }
 }
 
@@ -108,7 +95,6 @@ diesel::table! {
         address -> Varchar,
         validator_id -> Int4,
         raw_amount -> Varchar,
-        epoch -> Int4,
         withdraw_epoch -> Int4,
     }
 }
@@ -120,7 +106,7 @@ diesel::table! {
         voting_power -> Int4,
         max_commission -> Varchar,
         commission -> Varchar,
-        email -> Varchar,
+        email -> Nullable<Varchar>,
         website -> Nullable<Varchar>,
         description -> Nullable<Varchar>,
         discord_handle -> Nullable<Varchar>,

orm/src/validators.rs

@@ -15,7 +15,7 @@ pub struct ValidatorDb {
     pub voting_power: i32,
     pub max_commission: String,
     pub commission: String,
-    pub email: String,
+    pub email: Option<String>,
     pub website: Option<String>,
     pub description: Option<String>,
     pub discord_handle: Option<String>,
@@ -30,11 +30,6 @@ pub struct ValidatorInsertDb {
     pub voting_power: i32,
     pub max_commission: String,
     pub commission: String,
-    pub email: String,
-    pub website: Option<String>,
-    pub description: Option<String>,
-    pub discord_handle: Option<String>,
-    pub avatar: Option<String>,
 }
 
 #[derive(Serialize, AsChangeset, Clone)]
@@ -57,11 +52,6 @@ impl ValidatorInsertDb {
                 as i32,
             max_commission: validator.max_commission.clone(),
             commission: validator.commission.clone(),
-            email: validator.email.clone(),
-            website: validator.website.clone(),
-            description: validator.description.clone(),
-            discord_handle: validator.discord_handler.clone(),
-            avatar: validator.avatar.clone(),
         }
     }
 }

pos/src/services/namada.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Context};
+use anyhow::Context;
 use futures::{StreamExt, TryStreamExt};
 use namada_core::storage::Epoch as NamadaSdkEpoch;
 use namada_sdk::rpc;
@@ -36,40 +36,38 @@ pub async fn get_validator_set_at_epoch(
                     })
             };
 
-            let meta_and_comm_fut = async {
-                let (metadata, commission) =
-                    rpc::query_metadata(client, &address, Some(namada_epoch))
-                        .await
-                        .with_context(|| {
-                            format!(
-                                "Failed to query metadata of validator \
-                                 {address} at epoch {namada_epoch}"
-                            )
-                        })?;
-                metadata.zip(Some(commission)).ok_or_else(|| {
-                    anyhow!(
-                        "Metadata and commission must be present for \
-                         validator {address} at epoch {namada_epoch}"
+            let commission_fut = async {
+                rpc::query_commission_rate(client, &address, Some(namada_epoch))
+                    .await
+                    .with_context(|| {
+                        format!(
+                        "Failed to query commission of validator {address} \
+                             at epoch {namada_epoch}"
                     )
-                })
+                    })
             };
 
-            let (voting_power, (metadata, commission)) =
-                futures::try_join!(voting_power_fut, meta_and_comm_fut)?;
+            let (voting_power, commission_pair) =
+                futures::try_join!(voting_power_fut, commission_fut)?;
+            let commission = commission_pair
+                .commission_rate
+                .expect("Commission rate has to exist")
+                .to_string();
+            let max_commission = commission_pair
+                .max_commission_change_per_epoch
+                .expect("Max commission rate change has to exist")
+                .to_string();
 
             anyhow::Ok(Validator {
                 address: Id::Account(address.to_string()),
                 voting_power: voting_power.to_string_native(),
-                max_commission: commission
-                    .max_commission_change_per_epoch
-                    .unwrap()
-                    .to_string(), // this should be safe
-                commission: commission.commission_rate.unwrap().to_string(), // this should be safe
-                email: metadata.email,
-                description: metadata.description,
-                website: metadata.website,
-                discord_handler: metadata.discord_handle,
-                avatar: metadata.avatar,
+                max_commission,
+                commission,
+                email: None,
+                description: None,
+                website: None,
+                discord_handler: None,
+                avatar: None,
             })
         })
         .buffer_unordered(100)

shared/src/validator.rs

@@ -19,7 +19,7 @@ pub struct Validator {
     pub voting_power: VotingPower,
     pub max_commission: String,
     pub commission: String,
-    pub email: String,
+    pub email: Option<String>,
     pub description: Option<String>,
     pub website: Option<String>,
     pub discord_handler: Option<String>,
@@ -45,7 +45,7 @@ impl Validator {
         let max_commission =
             ((0..100).fake::<u64>() as f64 / 100_f64).to_string();
         let commission = ((0..100).fake::<u64>() as f64 / 100_f64).to_string();
-        let email: String = SafeEmail().fake();
+        let email = Some(SafeEmail().fake());
         let description: Option<String> = CatchPhase().fake();
         let website: Option<String> = Some(format!(
             "{}.{}",

swagger.yml

@@ -328,6 +328,7 @@ components:
   schemas:
     Validator:
       type: object
+      required: [id, address, name, voting_power, max_commission, commission]
       properties:
         id:
           type: integer

webserver/src/response/pos.rs

@@ -11,7 +11,7 @@ pub struct Validator {
     pub voting_power: String,
     pub max_commission: String,
     pub commission: String,
-    pub email: String,
+    pub email: Option<String>,
     pub website: Option<String>,
     pub description: Option<String>,
     pub discord_handle: Option<String>,