Merge pull request #217 from MELLIFERA-Labs/fix/insert-db-rewards-in-…

…batch

fix: insert rewards in batch

rewards/src/repository/mod.rs

@@ -1,2 +1,3 @@
 pub mod crawler_state;
 pub mod pos_rewards;
+mod utils;

rewards/src/repository/pos_rewards.rs

@@ -1,12 +1,33 @@
+use anyhow::Context;
 use diesel::upsert::excluded;
 use diesel::{ExpressionMethods, PgConnection, QueryDsl, RunQueryDsl};
 use orm::pos_rewards::PosRewardInsertDb;
 use orm::schema::{pos_rewards, validators};
 use shared::rewards::Reward;
+use shared::tuple_len::TupleLen;
+
+use super::utils::MAX_PARAM_SIZE;
 
 pub fn upsert_rewards(
     transaction_conn: &mut PgConnection,
     rewards: Vec<Reward>,
+) -> anyhow::Result<()> {
+    let rewards_col_count = pos_rewards::all_columns.len() as i64;
+
+    for chunk in rewards
+        .into_iter()
+        .collect::<Vec<_>>()
+        .chunks((MAX_PARAM_SIZE as i64 / rewards_col_count) as usize)
+    {
+        upsert_rewards_chunk(transaction_conn, chunk.to_vec())?;
+    }
+
+    anyhow::Ok(())
+}
+
+fn upsert_rewards_chunk(
+    transaction_conn: &mut PgConnection,
+    rewards: Vec<Reward>,
 ) -> anyhow::Result<()> {
     diesel::insert_into(pos_rewards::table)
         .values::<Vec<PosRewardInsertDb>>(
@@ -37,7 +58,8 @@ pub fn upsert_rewards(
             pos_rewards::columns::raw_amount
                 .eq(excluded(pos_rewards::columns::raw_amount)),
         )
-        .execute(transaction_conn)?;
+        .execute(transaction_conn)
+        .context("Failed to upsert rewards in db")?;
 
     Ok(())
 }

rewards/src/repository/utils.rs

@@ -0,0 +1,4 @@
+// Represents maximum number of parameters that we can insert into postgres in
+// one go. To get the number of rows that we can insert in one chunk, we have to
+// divide MAX_PARAM_SIZE by the number of columns in the given table.
+pub const MAX_PARAM_SIZE: u16 = u16::MAX;