Speed up v2 report sync

teambtcmap · Oct 23, 2024 · 542b450 · 542b450
1 parent 5020456
commit 542b450
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 36 deletions.
diff --git a/migrations/66.sql b/migrations/66.sql
@@ -0,0 +1,10 @@
+ALTER TABLE area ADD COLUMN alias TEXT;
+
+UPDATE area SET alias = json_extract(tags, '$.url_alias');
+
+DROP TRIGGER area_updated_at;
+
+CREATE TRIGGER area_updated_at UPDATE OF tags, alias, created_at, deleted_at ON area
+BEGIN
+    UPDATE area SET updated_at = strftime('%Y-%m-%dT%H:%M:%fZ') WHERE id = old.id;
+END;
diff --git a/src/area/model.rs b/src/area/model.rs
@@ -30,18 +30,22 @@ impl Area {
     pub fn insert(
         geo_json: GeoJson,
         mut tags: Map<String, Value>,
+        alias: &str,
         conn: &Connection,
     ) -> Result<Option<Area>> {
         tags.insert("geo_json".into(), geo_json.into());
         let query = format!(
             r#"
-                INSERT INTO {TABLE} ({COL_TAGS}) 
-                VALUES (json(:tags))
+                INSERT INTO {TABLE} ({COL_TAGS}, alias)
+                VALUES (json(:tags), :alias)
             "#
         );
         #[cfg(not(test))]
         sleep(Duration::from_millis(10));
-        conn.execute(&query, named_params! { ":tags": Value::from(tags) })?;
+        conn.execute(
+            &query,
+            named_params! { ":tags": Value::from(tags), ":alias": alias },
+        )?;
         let res = Area::select_by_id(conn.last_insert_rowid(), conn)?;
         Ok(res)
     }
@@ -375,6 +379,7 @@ mod test {
         let res = Area::insert(
             GeoJson::Feature(Feature::default()),
             tags.clone(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -388,9 +393,27 @@ mod test {
         let conn = mock_conn();
         assert_eq!(
             vec![
-                Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap(),
-                Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap(),
-                Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap(),
+                Area::insert(
+                    GeoJson::Feature(Feature::default()),
+                    Map::new(),
+                    "test",
+                    &conn
+                )?
+                .unwrap(),
+                Area::insert(
+                    GeoJson::Feature(Feature::default()),
+                    Map::new(),
+                    "test",
+                    &conn
+                )?
+                .unwrap(),
+                Area::insert(
+                    GeoJson::Feature(Feature::default()),
+                    Map::new(),
+                    "test",
+                    &conn
+                )?
+                .unwrap(),
             ],
             Area::select_all(&conn)?,
         );
@@ -403,6 +426,7 @@ mod test {
         let _area_1 = Area::insert(
             GeoJson::Feature(Feature::default()),
             mock_tags(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -411,6 +435,7 @@ mod test {
         let area_2 = Area::insert(
             GeoJson::Feature(Feature::default()),
             mock_tags(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -420,6 +445,7 @@ mod test {
         let area_3 = Area::insert(
             GeoJson::Feature(Feature::default()),
             mock_tags(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -436,7 +462,13 @@ mod test {
     #[test]
     async fn select_by_id() -> Result<()> {
         let conn = mock_conn();
-        let area = Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap();
+        let area = Area::insert(
+            GeoJson::Feature(Feature::default()),
+            Map::new(),
+            "test",
+            &conn,
+        )?
+        .unwrap();
         assert_eq!(area, Area::select_by_id(area.id, &conn)?.unwrap());
         Ok(())
     }
@@ -447,7 +479,7 @@ mod test {
         let url_alias = json!("url_alias_value");
         let mut tags = Map::new();
         tags.insert("url_alias".into(), url_alias.clone());
-        Area::insert(GeoJson::Feature(Feature::default()), tags, &conn)?;
+        Area::insert(GeoJson::Feature(Feature::default()), tags, "test", &conn)?;
         let area = Area::select_by_alias(url_alias.as_str().unwrap(), &conn)?;
         assert!(area.is_some());
         let area = area.unwrap();
@@ -464,8 +496,13 @@ mod test {
         let tag_2_value = json!("tag_2_value");
         let mut tags = Map::new();
         tags.insert(tag_1_name.into(), tag_1_value.clone());
-        let area =
-            Area::insert(GeoJson::Feature(Feature::default()), tags.clone(), &conn)?.unwrap();
+        let area = Area::insert(
+            GeoJson::Feature(Feature::default()),
+            tags.clone(),
+            "test",
+            &conn,
+        )?
+        .unwrap();
         assert_eq!(tag_1_value, area.tags[tag_1_name]);
         tags.insert(tag_2_name.into(), tag_2_value.clone());
         let area = Area::patch_tags(area.id, tags, &conn)?.unwrap();
@@ -477,7 +514,13 @@ mod test {
     #[test]
     async fn set_deleted_at() -> Result<()> {
         let conn = mock_conn();
-        let area = Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap();
+        let area = Area::insert(
+            GeoJson::Feature(Feature::default()),
+            Map::new(),
+            "test",
+            &conn,
+        )?
+        .unwrap();
         let area = Area::set_deleted_at(area.id, Some(OffsetDateTime::now_utc()), &conn)?.unwrap();
         assert!(area.deleted_at.is_some());
         let area = Area::set_deleted_at(area.id, None, &conn)?.unwrap();

diff --git a/src/area/service.rs b/src/area/service.rs
@@ -17,7 +17,8 @@ pub fn insert(tags: Map<String, Value>, conn: &Connection) -> Result<Area> {
     if !tags.contains_key("geo_json") {
         return Err(Error::InvalidInput("geo_json tag is missing".into()));
     }
-    let url_alias = tags
+    let cloned_tags = tags.clone();
+    let url_alias = cloned_tags
         .get("url_alias")
         .ok_or(Error::InvalidInput(
             "Mandatory tag is missing: url_alias".into(),
@@ -42,7 +43,7 @@ pub fn insert(tags: Map<String, Value>, conn: &Connection) -> Result<Area> {
     if Area::select_by_alias(url_alias, &conn)?.is_some() {
         Err(Error::Conflict("This url_alias is already in use".into()))?
     }
-    let area = Area::insert(geo_json.unwrap(), tags, &conn)?.unwrap();
+    let area = Area::insert(geo_json.unwrap(), tags, url_alias, &conn)?.unwrap();
     let area_elements = element::service::find_in_area(&area, conn)?;
     element::service::generate_areas_mapping_old(&area_elements, conn)?;
     Ok(area)
@@ -255,6 +256,7 @@ mod test {
         let area = Area::insert(
             GeoJson::from_json_value(phuket_geo_json()).unwrap(),
             tags,
+            url_alias.as_str().unwrap(),
             &mut conn,
         )?
         .unwrap();
@@ -290,6 +292,7 @@ mod test {
         let area = Area::insert(
             GeoJson::from_json_value(phuket_geo_json()).unwrap(),
             tags.clone(),
+            url_alias.as_str().unwrap(),
             &mut conn,
         )?
         .unwrap();
@@ -304,7 +307,13 @@ mod test {
     #[test]
     fn soft_delete() -> Result<()> {
         let mut conn = mock_conn();
-        let area = Area::insert(GeoJson::Feature(Feature::default()), Map::new(), &conn)?.unwrap();
+        let area = Area::insert(
+            GeoJson::Feature(Feature::default()),
+            Map::new(),
+            "test",
+            &conn,
+        )?
+        .unwrap();
         super::soft_delete(&area.id.to_string(), &mut conn)?;
         let db_area = Area::select_by_id(area.id, &conn)?.unwrap();
         assert!(db_area.deleted_at.is_some());
@@ -327,6 +336,7 @@ mod test {
         let area = Area::insert(
             GeoJson::from_json_value(phuket_geo_json()).unwrap(),
             tags,
+            url_alias.as_str().unwrap(),
             &mut conn,
         )?
         .unwrap();

diff --git a/src/area/v2.rs b/src/area/v2.rs
@@ -133,7 +133,12 @@ mod tests {
         let state = mock_state().await;
         let mut tags = Map::new();
         tags.insert("url_alias".into(), "test".into());
-        Area::insert(GeoJson::Feature(Feature::default()), tags, &state.conn)?;
+        Area::insert(
+            GeoJson::Feature(Feature::default()),
+            tags,
+            "test",
+            &state.conn,
+        )?;
         let app = test::init_service(
             App::new()
                 .app_data(Data::new(state.pool))
@@ -154,14 +159,21 @@ mod tests {
         Area::insert(
             GeoJson::Feature(Feature::default()),
             tags.clone(),
+            "test",
             &state.conn,
         )?;
         Area::insert(
             GeoJson::Feature(Feature::default()),
             tags.clone(),
+            "test",
+            &state.conn,
+        )?;
+        Area::insert(
+            GeoJson::Feature(Feature::default()),
+            tags,
+            "test",
             &state.conn,
         )?;
-        Area::insert(GeoJson::Feature(Feature::default()), tags, &state.conn)?;
         let app = test::init_service(
             App::new()
                 .app_data(Data::new(state.pool))
@@ -182,7 +194,12 @@ mod tests {
         let area_url_alias = "test";
         let mut tags = Map::new();
         tags.insert("url_alias".into(), Value::String(area_url_alias.into()));
-        Area::insert(GeoJson::Feature(Feature::default()), tags, &state.conn)?;
+        Area::insert(
+            GeoJson::Feature(Feature::default()),
+            tags,
+            "test",
+            &state.conn,
+        )?;
         let app = test::init_service(
             App::new()
                 .app_data(Data::new(state.pool))

diff --git a/src/area/v3.rs b/src/area/v3.rs
@@ -149,6 +149,7 @@ mod test {
         let area = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -172,18 +173,21 @@ mod test {
         let area_1 = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();
         let area_2 = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();
         let _area_3 = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();
@@ -207,13 +211,15 @@ mod test {
         let area_1 = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();
         Area::set_updated_at(area_1.id, &datetime!(2022-01-05 00:00 UTC), &state.conn)?;
         let area_2 = Area::insert(
             GeoJson::Feature(Feature::default()),
             Map::new(),
+            "test",
             &state.conn,
         )?
         .unwrap();

diff --git a/src/main.rs b/src/main.rs
@@ -75,7 +75,7 @@ async fn main() -> Result<()> {
             .wrap(from_fn(log::log))
             .wrap(NormalizePath::trim())
             .wrap(Compress::default())
-            .app_data(Data::from(pool.clone()))
+            .app_data(Data::new(pool.clone()))
             .app_data(QueryConfig::default().error_handler(error::query_error_handler))
             .service(
                 service("rpc").guard(actix_web::guard::Post()).finish(