Add Docker tests in remi-gridfs, fix regex in regex custom manager …

…(hopefully)

crates/azure/src/service.rs

@@ -177,6 +177,10 @@ impl remi::StorageService for StorageService {
         );
 
         let client = self.container.blob_client(self.sanitize_path(path)?);
+        if !client.exists().await? {
+            return Ok(None);
+        }
+
         let props = client.get_properties().await?;
         let data = Bytes::from(client.get_content().await?);
 
@@ -443,7 +447,7 @@ mod tests {
 
     const IMAGE: &str = "mcr.microsoft.com/azure-storage/azurite";
 
-    // renovate: image=microsoft-azure-storage-azurite
+    // renovate: image="microsoft-azure-storage-azurite"
     const TAG: &str = "3.29.0";
 
     fn container() -> GenericImage {
@@ -532,5 +536,41 @@ mod tests {
             assert!(storage.exists("./wuff.json").await.expect("failed to query ./wuff.json"));
             assert_eq!(contents, storage.open("./wuff.json").await.expect("failed to open ./wuff.json").expect("it should exist"));
         }
+
+        async fn list_blobs(storage) {
+            for i in 0..100 {
+                let contents: remi::Bytes = format!("{{\"blob\":{i}}}").into();
+                storage.upload(format!("./wuff.{i}.json"), UploadRequest::default()
+                    .with_content_type(Some("application/json"))
+                    .with_data(contents)
+                ).await.expect("failed to upload blob");
+            }
+
+            let blobs = storage.blobs(None::<&str>, None).await.expect("failed to list all blobs");
+            let iter = blobs.iter().filter_map(|x| match x {
+                remi::Blob::File(file) => Some(file),
+                _ => None
+            });
+
+            assert!(iter.clone().all(|x|
+                x.content_type == Some(String::from("application/json")) &&
+                !x.is_symlink &&
+                x.data.starts_with(&[/* b"{" */ 123])
+            ));
+        }
+
+        async fn query_single_blob(storage) {
+            for i in 0..100 {
+                let contents: remi::Bytes = format!("{{\"blob\":{i}}}").into();
+                storage.upload(format!("./wuff.{i}.json"), UploadRequest::default()
+                    .with_content_type(Some("application/json"))
+                    .with_data(contents)
+                ).await.expect("failed to upload blob");
+            }
+
+            assert!(storage.blob("./wuff.98.json").await.expect("failed to query single blob").is_some());
+            assert!(storage.blob("./wuff.95.json").await.expect("failed to query single blob").is_some());
+            assert!(storage.blob("~/doesnt/exist").await.expect("failed to query single blob").is_none());
+        }
     }
 }

crates/gridfs/Cargo.toml

@@ -45,5 +45,12 @@ serde = { version = "1.0.200", features = ["derive"], optional = true }
 tokio-util = "0.7.11"
 tracing = { version = "0.1.40", optional = true }
 
+[dev-dependencies]
+bollard = "0.16.1"
+testcontainers = "0.16.7"
+tokio = { version = "1.37.0", features = ["rt", "macros"] }
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+
 [package.metadata.docs.rs]
 all-features = true

crates/gridfs/src/service.rs

@@ -24,7 +24,7 @@ use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
 use futures_util::{AsyncWriteExt, StreamExt};
 use mongodb::{
-    bson::{doc, raw::ValueAccessErrorKind, Bson, RawDocument},
+    bson::{doc, raw::ValueAccessErrorKind, Bson, Document, RawDocument},
     options::{GridFsFindOptions, GridFsUploadOptions},
     Client, Database, GridFsBucket,
 };
@@ -54,10 +54,15 @@ fn document_to_blob(bytes: Bytes, doc: &RawDocument) -> Result<File, mongodb::er
     let filename = doc.get_str("filename").map_err(value_access_err_to_error)?;
     let length = doc.get_i64("length").map_err(value_access_err_to_error)?;
     let created_at = doc.get_datetime("uploadDate").map_err(value_access_err_to_error)?;
-    let content_type = match doc.get_str("contentType") {
+    let metadata = doc.get_document("metadata").map_err(value_access_err_to_error)?;
+
+    let content_type = match metadata.get_str("contentType") {
         Ok(res) => Some(res),
         Err(e) => match e.kind {
-            ValueAccessErrorKind::NotPresent => None,
+            ValueAccessErrorKind::NotPresent => match metadata.get_str("contentType") {
+                Ok(res) => Some(res),
+                Err(e) => return Err(value_access_err_to_error(e)),
+            },
             _ => return Err(value_access_err_to_error(e)),
         },
     };
@@ -68,9 +73,9 @@ fn document_to_blob(bytes: Bytes, doc: &RawDocument) -> Result<File, mongodb::er
     // For brevity and compatibility with other storage services, we only use strings
     // when including metadata.
     let mut map = HashMap::new();
-    for ref_ in doc.into_iter() {
+    for ref_ in metadata.into_iter() {
         let (name, doc) = ref_?;
-        if name != "filename" || name != "length" || name != "uploadDate" || name != "contentType" {
+        if name != "contentType" {
             if let Some(s) = doc.as_str() {
                 map.insert(name.into(), s.into());
             }
@@ -110,6 +115,20 @@ fn document_to_blob(bytes: Bytes, doc: &RawDocument) -> Result<File, mongodb::er
     })
 }
 
+fn resolve_path(path: &Path) -> Result<String, mongodb::error::Error> {
+    let path = path.to_str().ok_or_else(|| {
+        <mongodb::error::Error as From<io::Error>>::from(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "expected valid utf-8 string",
+        ))
+    })?;
+
+    // trim `./` and `~/` since Gridfs doesn't accept ./ or ~/ as valid paths
+    let path = path.trim_start_matches("~/").trim_start_matches("./");
+
+    Ok(path.to_owned())
+}
+
 #[derive(Debug, Clone)]
 pub struct StorageService {
     config: Option<StorageConfig>,
@@ -150,17 +169,7 @@ impl StorageService {
     }
 
     fn resolve_path<P: AsRef<Path>>(&self, path: P) -> Result<String, mongodb::error::Error> {
-        let path = path.as_ref().to_str().ok_or_else(|| {
-            <mongodb::error::Error as From<io::Error>>::from(io::Error::new(
-                io::ErrorKind::InvalidData,
-                "expected valid utf-8 string",
-            ))
-        })?;
-
-        // trim `./` and `~/` since S3 doesn't accept ./ or ~/ as valid paths
-        let path = path.trim_start_matches("~/").trim_start_matches("./");
-
-        Ok(path.to_owned())
+        resolve_path(path.as_ref())
     }
 }
 
@@ -448,27 +457,147 @@ impl remi::StorageService for StorageService {
         #[cfg(feature = "log")]
         ::log::info!("uploading file [{}] to GridFS", path);
 
+        let mut metadata = options
+            .metadata
+            .into_iter()
+            .map(|(key, value)| (key, Bson::String(value)))
+            .collect::<Document>();
+
+        if let Some(ct) = options.content_type {
+            metadata.insert("contentType", ct);
+        }
+
         let opts = GridFsUploadOptions::builder()
             .chunk_size_bytes(Some(
                 self.config.clone().unwrap_or_default().chunk_size.unwrap_or(255 * 1024),
             ))
-            .metadata(match options.metadata.is_empty() {
-                true => None,
-                false => Some(
-                    options
-                        .metadata
-                        .into_iter()
-                        .map(|(k, v)| (k, Bson::String(v)))
-                        .collect(),
-                ),
-            })
+            .metadata(metadata)
             .build();
 
         let mut stream = self.bucket.open_upload_stream(path, Some(opts));
         stream.write_all(&options.data[..]).await?;
         stream.close().await.map_err(From::from)
+    }
+}
+
+#[cfg(test)]
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+mod tests {
+    use crate::service::resolve_path;
+    use remi::{StorageService, UploadRequest};
+    use std::path::Path;
+    use testcontainers::{runners::AsyncRunner, GenericImage};
+    use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
+    const IMAGE: &str = "mongo";
+
+    // renovate: image="mongo"
+    const TAG: &str = "7.0.9";
+
+    fn container() -> GenericImage {
+        GenericImage::new(IMAGE, TAG).with_exposed_port(27017)
+    }
 
-        // TODO(@auguwu): add metadata to document that was created and the given content type
-        // if one was supplied.
+    #[test]
+    fn test_resolve_paths() {
+        assert_eq!(resolve_path(Path::new("./weow.txt")).unwrap(), String::from("weow.txt"));
+        assert_eq!(resolve_path(Path::new("~/weow.txt")).unwrap(), String::from("weow.txt"));
+        assert_eq!(resolve_path(Path::new("weow.txt")).unwrap(), String::from("weow.txt"));
+        assert_eq!(
+            resolve_path(Path::new("~/weow/fluff/mooo.exe")).unwrap(),
+            String::from("weow/fluff/mooo.exe")
+        );
+    }
+
+    macro_rules! build_testcases {
+        (
+            $(
+                $(#[$meta:meta])*
+                async fn $name:ident($storage:ident) $code:block
+            )*
+        ) => {
+            $(
+                #[cfg_attr(target_os = "linux", tokio::test)]
+                #[cfg_attr(not(target_os = "linux"), ignore = "azurite image can be only used on Linux")]
+                $(#[$meta])*
+                async fn $name() {
+                    if ::bollard::Docker::connect_with_defaults().is_err() {
+                        eprintln!("[remi-gridfs] `docker` cannot be probed by default settings; skipping test");
+                        return;
+                    }
+
+                    let _guard = tracing_subscriber::registry()
+                        .with(tracing_subscriber::fmt::layer())
+                        .set_default();
+
+                    let container = container().start().await;
+                    let $storage = crate::StorageService::from_conn_string(
+                        format!("mongodb://{}:{}", container.get_host().await,container.get_host_port_ipv4(27017).await),
+                        $crate::StorageConfig {
+                            database: Some(String::from("remi")),
+                            bucket: String::from("fs"),
+
+                            ..Default::default()
+                        }
+                    ).await.expect("failed to create storage service");
+
+                    ($storage).init().await.expect("failed to initialize storage service");
+
+                    let __ret = $code;
+                    __ret
+                }
+            )*
+        };
+    }
+
+    build_testcases! {
+        async fn prepare_mongo_container_usage(_storage) {}
+
+        async fn test_uploading_file(storage) {
+            let contents: remi::Bytes = "{\"wuff\":true}".into();
+            storage.upload("./wuff.json", UploadRequest::default()
+                .with_content_type(Some("application/json"))
+                .with_data(contents.clone())
+            ).await.expect("failed to upload");
+
+            assert!(storage.exists("./wuff.json").await.expect("failed to query ./wuff.json"));
+            assert_eq!(contents, storage.open("./wuff.json").await.expect("failed to open ./wuff.json").expect("it should exist"));
+        }
+
+        async fn list_blobs(storage) {
+            for i in 0..100 {
+                let contents: remi::Bytes = format!("{{\"blob\":{i}}}").into();
+                storage.upload(format!("./wuff.{i}.json"), UploadRequest::default()
+                    .with_content_type(Some("application/json"))
+                    .with_data(contents)
+                ).await.expect("failed to upload blob");
+            }
+
+            let blobs = storage.blobs(None::<&str>, None).await.expect("failed to list all blobs");
+            let mut iter = blobs.iter().filter_map(|x| match x {
+                remi::Blob::File(file) => Some(file),
+                _ => None
+            });
+
+            assert!(iter.all(|x|
+                x.content_type == Some(String::from("application/json")) &&
+                !x.is_symlink &&
+                x.data.starts_with(&[/* b"{" */ 123])
+            ));
+        }
+
+        async fn query_single_blob(storage) {
+            for i in 0..100 {
+                let contents: remi::Bytes = format!("{{\"blob\":{i}}}").into();
+                storage.upload(format!("./wuff.{i}.json"), UploadRequest::default()
+                    .with_content_type(Some("application/json"))
+                    .with_data(contents)
+                ).await.expect("failed to upload blob");
+            }
+
+            assert!(storage.blob("./wuff.98.json").await.expect("failed to query single blob").is_some());
+            assert!(storage.blob("./wuff.95.json").await.expect("failed to query single blob").is_some());
+            assert!(storage.blob("~/doesnt/exist").await.expect("failed to query single blob").is_none());
+        }
     }
 }

remi/src/blob.rs

@@ -23,6 +23,7 @@ use bytes::Bytes;
 use std::{collections::HashMap, fmt::Display};
 
 /// Represents a file or directory from any storage service.
+#[derive(Debug, Clone)]
 pub enum Blob {
     /// Represents a directory that was located somewhere.
     Directory(Directory),
@@ -69,16 +70,13 @@ pub struct File {
 
 impl Display for File {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // file "assets/openapi.json" (12345 bytes) | application/json; charset=utf-8
-        f.write_fmt(format_args!(
-            "file [{}] ({} bytes){}",
-            self.path,
-            self.size,
-            match self.content_type {
-                Some(ref ct) => format!(" | {ct}"),
-                None => "".into(),
-            }
-        ))
+        // file "file:///assets/openapi.json" (12345 bytes) | application/json; charset=utf-8
+        write!(f, "file [{}] ({} bytes)", self.path, self.size)?;
+        if let Some(ref ct) = self.content_type {
+            write!(f, " | {ct}")?;
+        }
+
+        Ok(())
     }
 }
 
@@ -98,6 +96,6 @@ pub struct Directory {
 
 impl Display for Directory {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_fmt(format_args!("directory {}", self.path))
+        write!(f, "directory {}", self.path)
     }
 }

renovate.json

@@ -9,10 +9,10 @@
     "customManagers": [
         {
             "customType": "regex",
-            "fileMatch": [".rs$"],
+            "fileMatch": ["**/*.rs"],
             "datasourceTemplate": "docker",
             "matchStrings": [
-                "\/\/ renovate: image=(?<depName>.*)\nconst TAG: &str = \"(?<currentValue>.*)\";"
+                "\/\/\\s?renovate: image=\"(?<depName>.*?)\"\\s?const ([A-Z]*): &str = \"?(?<currentValue>[\\w.-]*)\";"
             ]
         }
     ]