Skip to content

Commit

Permalink
feature: more progress?
Browse files Browse the repository at this point in the history
  • Loading branch information
@timvw
timvw committed Apr 23, 2024
1 parent d2efe8c commit 7344f4e
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 6 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 6 additions & 4 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,18 @@ path = "src/lib.rs"
aws-config = "1"
aws-sdk-glue = "1"
aws-types = "1"
dashmap = "5"
datafusion = { version = "36", features = ["avro"] }
iceberg-rust = { version = "0.3", optional = true }
object_store = { version = "0.9", features = ["aws"] }
pest = "2"
pest_derive = "2"
serde = { version = "^1.0", optional = true }
serde_derive = { version = "^1.0", optional = true }
serde_json = { version = "^1.0", optional = true }
tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "fs"] }
url = "2"
dashmap = "5"
iceberg-rust = { version = "0.3", optional = true }
serde_json = { version = "1.0.116", optional = true }

[features]
default = ["iceberg"]
iceberg = ["iceberg-rust", "serde_json"]
iceberg = ["iceberg-rust", "serde", "serde_derive", "serde_json"]
10 changes: 8 additions & 2 deletions src/catalog_provider/glue.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ use datafusion::datasource::object_store::ObjectStoreRegistry;
use iceberg_rust::catalog::Catalog;

Check failure on line 27 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::catalog::Catalog` 

error: unused import: `iceberg_rust::catalog::Catalog`
  --> src/catalog_provider/glue.rs:27:5
   |
27 | use iceberg_rust::catalog::Catalog;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Check failure on line 27 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::catalog::Catalog` 

error: unused import: `iceberg_rust::catalog::Catalog`
  --> src/catalog_provider/glue.rs:27:5
   |
27 | use iceberg_rust::catalog::Catalog;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
use iceberg_rust::error::Error::TableMetadataBuilder;

Check failure on line 28 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::error::Error::TableMetadataBuilder` 

error: unused import: `iceberg_rust::error::Error::TableMetadataBuilder`
  --> src/catalog_provider/glue.rs:28:5
   |
28 | use iceberg_rust::error::Error::TableMetadataBuilder;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Check failure on line 28 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::error::Error::TableMetadataBuilder` 

error: unused import: `iceberg_rust::error::Error::TableMetadataBuilder`
  --> src/catalog_provider/glue.rs:28:5
   |
28 | use iceberg_rust::error::Error::TableMetadataBuilder;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
use iceberg_rust::spec::table_metadata::TableMetadata;
use iceberg_rust::spec::tabular::TabularMetadata;

Check failure on line 30 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::spec::tabular::TabularMetadata` 

error: unused import: `iceberg_rust::spec::tabular::TabularMetadata`
  --> src/catalog_provider/glue.rs:30:5
   |
30 | use iceberg_rust::spec::tabular::TabularMetadata;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Check failure on line 30 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::spec::tabular::TabularMetadata` 

error: unused import: `iceberg_rust::spec::tabular::TabularMetadata`
  --> src/catalog_provider/glue.rs:30:5
   |
30 | use iceberg_rust::spec::tabular::TabularMetadata;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
use iceberg_rust::table::table_builder::TableBuilder;

Check failure on line 31 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::table::table_builder::TableBuilder` 

error: unused import: `iceberg_rust::table::table_builder::TableBuilder`
  --> src/catalog_provider/glue.rs:31:5
   |
31 | use iceberg_rust::table::table_builder::TableBuilder;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Check failure on line 31 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::table::table_builder::TableBuilder` 

error: unused import: `iceberg_rust::table::table_builder::TableBuilder`
  --> src/catalog_provider/glue.rs:31:5
   |
31 | use iceberg_rust::table::table_builder::TableBuilder;
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
use iceberg_rust::util;

Check failure on line 32 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::util` 

error: unused import: `iceberg_rust::util`
  --> src/catalog_provider/glue.rs:32:5
   |
32 | use iceberg_rust::util;
   |     ^^^^^^^^^^^^^^^^^^

Check failure on line 32 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `iceberg_rust::util` 

error: unused import: `iceberg_rust::util`
  --> src/catalog_provider/glue.rs:32:5
   |
32 | use iceberg_rust::util;
   |     ^^^^^^^^^^^^^^^^^^
use iceberg_rust::util::strip_prefix;
Expand Down Expand Up @@ -258,12 +259,17 @@ impl GlueCatalogProvider {

let metadata_location = table_parameters.get("metadata_location").ok_or(GlueError::AWS(format!("Did not find metadata_location property in glue catalog")))?;

Check failure on line 260 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

useless use of `format!` 

error: useless use of `format!`
   --> src/catalog_provider/glue.rs:260:96
    |
260 | ...ion").ok_or(GlueError::AWS(format!("Did not find metadata_location property in glue catalog")))?;
    |                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.to_string()`: `"Did not find metadata_location property in glue catalog".to_string()`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#useless_format
    = note: `-D clippy::useless-format` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::useless_format)]`

Check failure on line 260 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

useless use of `format!` 

error: useless use of `format!`
   --> src/catalog_provider/glue.rs:260:96
    |
260 | ...ion").ok_or(GlueError::AWS(format!("Did not find metadata_location property in glue catalog")))?;
    |                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.to_string()`: `"Did not find metadata_location property in glue catalog".to_string()`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#useless_format
    = note: `-D clippy::useless-format` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::useless_format)]`
let path = Path::parse(&strip_prefix(metadata_location)).map_err(|_| GlueError::Other(format!("Failed to parse {} as path", metadata_location)))?;

Check failure on line 261 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

the borrowed expression implements the required traits 

error: the borrowed expression implements the required traits
   --> src/catalog_provider/glue.rs:261:32
    |
261 | ...   let path = Path::parse(&strip_prefix(metadata_location)).map_err(|_| GlueError::Other(format!("Failed to parse {} as path", metadat...
    |                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: change this to: `strip_prefix(metadata_location)`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrows_for_generic_args
    = note: `-D clippy::needless-borrows-for-generic-args` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::needless_borrows_for_generic_args)]`

Check failure on line 261 in src/catalog_provider/glue.rs

View workflow job for this annotation

GitHub Actions / clippy

the borrowed expression implements the required traits 

error: the borrowed expression implements the required traits
   --> src/catalog_provider/glue.rs:261:32
    |
261 | ...   let path = Path::parse(&strip_prefix(metadata_location)).map_err(|_| GlueError::Other(format!("Failed to parse {} as path", metadat...
    |                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: change this to: `strip_prefix(metadata_location)`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrows_for_generic_args
    = note: `-D clippy::needless-borrows-for-generic-args` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::needless_borrows_for_generic_args)]`
let metadata: TableMetadata = serde_json::from_slice(&object_store.get(&path)

let bytes = object_store.get(&path)
.await.map_err(|e| GlueError::Other(format!("Failed to fetch {e:?} at {path}")))?
.bytes()
.await.map_err(|e|GlueError::Other(format!("Failed to get bytes from {e:?} at {path}")))?)
.await.map_err(|e|GlueError::Other(format!("Failed to get bytes from {e:?} at {path}")))?;
//println!("bytes are as following: {bytes:?}");

let metadata: TableMetadata = serde_json::from_slice(&bytes)
.map_err(|e|GlueError::Other(format!("Failed to read metadata from {e:?} at {path}")))?;


println!("metadata: {metadata:?}");


Expand Down

0 comments on commit 7344f4e

Please sign in to comment.