diff --git a/Cargo.lock b/Cargo.lock index 30503bb..5103deb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,7 @@ dependencies = [ "duckdb", "env_logger", "fallible-streaming-iterator", + "half", "hex", "insta", "itertools", @@ -524,6 +525,8 @@ dependencies = [ "parquet", "postgres", "postgres-protocol", + "rand", + "rand_chacha", "rusqlite", "rust_decimal", "rust_decimal_macros", @@ -1902,17 +1905,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "test_generator" -version = "0.1.0" -dependencies = [ - "arrow", - "half", - "parquet", - "rand", - "rand_chacha", -] - [[package]] name = "thiserror" version = "1.0.56" diff --git a/Cargo.toml b/Cargo.toml index fec7155..b74c6db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["connector_arrow", "test_generator"] +members = ["connector_arrow"] resolver = "2" diff --git a/connector_arrow/Cargo.toml b/connector_arrow/Cargo.toml index e52459f..e6930d1 100644 --- a/connector_arrow/Cargo.toml +++ b/connector_arrow/Cargo.toml @@ -61,6 +61,10 @@ arrow = { version = "49", features = ["prettyprint"], default-features = false } parquet = { version = "49", features = ["arrow"], default-features = false } insta = { version = "1.34.0" } similar-asserts = { version = "1.5.0" } +half = "2.3.1" +rand = { version = "0.8.5", default-features = false } +rand_chacha = "0.3.1" + [features] all = ["src_sqlite", "src_duckdb", "src_postgres"] @@ -73,7 +77,7 @@ src_postgres = [ "rust_decimal", "rust_decimal_macros", "bytes", - "byteorder" + "byteorder", ] src_sqlite = ["rusqlite", "fallible-streaming-iterator", "urlencoding"] src_duckdb = [ diff --git a/connector_arrow/src/duckdb/mod.rs b/connector_arrow/src/duckdb/mod.rs index e8681a0..4110835 100644 --- a/connector_arrow/src/duckdb/mod.rs +++ b/connector_arrow/src/duckdb/mod.rs @@ -31,6 +31,7 @@ impl Connection for duckdb::Connection { fn coerce_type(ty: &DataType) -> Option { match ty { DataType::Null => Some(DataType::Int64), + DataType::Float16 => Some(DataType::Float32), _ => None, } } diff --git a/connector_arrow/src/util/coerce.rs b/connector_arrow/src/util/coerce.rs index 983697d..c4690fb 100644 --- a/connector_arrow/src/util/coerce.rs +++ b/connector_arrow/src/util/coerce.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use arrow::array::{Array, ArrayRef}; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::array::{Array, ArrayRef, AsArray, Float32Builder, Float64Builder}; +use arrow::datatypes::{DataType, Field, Float16Type, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use itertools::Itertools; @@ -38,7 +38,11 @@ where F: Fn(&DataType) -> Option + Copy, { match coerce_fn(array.data_type()) { - Some(new_ty) => arrow::compute::cast(&array, &new_ty), + Some(new_ty) => match (array.data_type(), &new_ty) { + (DataType::Float16, DataType::Float32) => Ok(coerce_float_16_to_32(&array)), + (DataType::Float16, DataType::Float64) => Ok(coerce_float_16_to_64(&array)), + _ => arrow::compute::cast(&array, &new_ty), + }, None => Ok(array), } } @@ -58,3 +62,35 @@ where .collect_vec(), )) } + +fn coerce_float_16_to_32(array: &dyn Array) -> ArrayRef { + // inefficient, but we don't need efficiency here + + let array = array.as_primitive::(); + let mut builder = Float32Builder::with_capacity(array.len()); + + for i in 0..array.len() { + if array.is_null(i) { + builder.append_null(); + } else { + builder.append_value(array.value(i).to_f32()); + } + } + Arc::new(builder.finish()) as ArrayRef +} + +fn coerce_float_16_to_64(array: &dyn Array) -> ArrayRef { + // inefficient, but we don't need efficiency here + + let array = array.as_primitive::(); + let mut builder = Float64Builder::with_capacity(array.len()); + + for i in 0..array.len() { + if array.is_null(i) { + builder.append_null(); + } else { + builder.append_value(array.value(i).to_f64()); + } + } + Arc::new(builder.finish()) as ArrayRef +} diff --git a/connector_arrow/tests/data/numeric.parquet b/connector_arrow/tests/data/numeric.parquet deleted file mode 100644 index 5a0c854..0000000 Binary files a/connector_arrow/tests/data/numeric.parquet and /dev/null differ diff --git a/connector_arrow/tests/data/temporal.parquet b/connector_arrow/tests/data/temporal.parquet deleted file mode 100644 index f13b972..0000000 Binary files a/connector_arrow/tests/data/temporal.parquet and /dev/null differ diff --git a/test_generator/src/main.rs b/connector_arrow/tests/it/generator.rs similarity index 68% rename from test_generator/src/main.rs rename to connector_arrow/tests/it/generator.rs index 622b2ba..26e6c6b 100644 --- a/test_generator/src/main.rs +++ b/connector_arrow/tests/it/generator.rs @@ -1,11 +1,143 @@ use arrow::array::*; -use arrow::datatypes::{DataType, Field, Int32Type, Int64Type, IntervalUnit, Schema, TimeUnit}; +use arrow::datatypes::*; use half::f16; -use rand::{Rng, SeedableRng}; -use std::fs::File; -use std::path::Path; +use rand::Rng; use std::sync::Arc; +pub fn generate_batch(column_specs: Vec, rng: &mut R) -> RecordBatch { + let mut arrays = Vec::new(); + let mut fields = Vec::new(); + for column in column_specs { + let array = generate_array(&column.data_type, &column.values, rng); + arrays.push(array); + + let field = Field::new(column.field_name, column.data_type, column.is_nullable); + fields.push(field); + } + let schema = Arc::new(Schema::new(fields)); + RecordBatch::try_new(schema, arrays).unwrap() +} + +pub fn spec_simple() -> Vec { + domains_to_batch_spec(&[DataType::Null, DataType::Boolean], &[false, true]) +} + +pub fn spec_numeric() -> Vec { + domains_to_batch_spec( + &[ + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + DataType::UInt64, + DataType::Float16, + DataType::Float32, + DataType::Float64, + ], + &[false, true], + ) +} + +pub fn spec_timestamp() -> Vec { + domains_to_batch_spec( + &[ + DataType::Timestamp(TimeUnit::Nanosecond, None), + DataType::Timestamp(TimeUnit::Microsecond, None), + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Timestamp(TimeUnit::Second, None), + DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::from("+07:30"))), + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("+07:30"))), + DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::from("+07:30"))), + DataType::Timestamp(TimeUnit::Second, Some(Arc::from("+07:30"))), + ], + &[true], + ) +} +pub fn spec_date() -> Vec { + domains_to_batch_spec(&[DataType::Date32, DataType::Date64], &[true]) +} +pub fn spec_time() -> Vec { + domains_to_batch_spec( + &[ + DataType::Time32(TimeUnit::Millisecond), + DataType::Time32(TimeUnit::Second), + DataType::Time64(TimeUnit::Nanosecond), + DataType::Time64(TimeUnit::Microsecond), + ], + &[true], + ) +} +pub fn spec_duration() -> Vec { + domains_to_batch_spec( + &[ + DataType::Duration(TimeUnit::Nanosecond), + DataType::Duration(TimeUnit::Microsecond), + DataType::Duration(TimeUnit::Millisecond), + DataType::Duration(TimeUnit::Second), + ], + &[true], + ) +} +pub fn spec_interval() -> Vec { + domains_to_batch_spec( + &[ + DataType::Interval(IntervalUnit::YearMonth), + DataType::Interval(IntervalUnit::MonthDayNano), + DataType::Interval(IntervalUnit::DayTime), + ], + &[true], + ) +} + +pub fn domains_to_batch_spec( + data_types_domain: &[DataType], + is_nullable_domain: &[bool], +) -> Vec { + let value_gen_process_domain = [ + ValueGenProcess::Low, + ValueGenProcess::High, + ValueGenProcess::Null, + ValueGenProcess::RandomUniform, + ]; + + let mut columns = Vec::new(); + for data_type in data_types_domain { + for is_nullable in is_nullable_domain { + let is_nullable = *is_nullable; + if matches!(data_type, &DataType::Null) && !is_nullable { + continue; + } + + let mut field_name = data_type.to_string(); + if is_nullable { + field_name += "_null"; + } + let mut col = ColumnSpec { + field_name, + data_type: data_type.clone(), + is_nullable, + values: Vec::new(), + }; + + for gen_process in value_gen_process_domain { + col.values.push(ValuesSpec { + gen_process: if matches!(gen_process, ValueGenProcess::Null) && !is_nullable { + ValueGenProcess::RandomUniform + } else { + gen_process + }, + repeat: 1, + }); + } + columns.push(col); + } + } + columns +} + #[derive(Clone, Copy)] enum ValueGenProcess { Null, @@ -14,19 +146,19 @@ enum ValueGenProcess { RandomUniform, } -struct ValuesDesc { +struct ValuesSpec { gen_process: ValueGenProcess, repeat: usize, } -struct ColumnDesc { +pub struct ColumnSpec { field_name: String, is_nullable: bool, data_type: DataType, - values: Vec, + values: Vec, } -fn count_values(values: &[ValuesDesc]) -> usize { +fn count_values(values: &[ValuesSpec]) -> usize { values.iter().map(|v| v.repeat).sum() } @@ -48,7 +180,7 @@ macro_rules! gen_array { }}; } -fn generate_array(data_type: &DataType, values: &[ValuesDesc], rng: &mut R) -> ArrayRef { +fn generate_array(data_type: &DataType, values: &[ValuesSpec], rng: &mut R) -> ArrayRef { match data_type { DataType::Null => { let mut builder = NullBuilder::with_capacity(count_values(values)); @@ -258,160 +390,3 @@ fn generate_array(data_type: &DataType, values: &[ValuesDesc], rng: &mut DataType::RunEndEncoded(_, _) => todo!(), } } - -fn generate_batch(columns_desc: Vec, rng: &mut R) -> RecordBatch { - let mut arrays = Vec::new(); - let mut fields = Vec::new(); - for column in columns_desc { - let array = generate_array(&column.data_type, &column.values, rng); - arrays.push(array); - - let field = Field::new(column.field_name, column.data_type, column.is_nullable); - fields.push(field); - } - let schema = Arc::new(Schema::new(fields)); - RecordBatch::try_new(schema, arrays).unwrap() -} - -fn numeric() -> Vec { - let data_types_domain = [ - DataType::Null, - DataType::Boolean, - DataType::Int8, - DataType::Int16, - DataType::Int32, - DataType::Int64, - DataType::UInt8, - DataType::UInt16, - DataType::UInt32, - DataType::UInt64, - // DataType::Float16, - DataType::Float32, - DataType::Float64, - ]; - let is_nullable_domain = [false, true]; - let value_gen_process_domain = [ - ValueGenProcess::Low, - ValueGenProcess::High, - ValueGenProcess::Null, - ValueGenProcess::RandomUniform, - ]; - - let mut columns = Vec::new(); - for data_type in &data_types_domain { - for is_nullable in is_nullable_domain { - if matches!(data_type, &DataType::Null) && !is_nullable { - continue; - } - - let mut field_name = data_type.to_string(); - if is_nullable { - field_name += "_null"; - } - let mut col = ColumnDesc { - field_name, - data_type: data_type.clone(), - is_nullable, - values: Vec::new(), - }; - - for gen_process in value_gen_process_domain { - col.values.push(ValuesDesc { - gen_process: if matches!(gen_process, ValueGenProcess::Null) && !is_nullable { - ValueGenProcess::RandomUniform - } else { - gen_process - }, - repeat: 1, - }); - } - columns.push(col); - } - } - columns -} - -fn temporal() -> Vec { - let data_types_domain = [ - DataType::Timestamp(TimeUnit::Nanosecond, None), - DataType::Timestamp(TimeUnit::Microsecond, None), - DataType::Timestamp(TimeUnit::Millisecond, None), - DataType::Timestamp(TimeUnit::Second, None), - DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::from("+07:30"))), - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("+07:30"))), - DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::from("+07:30"))), - DataType::Timestamp(TimeUnit::Second, Some(Arc::from("+07:30"))), - DataType::Date32, - DataType::Date64, - DataType::Time32(TimeUnit::Millisecond), - DataType::Time32(TimeUnit::Second), - DataType::Time64(TimeUnit::Nanosecond), - DataType::Time64(TimeUnit::Microsecond), - // DataType::Duration(TimeUnit::Nanosecond), - // DataType::Duration(TimeUnit::Microsecond), - // DataType::Duration(TimeUnit::Millisecond), - // DataType::Duration(TimeUnit::Second), - DataType::Interval(IntervalUnit::YearMonth), - // DataType::Interval(IntervalUnit::MonthDayNano), - DataType::Interval(IntervalUnit::DayTime), - ]; - let is_nullable_domain = [true]; - let value_gen_process_domain = [ - ValueGenProcess::Low, - ValueGenProcess::High, - ValueGenProcess::Null, - ValueGenProcess::RandomUniform, - ]; - - let mut columns = Vec::new(); - for data_type in &data_types_domain { - for is_nullable in is_nullable_domain { - if matches!(data_type, &DataType::Null) && !is_nullable { - continue; - } - - let mut field_name = data_type.to_string(); - if is_nullable { - field_name += "_null"; - } - let mut col = ColumnDesc { - field_name, - data_type: data_type.clone(), - is_nullable, - values: Vec::new(), - }; - - for gen_process in value_gen_process_domain { - col.values.push(ValuesDesc { - gen_process: if matches!(gen_process, ValueGenProcess::Null) && !is_nullable { - ValueGenProcess::RandomUniform - } else { - gen_process - }, - repeat: 1, - }); - } - columns.push(col); - } - } - columns -} - -fn write_parquet_to_file(batch: RecordBatch, file_name: &str) { - let path = Path::new("connector_arrow/tests/data/file").with_file_name(file_name); - - let mut file = File::create(path).unwrap(); - - let schema = batch.schema(); - let mut writer = - parquet::arrow::arrow_writer::ArrowWriter::try_new(&mut file, schema, None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); -} - -fn main() { - let mut rng = rand_chacha::ChaCha8Rng::from_seed([0; 32]); - - write_parquet_to_file(generate_batch(numeric(), &mut rng), "numeric.parquet"); - write_parquet_to_file(generate_batch(temporal(), &mut rng), "temporal.parquet"); -} diff --git a/connector_arrow/tests/it/main.rs b/connector_arrow/tests/it/main.rs index 1fbd683..f3a2410 100644 --- a/connector_arrow/tests/it/main.rs +++ b/connector_arrow/tests/it/main.rs @@ -1,3 +1,4 @@ +mod generator; mod tests; mod util; diff --git a/connector_arrow/tests/it/test_duckdb.rs b/connector_arrow/tests/it/test_duckdb.rs index 4b89a06..cabdb71 100644 --- a/connector_arrow/tests/it/test_duckdb.rs +++ b/connector_arrow/tests/it/test_duckdb.rs @@ -29,22 +29,21 @@ fn roundtrip_empty() { } #[test] -fn roundtrip_numeric() { - let table_name = "roundtrip_number"; - let file_name = "numeric.parquet"; +fn roundtrip_simple() { + let table_name = "roundtrip_simple"; let mut conn = init(); - super::tests::roundtrip_of_parquet(&mut conn, file_name, table_name); + let column_spec = super::generator::spec_simple(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); } #[test] -#[ignore] -fn roundtrip_temporal() { - let table_name = "roundtrip_temporal"; - let file_name = "temporal.parquet"; +fn roundtrip_numeric() { + let table_name = "roundtrip_numeric"; let mut conn = init(); - super::tests::roundtrip_of_parquet(&mut conn, file_name, table_name); + let column_spec = super::generator::spec_numeric(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); } #[test] diff --git a/connector_arrow/tests/it/test_sqlite.rs b/connector_arrow/tests/it/test_sqlite.rs index 1c4c9fd..295352a 100644 --- a/connector_arrow/tests/it/test_sqlite.rs +++ b/connector_arrow/tests/it/test_sqlite.rs @@ -29,23 +29,72 @@ fn roundtrip_empty() { super::tests::roundtrip_of_parquet(&mut conn, file_name, table_name); } +#[test] +fn roundtrip_simple() { + let table_name = "roundtrip_simple"; + + let mut conn = init(); + let column_spec = super::generator::spec_simple(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); +} + #[test] fn roundtrip_numeric() { - let table_name = "roundtrip_number"; - let file_name = "numeric.parquet"; + let table_name = "roundtrip_numeric"; let mut conn = init(); - super::tests::roundtrip_of_parquet(&mut conn, file_name, table_name); + let column_spec = super::generator::spec_numeric(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); } #[test] #[ignore] -fn roundtrip_temporal() { - let table_name = "roundtrip_temporal"; - let file_name = "temporal.parquet"; +fn roundtrip_timestamp() { + let table_name = "roundtrip_timestamp"; let mut conn = init(); - super::tests::roundtrip_of_parquet(&mut conn, file_name, table_name); + let column_spec = super::generator::spec_timestamp(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); +} + +#[test] +#[ignore] +fn roundtrip_date() { + let table_name = "roundtrip_date"; + + let mut conn = init(); + let column_spec = super::generator::spec_date(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); +} + +#[test] +#[ignore] +fn roundtrip_time() { + let table_name = "roundtrip_time"; + + let mut conn = init(); + let column_spec = super::generator::spec_time(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); +} + +#[test] +#[ignore] +fn roundtrip_duration() { + let table_name = "roundtrip_duration"; + + let mut conn = init(); + let column_spec = super::generator::spec_duration(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); +} + +#[test] +#[ignore] +fn roundtrip_interval() { + let table_name = "roundtrip_interval"; + + let mut conn = init(); + let column_spec = super::generator::spec_interval(); + super::tests::roundtrip_of_generated(&mut conn, table_name, column_spec); } #[test] diff --git a/connector_arrow/tests/it/tests.rs b/connector_arrow/tests/it/tests.rs index aab49d7..ee3192f 100644 --- a/connector_arrow/tests/it/tests.rs +++ b/connector_arrow/tests/it/tests.rs @@ -5,8 +5,13 @@ use connector_arrow::{ api::{Connection, ResultReader, SchemaEdit, SchemaGet, Statement}, TableCreateError, TableDropError, }; +use rand::SeedableRng; -use crate::util::{load_parquet_into_table, query_table}; +use crate::util::{load_into_table, query_table}; +use crate::{ + generator::{generate_batch, ColumnSpec}, + util::read_parquet, +}; #[track_caller] pub fn query_01(conn: &mut C) { @@ -15,11 +20,11 @@ pub fn query_01(conn: &mut C) { similar_asserts::assert_eq!( pretty_format_batches(&results).unwrap().to_string(), - "+---+---+ -| a | b | -+---+---+ -| 1 | | -+---+---+" + "+---+---+\n\ + | a | b |\n\ + +---+---+\n\ + | 1 | |\n\ + +---+---+" ); } @@ -29,21 +34,36 @@ where { let file_path = Path::new("./tests/data/a").with_file_name(file_name); + let (schema_file, batches_file) = read_parquet(&file_path).unwrap(); let (schema_file, batches_file) = - load_parquet_into_table(conn, &file_path, table_name).unwrap(); + load_into_table(conn, schema_file, batches_file, table_name).unwrap(); let (schema_query, batches_query) = query_table(conn, table_name).unwrap(); similar_asserts::assert_eq!(schema_file, schema_query); similar_asserts::assert_eq!(batches_file, batches_query); } +pub fn roundtrip_of_generated(conn: &mut C, table_name: &str, column_specs: Vec) +where + C: Connection + SchemaEdit, +{ + let mut rng = rand_chacha::ChaCha8Rng::from_seed([0; 32]); + let batch = generate_batch(column_specs, &mut rng); + + let (_, batches_file) = load_into_table(conn, batch.schema(), vec![batch], table_name).unwrap(); + + let (_, batches_query) = query_table(conn, table_name).unwrap(); + + similar_asserts::assert_eq!(batches_file, batches_query); +} + pub fn introspection(conn: &mut C, file_name: &str, table_name: &str) where C: Connection + SchemaEdit + SchemaGet, { let file_path = Path::new("./tests/data/a").with_file_name(file_name); - let (schema_loaded, _) = - super::util::load_parquet_into_table(conn, &file_path, table_name).unwrap(); + let (schema_file, batches_file) = read_parquet(&file_path).unwrap(); + let (schema_loaded, _) = load_into_table(conn, schema_file, batches_file, table_name).unwrap(); let schema_introspection = conn.table_get(table_name).unwrap(); similar_asserts::assert_eq!(schema_loaded, schema_introspection); @@ -55,7 +75,8 @@ where { let file_path = Path::new("./tests/data/a").with_file_name(file_name); - let (schema, _) = super::util::load_parquet_into_table(conn, &file_path, table_name).unwrap(); + let (schema_file, batches_file) = read_parquet(&file_path).unwrap(); + let (schema, _) = load_into_table(conn, schema_file, batches_file, table_name).unwrap(); let table_name2 = table_name.to_string() + "2"; diff --git a/connector_arrow/tests/it/util.rs b/connector_arrow/tests/it/util.rs index 8ea9840..42074c8 100644 --- a/connector_arrow/tests/it/util.rs +++ b/connector_arrow/tests/it/util.rs @@ -22,16 +22,26 @@ pub fn read_parquet(file_path: &Path) -> Result<(SchemaRef, Vec), A Ok((schema, batches)) } -pub fn load_parquet_into_table( +#[allow(dead_code)] +pub fn write_parquet(path: &Path, batch: RecordBatch) { + let mut file = File::create(path).unwrap(); + + let schema = batch.schema(); + let mut writer = + parquet::arrow::arrow_writer::ArrowWriter::try_new(&mut file, schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +pub fn load_into_table( conn: &mut C, - file_path: &Path, + schema: SchemaRef, + batches: Vec, table_name: &str, ) -> Result<(SchemaRef, Vec), ConnectorError> where C: Connection + SchemaEdit, { - let (schema_file, batches_file) = read_parquet(file_path)?; - // table drop match conn.table_drop(table_name) { Ok(_) | Err(TableDropError::TableNonexistent) => (), @@ -39,7 +49,7 @@ where } // table create - match conn.table_create(table_name, schema_file.clone()) { + match conn.table_create(table_name, schema.clone()) { Ok(_) => (), Err(TableCreateError::TableExists) => { panic!("table was just deleted, how can it exist now?") @@ -50,14 +60,14 @@ where // write into table { let mut appender = conn.append(&table_name).unwrap(); - for batch in batches_file.clone() { + for batch in batches.clone() { appender.append(batch).unwrap(); } appender.finish().unwrap(); } - let schema_coerced = coerce::coerce_schema(schema_file, &C::coerce_type); - let batches_coerced = coerce::coerce_batches(&batches_file, C::coerce_type).unwrap(); + let schema_coerced = coerce::coerce_schema(schema, &C::coerce_type); + let batches_coerced = coerce::coerce_batches(&batches, C::coerce_type).unwrap(); Ok((schema_coerced, batches_coerced)) } diff --git a/test_generator/Cargo.toml b/test_generator/Cargo.toml deleted file mode 100644 index f083cc2..0000000 --- a/test_generator/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "test_generator" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -arrow = { version = "49", features = ["prettyprint"], default-features = false } -half = "2.3.1" -parquet = { version = "49", features = ["arrow"], default-features = false } -rand = {version = "0.8.5", default-features = false} -rand_chacha = "0.3.1"