Cargo.toml

[package]
name = "dolma"
version = "1.1.0"
edition = "2021"
license = "Apache-2.0"


# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "dolma"
crate-type = ["cdylib"]

[dependencies]
ahash = { version = "0.8.1", features = ["runtime-rng"] }
anyhow = "1.0"
atomic-traits = "0.3"
aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
aws-sdk-s3 = "1.22.0"
byteorder = "1"
clap = { version = "4.1.11", features = ["derive"] }
console = "0.15"
env_logger = "0.10.0"
flate2 = { version = "1.0.28", features = [
  "zlib-ng",
], default-features = false }
glob = "0.3.1"
humantime = "2.1"
indicatif = "0.17"
jsonpath-rust = "0.3.0"
log = "0.4.17"
num_cpus = "1.0"
num-traits = "0.2"
parse-size = "1.0"
pyo3 = { version = "0.19.0", features = ["extension-module"] }
rand = "0.8.4"
rayon = "1.7.0"
regex = "1.8.4"
serde = { version = "1.0.160", features = ["derive", "rc"] }
serde_json = "1.0.108"
simple_logger = { version = "3.0", features = [
  "stderr",
  "colors",
], default-features = false, optional = true }
structopt = { version = "0.3", optional = true }
thousands = "0.2"
threadpool = "1.8.1"
tokenizers = { version = "0.15.0", features = ["http"] }
tokio = { version = "1.27.0", features = ["full"] }
tokio-util = "0.7.7"
time = "0.3.36"
unicode-segmentation = "1.7"
openssl = { version = "0.10.66", features = ["vendored"] }
adblock = { version = "0.8.6", features = ["content-blocking"] }
jaq-core = "1.2.1"
jaq-std = "1.2.1"
jaq-parse = "1.0.2"
jaq-interpret = { version = "1.2.1", features = ["serde_json"] }
zstd = "0.13.1"

[dev-dependencies]
tempfile = "3.10.1"

# [target.'cfg(target_arch = "aarch64")'.dependencies]
# openssl = { version = "0.10.63", features = ["vendored"] }

# [target.'cfg(target_arch = "arm")'.dependencies]
# openssl = { version = "0.10.63", features = ["vendored"] }