diff --git a/.appveyor.yml b/.appveyor.yml index 91ac60e1..58bb395e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,9 +17,6 @@ build: false test_script: - cargo test --release --tests --locked - cargo test --release --doc - - cd binary-install - - cargo test - - cd .. before_deploy: - ps: | diff --git a/Cargo.lock b/Cargo.lock index e4cc0348..cb35ec88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "adler32" version = "1.0.3" @@ -87,6 +89,7 @@ dependencies = [ [[package]] name = "binary-install" version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "curl 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1273,7 +1276,7 @@ version = "0.7.0" dependencies = [ "assert_cmd 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "binary-install 0.0.2", + "binary-install 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "cargo_metadata 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "console 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "curl 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1395,6 +1398,7 @@ dependencies = [ "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum binary-install 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b5bc5f8c50dd6a80d0b303ddab79f42ddcb52fd43d68107ecf622c551fd4cd4" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" diff --git a/Cargo.toml b/Cargo.toml index 41a4b738..b72a37a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ siphasher = "0.2.3" structopt = "0.2" toml = "0.4" which = "2.0.0" -binary-install = { version = "0.0.2", path = "./binary-install" } +binary-install = "0.0.2" walkdir = "2" [dev-dependencies] diff --git a/binary-install/Cargo.toml b/binary-install/Cargo.toml deleted file mode 100644 index 10c82e56..00000000 --- a/binary-install/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "binary-install" -description = "install a binary from a path to a global cache" -authors = ["The wasm-pack team"] -repository = "https://github.com/rustwasm/wasm-pack/tree/master/binary-install" -license = "MIT/Apache-2.0" -version = "0.0.2" -documentation = "https://docs.rs/binary-install" -readme = "./README.md" - -[dependencies] -curl = "0.4.13" -dirs = "1.0.4" -failure = "0.1.2" -flate2 = "1.0.2" -hex = "0.3" -is_executable = "0.1.2" -siphasher = "0.2.3" -tar = "0.4.16" -zip = "0.5.0" - -[dev-dependencies] -tempfile = "3.0.5" diff --git a/binary-install/README.md b/binary-install/README.md deleted file mode 100644 index a79d5df5..00000000 --- a/binary-install/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# `binary-install` -> install a binary from a path to a global cache diff --git a/binary-install/src/lib.rs b/binary-install/src/lib.rs deleted file mode 100644 index 37d66148..00000000 --- a/binary-install/src/lib.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! Utilities for finding and installing binaries that we depend on. - -extern crate curl; -#[macro_use] -extern crate failure; -extern crate dirs; -extern crate flate2; -extern crate hex; -extern crate is_executable; -extern crate siphasher; -extern crate tar; -extern crate zip; - -use failure::{Error, ResultExt}; -use siphasher::sip::SipHasher13; -use std::collections::HashSet; -use std::env; -use std::ffi; -use std::fs; -use std::hash::{Hash, Hasher}; -use std::io; -use std::path::{Path, PathBuf}; - -/// Global cache for wasm-pack, currently containing binaries downloaded from -/// urls like wasm-bindgen and such. -#[derive(Debug)] -pub struct Cache { - destination: PathBuf, -} - -/// Representation of a downloaded tarball/zip -#[derive(Debug)] -pub struct Download { - root: PathBuf, -} - -impl Cache { - /// Returns the global cache directory, as inferred from env vars and such. - /// - /// This function may return an error if a cache directory cannot be - /// determined. - pub fn new(name: &str) -> Result { - let cache_name = format!(".{}", name); - let destination = dirs::cache_dir() - .map(|p| p.join(&cache_name)) - .or_else(|| { - let home = dirs::home_dir()?; - Some(home.join(&cache_name)) - }) - .ok_or_else(|| format_err!("couldn't find your home directory, is $HOME not set?"))?; - Ok(Cache::at(&destination)) - } - - /// Creates a new cache specifically at a particular directory, useful in - /// testing and such. - pub fn at(path: &Path) -> Cache { - Cache { - destination: path.to_path_buf(), - } - } - - /// Joins a path to the destination of this cache, returning the result - pub fn join(&self, path: &Path) -> PathBuf { - self.destination.join(path) - } - - /// Downloads a tarball or zip file from the specified url, extracting it - /// locally and returning the directory that the contents were extracted - /// into. - /// - /// Note that this function requries that the contents of `url` never change - /// as the contents of the url are globally cached on the system and never - /// invalidated. - /// - /// The `name` is a human-readable name used to go into the folder name of - /// the destination, and `binaries` is a list of binaries expected to be at - /// the url. If the URL's extraction doesn't contain all the binaries this - /// function will return an error. - pub fn download( - &self, - install_permitted: bool, - name: &str, - binaries: &[&str], - url: &str, - ) -> Result, Error> { - let dirname = hashed_dirname(url, name); - - let destination = self.destination.join(&dirname); - - if destination.exists() { - return Ok(Some(Download { root: destination })); - } - - if !install_permitted { - return Ok(None); - } - - let data = curl(&url).with_context(|_| format!("failed to download from {}", url))?; - - // Extract everything in a temporary directory in case we're ctrl-c'd. - // Don't want to leave around corrupted data! - let temp = self.destination.join(&format!(".{}", dirname)); - drop(fs::remove_dir_all(&temp)); - fs::create_dir_all(&temp)?; - - if url.ends_with(".tar.gz") { - self.extract_tarball(&data, &temp, binaries) - .with_context(|_| format!("failed to extract tarball from {}", url))?; - } else if url.ends_with(".zip") { - self.extract_zip(&data, &temp, binaries) - .with_context(|_| format!("failed to extract zip from {}", url))?; - } else { - // panic instead of runtime error as it's a static violation to - // download a different kind of url, all urls should be encoded into - // the binary anyway - panic!("don't know how to extract {}", url) - } - - // Now that everything is ready move this over to our destination and - // we're good to go. - fs::rename(&temp, &destination)?; - Ok(Some(Download { root: destination })) - } - - fn extract_tarball(&self, tarball: &[u8], dst: &Path, binaries: &[&str]) -> Result<(), Error> { - let mut binaries: HashSet<_> = binaries.into_iter().map(ffi::OsStr::new).collect(); - let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(tarball)); - - for entry in archive.entries()? { - let mut entry = entry?; - - let dest = match entry.path()?.file_stem() { - Some(f) if binaries.contains(f) => { - binaries.remove(f); - dst.join(entry.path()?.file_name().unwrap()) - } - _ => continue, - }; - - entry.unpack(dest)?; - } - - if !binaries.is_empty() { - bail!( - "the tarball was missing expected executables: {}", - binaries - .into_iter() - .map(|s| s.to_string_lossy()) - .collect::>() - .join(", "), - ) - } - - Ok(()) - } - - fn extract_zip(&self, zip: &[u8], dst: &Path, binaries: &[&str]) -> Result<(), Error> { - let mut binaries: HashSet<_> = binaries.into_iter().map(ffi::OsStr::new).collect(); - - let data = io::Cursor::new(zip); - let mut zip = zip::ZipArchive::new(data)?; - - for i in 0..zip.len() { - let mut entry = zip.by_index(i).unwrap(); - let entry_path = entry.sanitized_name(); - match entry_path.file_stem() { - Some(f) if binaries.contains(f) => { - binaries.remove(f); - let mut dest = bin_open_options() - .write(true) - .create_new(true) - .open(dst.join(entry_path.file_name().unwrap()))?; - io::copy(&mut entry, &mut dest)?; - } - _ => continue, - }; - } - - if !binaries.is_empty() { - bail!( - "the zip was missing expected executables: {}", - binaries - .into_iter() - .map(|s| s.to_string_lossy()) - .collect::>() - .join(", "), - ) - } - - return Ok(()); - - #[cfg(unix)] - fn bin_open_options() -> fs::OpenOptions { - use std::os::unix::fs::OpenOptionsExt; - - let mut opts = fs::OpenOptions::new(); - opts.mode(0o755); - opts - } - - #[cfg(not(unix))] - fn bin_open_options() -> fs::OpenOptions { - fs::OpenOptions::new() - } - } -} - -impl Download { - /// Manually constructs a download at the specified path - pub fn at(path: &Path) -> Download { - Download { - root: path.to_path_buf(), - } - } - - /// Returns the path to the binary `name` within this download - pub fn binary(&self, name: &str) -> Result { - use is_executable::IsExecutable; - - let ret = self - .root - .join(name) - .with_extension(env::consts::EXE_EXTENSION); - - if !ret.is_file() { - bail!("{} binary does not exist", ret.display()); - } - if !ret.is_executable() { - bail!("{} is not executable", ret.display()); - } - - Ok(ret) - } -} - -fn curl(url: &str) -> Result, Error> { - let mut data = Vec::new(); - - let mut easy = curl::easy::Easy::new(); - easy.follow_location(true)?; - easy.url(url)?; - easy.get(true)?; - { - let mut transfer = easy.transfer(); - transfer.write_function(|part| { - data.extend_from_slice(part); - Ok(part.len()) - })?; - transfer.perform()?; - } - - let status_code = easy.response_code()?; - if 200 <= status_code && status_code < 300 { - Ok(data) - } else { - bail!( - "received a bad HTTP status code ({}) when requesting {}", - status_code, - url - ) - } -} - -fn hashed_dirname(url: &str, name: &str) -> String { - let mut hasher = SipHasher13::new(); - url.hash(&mut hasher); - let result = hasher.finish(); - let hex = hex::encode(&[ - (result >> 0) as u8, - (result >> 8) as u8, - (result >> 16) as u8, - (result >> 24) as u8, - (result >> 32) as u8, - (result >> 40) as u8, - (result >> 48) as u8, - (result >> 56) as u8, - ]); - format!("{}-{}", name, hex) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_returns_same_hash_for_same_name_and_url() { - let name = "wasm-pack"; - let url = "http://localhost:7878/wasm-pack-v0.6.0.tar.gz"; - - let first = hashed_dirname(url, name); - let second = hashed_dirname(url, name); - - assert!(!first.is_empty()); - assert!(!second.is_empty()); - assert_eq!(first, second); - } - - #[test] - fn it_returns_different_hashes_for_different_urls() { - let name = "wasm-pack"; - let url = "http://localhost:7878/wasm-pack-v0.5.1.tar.gz"; - let second_url = "http://localhost:7878/wasm-pack-v0.6.0.tar.gz"; - - let first = hashed_dirname(url, name); - let second = hashed_dirname(second_url, name); - - assert_ne!(first, second); - } - - #[test] - fn it_returns_cache_dir() { - let name = "wasm-pack"; - let cache = Cache::new(name); - - let expected = dirs::cache_dir() - .unwrap() - .join(PathBuf::from(".".to_owned() + name)); - - assert!(cache.is_ok()); - assert_eq!(cache.unwrap().destination, expected); - } - - #[test] - fn it_returns_destination_if_binary_already_exists() { - use std::fs; - - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - let url = &format!("{}/{}.tar.gz", "http://localhost:7878", binary_name); - - let dirname = hashed_dirname(&url, &binary_name); - let full_path = dir.path().join(dirname); - - // Create temporary directory and binary to simulate that - // a cached binary already exists. - fs::create_dir_all(full_path).unwrap(); - - let dl = cache.download(true, binary_name, &binaries, url); - - assert!(dl.is_ok()); - assert!(dl.unwrap().is_some()) - } -} diff --git a/binary-install/tests/all/cache.rs b/binary-install/tests/all/cache.rs deleted file mode 100644 index 6fa07ba9..00000000 --- a/binary-install/tests/all/cache.rs +++ /dev/null @@ -1,142 +0,0 @@ -use binary_install::Cache; -use std::path::Path; -use utils; - -#[test] -fn it_returns_none_if_install_is_not_permitted() { - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - let dl = cache.download( - false, - binary_name, - &binaries, - &format!("{}/{}.tar.gz", "", binary_name), - ); - - assert!(dl.is_ok()); - assert!(dl.unwrap().is_none()) -} - -#[test] -fn it_downloads_tarball() { - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - // Create a temporary tarball. - let tarball = utils::create_tarball(binary_name).ok(); - - // Spin up a local TcpListener. - let server_port = utils::start_server(tarball, None).recv().unwrap(); - - let url = format!("http://{}:{}", utils::TEST_SERVER_HOST, server_port); - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - let dl = cache.download( - true, - binary_name, - &binaries, - &format!("{}/{}.tar.gz", &url, binary_name), - ); - - assert!(dl.is_ok()); - assert!(dl.unwrap().is_some()) -} - -#[test] -fn it_returns_error_when_it_failed_to_download() { - let server_port = 7881; - let url = format!("http://{}:{}", utils::TEST_SERVER_HOST, server_port); - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - let full_url = &format!("{}/{}.tar.gz", &url, binary_name); - - let dl = cache.download(true, binary_name, &binaries, full_url); - - assert!(dl.is_err()); - assert_eq!( - &format!("failed to download from {}", full_url), - &format!("{}", dl.unwrap_err()) - ); -} - -#[test] -fn it_returns_error_when_it_failed_to_extract_tarball() { - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - // Spin up a local TcpListener. - let server_port = utils::start_server(None, None).recv().unwrap(); - - let url = format!("http://{}:{}", utils::TEST_SERVER_HOST, server_port); - let full_url = &format!("{}/{}.tar.gz", &url, binary_name); - - let dl = cache.download(true, binary_name, &binaries, full_url); - - assert!(dl.is_err()); - assert_eq!( - &format!("failed to extract tarball from {}", full_url), - &format!("{}", dl.unwrap_err()) - ); -} - -#[test] -fn it_returns_error_when_it_failed_to_extract_zip() { - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - // Spin up a local TcpListener. - let server_port = utils::start_server(None, None).recv().unwrap(); - - let url = format!("http://{}:{}", utils::TEST_SERVER_HOST, server_port); - let full_url = &format!("{}/{}.zip", &url, binary_name); - - let dl = cache.download(true, binary_name, &binaries, full_url); - - assert!(dl.is_err()); - assert_eq!( - &format!("failed to extract zip from {}", full_url), - &format!("{}", dl.unwrap_err()) - ); -} - -#[test] -#[should_panic(expected = "don't know how to extract http://localhost:7884/wasm-pack.bin")] -fn it_panics_if_not_tarball_or_zip() { - let server_port = 7884; - let binary_name = "wasm-pack"; - let binaries = vec![binary_name]; - - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - // Spin up a local TcpListener. - utils::start_server(None, Some(server_port)).recv().unwrap(); - - let url = format!("http://{}:{}", utils::TEST_SERVER_HOST, server_port); - let full_url = &format!("{}/{}.bin", &url, binary_name); - - let _ = cache.download(true, binary_name, &binaries, full_url); -} - -#[test] -fn it_joins_path_with_destination() { - let dir = tempfile::TempDir::new().unwrap(); - let cache = Cache::at(dir.path()); - - assert_eq!(dir.path().join("hello"), cache.join(Path::new("hello"))); -} diff --git a/binary-install/tests/all/download.rs b/binary-install/tests/all/download.rs deleted file mode 100644 index d65bfb80..00000000 --- a/binary-install/tests/all/download.rs +++ /dev/null @@ -1,125 +0,0 @@ -use binary_install::Download; -use std::fs::OpenOptions; - -#[test] -#[cfg(unix)] -fn it_returns_binary_name_for_unix() { - use std::os::unix::fs::OpenOptionsExt; - - let binary_name = "wasm-pack"; - - let dir = tempfile::TempDir::new().unwrap(); - let download = Download::at(dir.path()); - - let full_path = dir.path().join(binary_name); - - let mut options = OpenOptions::new(); - options.create(true); - options.write(true); - - // Make the "binary" an executable. - options.mode(0o755); - - options.open(&full_path).unwrap(); - - let binary = download.binary(binary_name); - - assert!(binary.is_ok()); - assert_eq!(full_path, binary.unwrap()); -} - -#[test] -#[cfg(not(windows))] -fn it_bails_if_not_file_for_unix() { - let binary_name = "wasm-pack"; - - let dir = tempfile::TempDir::new().unwrap(); - let download = Download::at(dir.path()); - - let full_path = dir.path().join(binary_name); - - let mut options = OpenOptions::new(); - options.create(true); - options.write(true); - - let binary = download.binary(binary_name); - - assert!(binary.is_err()); - assert_eq!( - format!("{} binary does not exist", full_path.to_str().unwrap()), - binary.unwrap_err().to_string() - ); -} - -#[test] -#[cfg(windows)] -fn it_bails_if_not_file_for_windows() { - let binary_name = "wasm-pack.exe"; - - let dir = tempfile::TempDir::new().unwrap(); - let download = Download::at(dir.path()); - - let full_path = dir.path().join(binary_name); - - let mut options = OpenOptions::new(); - options.create(true); - options.write(true); - - let binary = download.binary(binary_name); - - assert!(binary.is_err()); - assert_eq!( - format!("{} binary does not exist", full_path.to_str().unwrap()), - binary.unwrap_err().to_string() - ); -} - -#[test] -#[cfg(not(windows))] -fn it_bails_if_not_executable_for_unix() { - let binary_name = "wasm-pack"; - - let dir = tempfile::TempDir::new().unwrap(); - let download = Download::at(dir.path()); - - let full_path = dir.path().join(binary_name); - - let mut options = OpenOptions::new(); - options.create(true); - options.write(true); - - options.open(&full_path).unwrap(); - - let binary = download.binary(binary_name); - - assert!(binary.is_err()); - assert_eq!( - format!("{} is not executable", full_path.to_str().unwrap()), - binary.unwrap_err().to_string() - ); -} - -#[test] -#[cfg(windows)] -fn it_bails_if_not_executable_for_windows() { - let binary_name = "wasm-pack.exe"; - - let dir = tempfile::TempDir::new().unwrap(); - let download = Download::at(dir.path()); - - let full_path = dir.path().join(binary_name); - - let mut options = OpenOptions::new(); - options.create(true); - options.write(true); - - options.open(&full_path).unwrap(); - - let binary = download.binary(binary_name); - - assert!(binary.is_err()); - assert_eq!( - format!("{} is not executable", full_path.to_str().unwrap()), - binary.unwrap_err().to_string() - ); -} diff --git a/binary-install/tests/all/main.rs b/binary-install/tests/all/main.rs deleted file mode 100644 index de1ad721..00000000 --- a/binary-install/tests/all/main.rs +++ /dev/null @@ -1,7 +0,0 @@ -extern crate binary_install; -extern crate flate2; -extern crate tar; - -mod cache; -mod download; -mod utils; diff --git a/binary-install/tests/all/utils/mod.rs b/binary-install/tests/all/utils/mod.rs deleted file mode 100644 index 65b13260..00000000 --- a/binary-install/tests/all/utils/mod.rs +++ /dev/null @@ -1,79 +0,0 @@ -use flate2::write::GzEncoder; -use flate2::Compression; -use std::fs::{File, OpenOptions}; -use std::io::{self, Read, Write}; -use std::net::TcpListener; -use std::sync::mpsc::{channel, Receiver}; -use std::thread; - -pub const TEST_SERVER_HOST: &'static str = "localhost"; - -pub fn start_server(tarball: Option>, server_port: Option) -> Receiver { - let (sender, receiver) = channel(); - - thread::spawn(move || { - TcpListener::bind(format!( - "{}:{}", - TEST_SERVER_HOST, - server_port.unwrap_or_else(|| 0) - )) - .map(|listener| { - sender.send(listener.local_addr().unwrap().port()).unwrap(); - - for stream in listener.incoming() { - let mut stream = stream.unwrap(); - - let mut buffer = [0; 512]; - - stream.read(&mut buffer).unwrap(); - - let response = "HTTP/1.1 200 OK\r\n\r\n"; - - stream.write(response.as_bytes()).unwrap(); - - match tarball.to_owned() { - Some(tar) => { - stream.write(tar.as_ref()).unwrap(); - } - None => {} - } - - stream.flush().unwrap(); - } - }) - .unwrap(); - }); - - receiver -} - -pub fn create_tarball(binary_name: &str) -> Result, io::Error> { - let temp_dir = tempfile::TempDir::new().unwrap(); - let full_path = temp_dir.path().join(binary_name.to_owned() + ".tar.gz"); - - let tar = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(&full_path)?; - - let mut file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(temp_dir.path().join(binary_name))?; - - let mut encoder = GzEncoder::new(tar, Compression::default()); - { - let mut archive = tar::Builder::new(&mut encoder); - archive.append_file(binary_name, &mut file)?; - } - - let mut contents = vec![]; - - encoder.finish()?; - - File::open(temp_dir.path().join(&full_path))?.read_to_end(&mut contents)?; - - Ok(contents) -}