Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to run compiled benchmarks #52

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Allow to run compiled benchmarks #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 138 additions & 104 deletions src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ pub struct SelfConfig {

/// Overall struct that represents all of the configuration data for this run.
#[derive(Debug)]
pub struct FullConfig {
pub struct ToCompileConfig {
/// The config settings for cargo-criterion
pub self_config: SelfConfig,
/// The arguments we pass through to cargo bench
Expand All @@ -194,6 +194,18 @@ pub struct FullConfig {
pub additional_args: Vec<OsString>,
}

pub struct PrecompiledConfig {
/// The config settings for cargo-criterion
pub self_config: SelfConfig,
/// The Paths for precompiled binaries
pub pre_compiled_bins_path: Vec<PathBuf>
}

pub enum Config {
ToCompile(ToCompileConfig),
Precompiled(PrecompiledConfig)
}

/// Call `cargo criterion` and parse the output to get the path to the target directory.
fn get_target_directory_from_metadata() -> Result<PathBuf> {
let out = Command::new("cargo")
Expand All @@ -213,7 +225,7 @@ fn get_target_directory_from_metadata() -> Result<PathBuf> {
/// Parse the command-line arguments, load the criterion.toml config file, and generate a
/// configuration object used for the rest of the run.
#[cfg_attr(feature = "cargo-clippy", allow(clippy::or_fun_call))]
pub fn configure() -> Result<FullConfig, anyhow::Error> {
pub fn configure() -> Result<Config, anyhow::Error> {
use clap::{App, AppSettings, Arg};

let matches = App::new("cargo-criterion")
Expand Down Expand Up @@ -387,7 +399,7 @@ pub fn configure() -> Result<FullConfig, anyhow::Error> {
not be optimized. This may be useful to reduce compile time when benchmarking code written in a
different language (eg. external C modules).

Note however that it will tend to increase the measurement overhead, as the measurement loops
Note however that it will tend to increase the measurement overhead, as the measurement loops
in the benchmark will not be optimized either. This may result in less-accurate measurements.
")
)
Expand Down Expand Up @@ -491,6 +503,14 @@ See the documentation for details on the data printed by each format.
Arg::with_name("BENCHNAME")
.help("If specified, only run benches with names that match this regex"),
)
.arg(
Arg::with_name("precompiled_bins")
.long("--precompiled-bins")
.takes_value(true)
.value_name("PATH")
.multiple(true)
.help("Run precompiled benchmark binaries within Criterion"),
)
.arg(
Arg::with_name("args")
.takes_value(true)
Expand All @@ -500,8 +520,8 @@ See the documentation for details on the data printed by each format.
.after_help(
"\
The benchmark filtering argument BENCHNAME and all the arguments following the
two dashes (`--`) are passed to the benchmark binaries and thus Criterion.rs.
If you're passing arguments to both Cargo and the binary, the ones after `--` go
two dashes (`--`) are passed to the benchmark binaries and thus Criterion.rs.
If you're passing arguments to both Cargo and the binary, the ones after `--` go
to the binary, the ones before go to Cargo. For details about Criterion.rs' arguments see
the output of `cargo criterion -- --help`.

Expand Down Expand Up @@ -541,99 +561,107 @@ Compilation can be customized with the `bench` profile in the manifest.
// Many arguments have to be passed along to Cargo, so construct the list of cargo arguments
// here.
let mut cargo_args: Vec<OsString> = vec![];
if matches.is_present("lib") {
cargo_args.push("--lib".into());
}
if let Some(values) = matches.values_of_os("bin") {
cargo_args.push("--bin".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("bins") {
cargo_args.push("--bins".into());
}
if let Some(values) = matches.values_of_os("example") {
cargo_args.push("--example".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("examples") {
cargo_args.push("--examples".into());
}
if let Some(values) = matches.values_of_os("test") {
cargo_args.push("--test".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("tests") {
cargo_args.push("--tests".into());
}
if let Some(values) = matches.values_of_os("bench") {
cargo_args.push("--bench".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("benches") {
cargo_args.push("--benches".into());
}
if matches.is_present("all-targets") {
cargo_args.push("--all-targets".into());
}
if let Some(values) = matches.values_of_os("package") {
cargo_args.push("--package".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("all") {
cargo_args.push("--all".into());
}
if matches.is_present("workspace") {
cargo_args.push("--workspace".into());
}
if let Some(values) = matches.values_of_os("exclude") {
cargo_args.push("--exclude".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if let Some(value) = matches.value_of_os("jobs") {
cargo_args.push("--jobs".into());
cargo_args.push(value.to_owned());
}
if let Some(values) = matches.values_of_os("features") {
cargo_args.push("--features".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("all-features") {
cargo_args.push("--all-features".into());
}
if matches.is_present("no-default-features") {
cargo_args.push("--no-default-features".into());
}
if let Some(value) = matches.value_of_os("target") {
cargo_args.push("--target".into());
cargo_args.push(value.to_owned());
}
if let Some(value) = matches.value_of_os("target-dir") {
cargo_args.push("--target-dir".into());
cargo_args.push(value.to_owned());
}
if let Some(value) = matches.value_of_os("manifest-path") {
cargo_args.push("--manifest-path".into());
cargo_args.push(value.to_owned());
}
for _ in 0..matches.occurrences_of("verbose") {
cargo_args.push("--verbose".into());
}
if let Some(value) = matches.value_of_os("color") {
cargo_args.push("--color".into());
cargo_args.push(value.to_owned());
}
if matches.is_present("frozen") {
cargo_args.push("--frozen".into());
}
if matches.is_present("locked") {
cargo_args.push("--locked".into());
}
if matches.is_present("offline") {
cargo_args.push("--offline".into());
}
if let Some(values) = matches.values_of_os("unstable_flags") {
cargo_args.push("-Z".into());
cargo_args.extend(values.map(ToOwned::to_owned));
let mut pre_compiled_bins_path: Vec<PathBuf> = vec![];

// Avoid all cargo args parsing if precompiled binaries flag is given, precompiled binaries no
// not need to execute Cargo.
if let Some(values) = matches.values_of_os("precompiled_bins") {
pre_compiled_bins_path.extend(values.map(|x| x.into()));
} else {
if matches.is_present("lib") {
cargo_args.push("--lib".into());
}
if let Some(values) = matches.values_of_os("bin") {
cargo_args.push("--bin".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("bins") {
cargo_args.push("--bins".into());
}
if let Some(values) = matches.values_of_os("example") {
cargo_args.push("--example".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("examples") {
cargo_args.push("--examples".into());
}
if let Some(values) = matches.values_of_os("test") {
cargo_args.push("--test".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("tests") {
cargo_args.push("--tests".into());
}
if let Some(values) = matches.values_of_os("bench") {
cargo_args.push("--bench".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("benches") {
cargo_args.push("--benches".into());
}
if matches.is_present("all-targets") {
cargo_args.push("--all-targets".into());
}
if let Some(values) = matches.values_of_os("package") {
cargo_args.push("--package".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("all") {
cargo_args.push("--all".into());
}
if matches.is_present("workspace") {
cargo_args.push("--workspace".into());
}
if let Some(values) = matches.values_of_os("exclude") {
cargo_args.push("--exclude".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if let Some(value) = matches.value_of_os("jobs") {
cargo_args.push("--jobs".into());
cargo_args.push(value.to_owned());
}
if let Some(values) = matches.values_of_os("features") {
cargo_args.push("--features".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
if matches.is_present("all-features") {
cargo_args.push("--all-features".into());
}
if matches.is_present("no-default-features") {
cargo_args.push("--no-default-features".into());
}
if let Some(value) = matches.value_of_os("target") {
cargo_args.push("--target".into());
cargo_args.push(value.to_owned());
}
if let Some(value) = matches.value_of_os("target-dir") {
cargo_args.push("--target-dir".into());
cargo_args.push(value.to_owned());
}
if let Some(value) = matches.value_of_os("manifest-path") {
cargo_args.push("--manifest-path".into());
cargo_args.push(value.to_owned());
}
for _ in 0..matches.occurrences_of("verbose") {
cargo_args.push("--verbose".into());
}
if let Some(value) = matches.value_of_os("color") {
cargo_args.push("--color".into());
cargo_args.push(value.to_owned());
}
if matches.is_present("frozen") {
cargo_args.push("--frozen".into());
}
if matches.is_present("locked") {
cargo_args.push("--locked".into());
}
if matches.is_present("offline") {
cargo_args.push("--offline".into());
}
if let Some(values) = matches.values_of_os("unstable_flags") {
cargo_args.push("-Z".into());
cargo_args.extend(values.map(ToOwned::to_owned));
}
}

// Set criterion home to (in descending order of preference):
Expand Down Expand Up @@ -687,12 +715,18 @@ Compilation can be customized with the `bench` profile in the manifest.
additional_args.extend(args.map(ToOwned::to_owned));
}

let configuration = FullConfig {
self_config,
cargo_args,
additional_args,
};
Ok(configuration)
if matches.is_present("precompiled_bins") {
return Ok(Config::Precompiled( PrecompiledConfig {
self_config,
pre_compiled_bins_path
}));
} else {
return Ok(Config::ToCompile( ToCompileConfig {
self_config,
cargo_args,
additional_args
}));
}
}

/// Load & parse the criterion.toml file (if present).
Expand Down
59 changes: 47 additions & 12 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ use crate::config::{OutputFormat, PlottingBackend, SelfConfig, TextColor};
use crate::connection::{AxisScale, PlotConfiguration};
use crate::plot::Plotter;
use crate::report::{Report, ReportContext};
use anyhow::Error;
use anyhow::{Error, Context};
use lazy_static::lazy_static;
use std::path::PathBuf;

lazy_static! {
static ref DEBUG_ENABLED: bool = std::env::var_os("CRITERION_DEBUG").is_some();
Expand Down Expand Up @@ -78,14 +79,37 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
configure_log();

// First, load the config file and parse the command-line args.
let configuration = config::configure()?;
let self_config = &configuration.self_config;
let self_config: config::SelfConfig;
let mut targets = vec![];
let mut library_paths = vec![];
let mut additional_args = vec![];

// Launch cargo to compile the crate and produce a list of the benchmark targets to run.
let compile::CompiledBenchmarks {
targets,
library_paths,
} = compile::compile(self_config.debug_build, &configuration.cargo_args)?;
match config::configure()? {
config::Config::Precompiled(config) => {
self_config = config.self_config;

let vec_size = config.pre_compiled_bins_path.len();
let mut counter: usize = 0;

while vec_size > counter {
targets.push(bench_target::BenchTarget {
name: filename_from_pathbuf(&config.pre_compiled_bins_path[counter])?,
executable: config.pre_compiled_bins_path[counter].to_path_buf().canonicalize()?,
});

counter += 1;
}
},
config::Config::ToCompile(config) => {
self_config = config.self_config;
additional_args = config.additional_args;

// Launch cargo to compile the crate and produce a list of the benchmark targets to run
let c = compile::compile(self_config.debug_build, &config.cargo_args)?;
targets = c.targets;
library_paths = c.library_paths;
}
}

// Load the saved measurements from the last run.
let mut run_model = model::Model::load(
Expand All @@ -96,10 +120,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
);

// Set up the reports. These receive notifications as the benchmarks proceed and generate output for the user.
let cli_report = configure_cli_output(self_config);
let cli_report = configure_cli_output(&self_config);
let bencher_report = crate::report::BencherReport;
let html_report = get_plotter(self_config)?.map(|plotter| crate::html::Html::new(plotter));
let machine_report = message_formats::create_machine_report(self_config);
let html_report = get_plotter(&self_config)?.map(|plotter| crate::html::Html::new(plotter));
let machine_report = message_formats::create_machine_report(&self_config);

let mut reports: Vec<&dyn crate::report::Report> = Vec::new();
match self_config.output_format {
Expand All @@ -122,7 +146,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Executing {} - {:?}", bench.name, bench.executable);
let err = bench.execute(
&self_config.criterion_home,
&configuration.additional_args,
&additional_args,
&library_paths,
&reports,
&mut run_model,
Expand Down Expand Up @@ -253,3 +277,14 @@ impl DurationExt for std::time::Duration {
self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos())
}
}

/// Convert a PathBuf reference to a String
#[inline(always)]
fn filename_from_pathbuf(path: &PathBuf) -> Result<String, anyhow::Error> {
// No good looking way to covert a PathBuf reference filename to a String
return Ok(path.file_name()
.context("Incorrect binary path")?
.to_str()
.context("Invalid UTF8 in binary name")?
.to_string());
}