From d2934c4fbb2464d2a22e40fb80bc7e96b33e8542 Mon Sep 17 00:00:00 2001 From: Colin Jones Date: Sat, 3 Oct 2020 23:35:49 +0100 Subject: [PATCH] CAS-461 Add gRPC method to enumerate host block devices. --- Cargo.lock | 2 + mayastor/Cargo.toml | 2 + mayastor/src/bin/cli/cli.rs | 3 + mayastor/src/bin/cli/device_cli.rs | 185 ++++++++++++++++ mayastor/src/grpc/mayastor_grpc.rs | 14 ++ mayastor/src/host/blk_device.rs | 327 +++++++++++++++++++++++++++++ mayastor/src/host/mod.rs | 1 + mayastor/src/lib.rs | 1 + nix/pkgs/mayastor/default.nix | 2 +- rpc/proto/mayastor.proto | 39 ++++ 10 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 mayastor/src/bin/cli/device_cli.rs create mode 100644 mayastor/src/host/blk_device.rs create mode 100644 mayastor/src/host/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1fcef3408..a44200530 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1545,6 +1545,7 @@ dependencies = [ "nix 0.16.1", "once_cell", "pin-utils", + "proc-mounts", "prost", "prost-derive", "prost-types", @@ -1567,6 +1568,7 @@ dependencies = [ "tracing-futures", "tracing-log", "tracing-subscriber", + "udev", "url", "uuid", ] diff --git a/mayastor/Cargo.toml b/mayastor/Cargo.toml index 5598e2aaf..8397c0a4d 100644 --- a/mayastor/Cargo.toml +++ b/mayastor/Cargo.toml @@ -56,6 +56,7 @@ log = "0.4" nix = "0.16" once_cell = "1.3.1" pin-utils = "0.1" +proc-mounts = "0.2" prost = "0.6" prost-derive = "0.6" prost-types = "0.6" @@ -72,6 +73,7 @@ tracing = "0.1" tracing-futures = "0.2.4" tracing-log = "0.1.1" tracing-subscriber = "0.2.0" +udev = "0.4" url = "2.1" smol = "1.0.0" dns-lookup = "1.0.4" diff --git a/mayastor/src/bin/cli/cli.rs b/mayastor/src/bin/cli/cli.rs index 93478ffc4..6ea82f30a 100644 --- a/mayastor/src/bin/cli/cli.rs +++ b/mayastor/src/bin/cli/cli.rs @@ -14,6 +14,7 @@ use crate::context::Context; mod bdev_cli; mod context; +mod device_cli; mod nexus_child_cli; mod nexus_cli; mod pool_cli; @@ -77,6 +78,7 @@ async fn main() -> Result<(), Status> { .subcommand(nexus_cli::subcommands()) .subcommand(replica_cli::subcommands()) .subcommand(bdev_cli::subcommands()) + .subcommand(device_cli::subcommands()) .subcommand(rebuild_cli::subcommands()) .subcommand(snapshot_cli::subcommands()) .get_matches(); @@ -85,6 +87,7 @@ async fn main() -> Result<(), Status> { match matches.subcommand() { ("bdev", Some(args)) => bdev_cli::handler(ctx, args).await?, + ("device", Some(args)) => device_cli::handler(ctx, args).await?, ("nexus", Some(args)) => nexus_cli::handler(ctx, args).await?, ("pool", Some(args)) => pool_cli::handler(ctx, args).await?, ("replica", Some(args)) => replica_cli::handler(ctx, args).await?, diff --git a/mayastor/src/bin/cli/device_cli.rs b/mayastor/src/bin/cli/device_cli.rs new file mode 100644 index 000000000..15b5943e0 --- /dev/null +++ b/mayastor/src/bin/cli/device_cli.rs @@ -0,0 +1,185 @@ +//! +//! methods to obtain information about block devices on the current host + +use super::context::Context; +use ::rpc::mayastor as rpc; +use clap::{App, AppSettings, Arg, ArgMatches, SubCommand}; +use colored_json::ToColoredJson; +use tonic::Status; + +pub fn subcommands<'a, 'b>() -> App<'a, 'b> { + let list = SubCommand::with_name("list") + .about("List available (ie. unused) block devices") + .arg( + Arg::with_name("all") + .short("a") + .long("all") + .takes_value(false) + .help("List all block devices (ie. also include devices currently in use)"), + ) + .arg( + Arg::with_name("raw") + .long("raw") + .takes_value(false) + .help("Display output as raw JSON"), + ); + + SubCommand::with_name("device") + .settings(&[ + AppSettings::SubcommandRequiredElseHelp, + AppSettings::ColoredHelp, + AppSettings::ColorAlways, + ]) + .about("Host devices") + .subcommand(list) +} + +pub async fn handler( + ctx: Context, + matches: &ArgMatches<'_>, +) -> Result<(), Status> { + match matches.subcommand() { + ("list", Some(args)) => list_block_devices(ctx, args).await, + (cmd, _) => { + Err(Status::not_found(format!("command {} does not exist", cmd))) + } + } +} + +fn get_partition_type(device: &rpc::BlockDevice) -> String { + if let Some(partition) = &device.partition { + format!("{}:{}", partition.scheme, partition.typeid) + } else { + String::from("") + } +} + +async fn list_block_devices( + mut ctx: Context, + matches: &ArgMatches<'_>, +) -> Result<(), Status> { + let all = matches.is_present("all"); + + ctx.v2(&format!( + "Requesting list of {} block devices", + if all { "all" } else { "available" } + )); + + let reply = ctx + .client + .list_block_devices(rpc::ListBlockDevicesRequest { + all, + }) + .await?; + + if matches.is_present("raw") { + println!( + "{}", + serde_json::to_string_pretty(&reply.into_inner()) + .unwrap() + .to_colored_json_auto() + .unwrap() + ); + return Ok(()); + } + + let devices: &Vec = &reply.get_ref().devices; + + if devices.is_empty() { + ctx.v1("No devices found"); + return Ok(()); + } + + if all { + let table = devices + .iter() + .map(|device| { + let fstype: String; + let uuid: String; + let mountpoint: String; + + if let Some(filesystem) = &device.filesystem { + fstype = filesystem.fstype.clone(); + uuid = filesystem.uuid.clone(); + mountpoint = filesystem.mountpoint.clone(); + } else { + fstype = String::from(""); + uuid = String::from(""); + mountpoint = String::from(""); + } + + vec![ + device.devname.clone(), + device.devtype.clone(), + device.devmajor.to_string(), + device.devminor.to_string(), + device.size.to_string(), + String::from(if device.available { "yes" } else { "no" }), + device.model.clone(), + get_partition_type(&device), + fstype, + uuid, + mountpoint, + device.devpath.clone(), + device + .devlinks + .iter() + .map(|s| format!("\"{}\"", s)) + .collect::>() + .join(" "), + ] + }) + .collect(); + + ctx.print_list( + vec![ + "DEVNAME", + "DEVTYPE", + ">MAJOR", + "MINOR", + ">SIZE", + "AVAILABLE", + "MODEL", + "PARTTYPE", + "FSTYPE", + "FSUUID", + "MOUNTPOINT", + "DEVPATH", + "DEVLINKS", + ], + table, + ); + } else { + let table = devices + .iter() + .map(|device| { + vec![ + device.devname.clone(), + device.devtype.clone(), + device.devmajor.to_string(), + device.devminor.to_string(), + device.size.to_string(), + device.model.clone(), + get_partition_type(&device), + device.devpath.clone(), + device + .devlinks + .iter() + .map(|s| format!("\"{}\"", s)) + .collect::>() + .join(" "), + ] + }) + .collect(); + + ctx.print_list( + vec![ + "DEVNAME", "DEVTYPE", ">MAJOR", "MINOR", ">SIZE", "MODEL", + "PARTTYPE", "DEVPATH", "DEVLINKS", + ], + table, + ); + } + + Ok(()) +} diff --git a/mayastor/src/grpc/mayastor_grpc.rs b/mayastor/src/grpc/mayastor_grpc.rs index 647dc6fad..74efd70cd 100644 --- a/mayastor/src/grpc/mayastor_grpc.rs +++ b/mayastor/src/grpc/mayastor_grpc.rs @@ -30,6 +30,7 @@ use crate::{ sync_config, GrpcResult, }, + host::blk_device, }; #[derive(Debug)] @@ -414,4 +415,17 @@ impl mayastor_server::Mayastor for MayastorSvc { }) .await } + + #[instrument(level = "debug", err)] + async fn list_block_devices( + &self, + request: Request, + ) -> GrpcResult { + let args = request.into_inner(); + let reply = ListBlockDevicesReply { + devices: blk_device::list_block_devices(args.all).await?, + }; + trace!("{:?}", reply); + Ok(Response::new(reply)) + } } diff --git a/mayastor/src/host/blk_device.rs b/mayastor/src/host/blk_device.rs new file mode 100644 index 000000000..af22fcc09 --- /dev/null +++ b/mayastor/src/host/blk_device.rs @@ -0,0 +1,327 @@ +//! +//! This module implements the list_block_devices() gRPC method +//! for listing available disk devices on the current host. +//! +//! The relevant information is obtained via udev. +//! The method works by iterating through udev records and selecting block +//! (ie. SUBSYSTEM=block) devices that represent either disks or disk +//! partitions. For each such device, it is then determined as to whether the +//! device is available for use. +//! +//! A device is currently deemed to be "available" if it satisfies the following +//! criteria: +//! - the device has a non-zero size +//! - the device is of an acceptable type as determined by well known device +//! numbers (eg. SCSI disks) +//! - the device represents either a disk with no partitions or a disk +//! partition of an acceptable type (Linux filesystem partitions only at +//! present) +//! - the device currently contains no filesystem or volume id (although this +//! logically implies that the device is not currently mounted, for the sake +//! of consistency, the mount table is also checked to ENSURE that the device +//! is not mounted) + +use std::{ + collections::HashMap, + ffi::{OsStr, OsString}, + io::Error, +}; + +use proc_mounts::{MountInfo, MountIter}; +use rpc::mayastor::{ + block_device::{Filesystem, Partition}, + BlockDevice, +}; +use udev::{Device, Enumerator}; + +// Struct representing a property value in a udev::Device struct (and possibly +// elsewhere). It is used to provide conversions via various "From" trait +// implementations below. +struct Property<'a>(Option<&'a OsStr>); + +impl From> for String { + fn from(property: Property) -> Self { + String::from(property.0.map(|s| s.to_str()).flatten().unwrap_or("")) + } +} + +impl From> for Option { + fn from(property: Property) -> Self { + property.0.map(|s| s.to_str()).flatten().map(String::from) + } +} + +impl From> for Option { + fn from(property: Property) -> Self { + Option::::from(property) + .map(|s| s.parse().ok()) + .flatten() + } +} + +impl From> for u32 { + fn from(property: Property) -> Self { + Option::::from(property).unwrap_or(0) + } +} + +impl From> for Option { + fn from(property: Property) -> Self { + Option::::from(property) + .map(|s| s.parse().ok()) + .flatten() + } +} + +impl From> for u64 { + fn from(property: Property) -> Self { + Option::::from(property).unwrap_or(0) + } +} + +// Determine the type of devices which may be potentially presented +// as "available" for use. +fn usable_device(devmajor: &u32) -> bool { + const DEVICE_TYPES: [u32; 4] = [ + 7, // Loopback devices + 8, // SCSI disk devices + 43, // Network block devices + 259, // Block Extended Major + ]; + + if DEVICE_TYPES.iter().any(|m| m == devmajor) { + return true; + } + + // TODO: add extra logic here as needed for devices with dynamically + // allocated major numbers + + false +} + +// Determine the type of partitions which may be potentially presented +// as "available" for use +fn usable_partition(partition: &Option) -> bool { + const GPT_PARTITION_TYPES: [&str; 1] = [ + "0fc63daf-8483-4772-8e79-3d69d8477de4", // Linux + ]; + + const MBR_PARTITION_TYPES: [&str; 1] = [ + "0x83", // Linux + ]; + + if let Some(part) = partition { + if part.scheme == "gpt" { + return GPT_PARTITION_TYPES.iter().any(|&s| s == part.typeid); + } + if part.scheme == "dos" { + return MBR_PARTITION_TYPES.iter().any(|&s| s == part.typeid); + } + return false; + } + + true +} + +// Determine if device is provided internally via mayastor. +// At present this simply involves examining the value of +// the udev "ID_MODEL" property. +fn mayastor_device(device: &Device) -> bool { + match device + .property_value("ID_MODEL") + .map(|s| s.to_str()) + .flatten() + { + Some("Mayastor NVMe controller") => true, // NVMF + Some("Nexus_CAS_Driver") => true, // iSCSI + _ => false, + } +} + +// Create a new Partition object from udev::Device properties +fn new_partition(parent: Option<&str>, device: &Device) -> Option { + if let Some(devtype) = device.property_value("DEVTYPE") { + if devtype.to_str() == Some("partition") { + return Some(Partition { + parent: String::from(parent.unwrap_or("")), + number: Property(device.property_value("PARTN")).into(), + name: Property(device.property_value("PARTNAME")).into(), + scheme: Property(device.property_value("ID_PART_ENTRY_SCHEME")) + .into(), + typeid: Property(device.property_value("ID_PART_ENTRY_TYPE")) + .into(), + uuid: Property(device.property_value("ID_PART_ENTRY_UUID")) + .into(), + }); + } + } + None +} + +// Create a new Filesystem object from udev::Device properties +// and the list of current filesystem mounts. +// Note that the result can be None if there is no filesystem +// associated with this Device. +fn new_filesystem( + device: &Device, + mountinfo: Option<&MountInfo>, +) -> Option { + let mut fstype: Option = + Property(device.property_value("ID_FS_TYPE")).into(); + + if fstype.is_none() { + fstype = mountinfo.map(|m| m.fstype.clone()); + } + + let label: Option = + Property(device.property_value("ID_FS_LABEL")).into(); + + let uuid: Option = + Property(device.property_value("ID_FS_UUID")).into(); + + // Do no return an actual object if none of the fields therein have actual + // values. + if fstype.is_none() + && label.is_none() + && uuid.is_none() + && mountinfo.is_none() + { + return None; + } + + Some(Filesystem { + fstype: fstype.unwrap_or_else(|| String::from("")), + label: label.unwrap_or_else(|| String::from("")), + uuid: uuid.unwrap_or_else(|| String::from("")), + mountpoint: mountinfo + .map(|m| String::from(m.dest.to_string_lossy())) + .unwrap_or_else(|| String::from("")), + }) +} + +// Create a new BlockDevice object from collected information. +// This function also contains the logic for determining whether +// or not the device that this represents is "available" for use. +fn new_device( + parent: Option<&str>, + include: bool, + device: &Device, + mounts: &HashMap, +) -> Option { + if let Some(devname) = device.property_value("DEVNAME") { + let partition = new_partition(parent, device); + let filesystem = new_filesystem(device, mounts.get(devname)); + let devmajor: u32 = Property(device.property_value("MAJOR")).into(); + let size: u64 = Property(device.attribute_value("size")).into(); + + let available = include + && size > 0 + && !mayastor_device(device) + && usable_device(&devmajor) + && (partition.is_none() || usable_partition(&partition)) + && filesystem.is_none(); + + return Some(BlockDevice { + devname: String::from(devname.to_str().unwrap_or("")), + devtype: Property(device.property_value("DEVTYPE")).into(), + devmajor, + devminor: Property(device.property_value("MINOR")).into(), + model: Property(device.property_value("ID_MODEL")).into(), + devpath: Property(device.property_value("DEVPATH")).into(), + devlinks: device + .property_value("DEVLINKS") + .map(|s| s.to_str()) + .flatten() + .unwrap_or("") + .split(' ') + .filter(|&s| s != "") + .map(String::from) + .collect(), + size, + partition, + filesystem, + available, + }); + } + None +} + +// Get the list of current filesystem mounts. +fn get_mounts() -> Result, Error> { + let mut table: HashMap = HashMap::new(); + + for entry in MountIter::new()? { + if let Ok(mount) = entry { + table.insert(OsString::from(mount.source.clone()), mount); + } + } + + Ok(table) +} + +// Iterate through udev to generate a list of all (block) devices +// with DEVTYPE == "disk" +fn get_disks( + all: bool, + mounts: &HashMap, +) -> Result, Error> { + let mut list: Vec = Vec::new(); + + let mut enumerator = Enumerator::new()?; + + enumerator.match_subsystem("block")?; + enumerator.match_property("DEVTYPE", "disk")?; + + for entry in enumerator.scan_devices()? { + if let Some(devname) = entry.property_value("DEVNAME") { + let partitions = get_partitions(devname.to_str(), &entry, mounts)?; + + if let Some(device) = + new_device(None, partitions.is_empty(), &entry, &mounts) + { + if all || device.available { + list.push(device); + } + } + + for device in partitions { + if all || device.available { + list.push(device); + } + } + } + } + + Ok(list) +} + +// Iterate through udev to generate a list of all (block) devices +// associated with parent device +fn get_partitions( + parent: Option<&str>, + disk: &Device, + mounts: &HashMap, +) -> Result, Error> { + let mut list: Vec = Vec::new(); + + let mut enumerator = Enumerator::new()?; + + enumerator.match_parent(disk)?; + enumerator.match_property("DEVTYPE", "partition")?; + + for entry in enumerator.scan_devices()? { + if let Some(device) = new_device(parent, true, &entry, &mounts) { + list.push(device); + } + } + + Ok(list) +} + +/// Return a list of block devices on the current host. +/// The parameter controls whether to return list containing +/// all matching devices, or just those deemed to be available. +pub async fn list_block_devices(all: bool) -> Result, Error> { + let mounts = get_mounts()?; + get_disks(all, &mounts) +} diff --git a/mayastor/src/host/mod.rs b/mayastor/src/host/mod.rs new file mode 100644 index 000000000..13d238869 --- /dev/null +++ b/mayastor/src/host/mod.rs @@ -0,0 +1 @@ +pub mod blk_device; diff --git a/mayastor/src/lib.rs b/mayastor/src/lib.rs index 869ec590d..8458120f8 100644 --- a/mayastor/src/lib.rs +++ b/mayastor/src/lib.rs @@ -14,6 +14,7 @@ pub mod core; pub mod delay; pub mod ffihelper; pub mod grpc; +pub mod host; pub mod jsonrpc; pub mod logger; pub mod lvs; diff --git a/nix/pkgs/mayastor/default.nix b/nix/pkgs/mayastor/default.nix index e8595dbff..867915a89 100644 --- a/nix/pkgs/mayastor/default.nix +++ b/nix/pkgs/mayastor/default.nix @@ -41,7 +41,7 @@ let buildProps = rec { name = "mayastor"; #cargoSha256 = "0000000000000000000000000000000000000000000000000000"; - cargoSha256 = "000shvfvfz3c3pr32zvmwcac9xh12y4ffy7xbfcpjfj0i310nbgi"; + cargoSha256 = "1m8097h48zz4d20gk9q1aw25548m2aqfxjlr6nck7chrqccvwr54"; inherit version; src = whitelistSource ../../../. [ "Cargo.lock" diff --git a/rpc/proto/mayastor.proto b/rpc/proto/mayastor.proto index 820d93fc0..614dc50ef 100644 --- a/rpc/proto/mayastor.proto +++ b/rpc/proto/mayastor.proto @@ -64,6 +64,9 @@ service Mayastor { // Snapshot operations rpc CreateSnapshot (CreateSnapshotRequest) returns (CreateSnapshotReply) {} + + // Enumerate block devices on current host + rpc ListBlockDevices (ListBlockDevicesRequest) returns (ListBlockDevicesReply) {} } // Means no arguments or no return value. @@ -368,3 +371,39 @@ message BdevUri { message CreateReply { string name = 1; } + +message BlockDevice { + message Partition { + string parent = 1; // devname of parent device to which this partition belongs + uint32 number = 2; // partition number + string name = 3; // partition name + string scheme = 4; // partition scheme: gpt, dos, ... + string typeid = 5; // partition type identifier + string uuid = 6; // UUID identifying partition + } + message Filesystem { + string fstype = 1; // filesystem type: ext3, ntfs, ... + string label = 2; // volume label + string uuid = 3; // UUID identifying the volume (filesystem) + string mountpoint = 4; // path where filesystem is currently mounted + } + string devname = 1; // entry in /dev associated with device + string devtype = 2; // currently "disk" or "partition" + uint32 devmajor = 3; // major device number + uint32 devminor = 4; // minor device number + string model = 5; // device model - useful for identifying mayastor devices + string devpath = 6; // official device path + repeated string devlinks = 7; // list of udev generated symlinks by which device may be identified + uint64 size = 8; // size of device in (512 byte) blocks + Partition partition = 9; // partition information in case where device represents a partition + Filesystem filesystem = 10; // filesystem information in case where a filesystem is present + bool available = 11; // identifies if device is available for use (ie. is not "currently" in use) +} + +message ListBlockDevicesRequest { + bool all = 1; // list "all" block devices found (not just "available" ones) +} + +message ListBlockDevicesReply { + repeated BlockDevice devices = 1; +}