diff --git a/mayastor/src/bdev/mod.rs b/mayastor/src/bdev/mod.rs index 9e5d77078..efa5a785e 100644 --- a/mayastor/src/bdev/mod.rs +++ b/mayastor/src/bdev/mod.rs @@ -21,7 +21,12 @@ pub use nexus::{ }, nexus_persistence::{ChildInfo, NexusInfo}, }; -pub use nvmx::nvme_io_ctx_pool_init; +pub use nvmx::{ + nvme_io_ctx_pool_init, + NvmeController, + NvmeControllerState, + NVME_CONTROLLERS, +}; mod aio; pub(crate) mod dev; @@ -33,7 +38,7 @@ pub(crate) mod nexus; mod null; mod nvme; mod nvmf; -mod nvmx; +pub(crate) mod nvmx; mod uring; pub mod util; diff --git a/mayastor/src/bdev/nvmx/mod.rs b/mayastor/src/bdev/nvmx/mod.rs index 7779ef473..c4bd90bde 100755 --- a/mayastor/src/bdev/nvmx/mod.rs +++ b/mayastor/src/bdev/nvmx/mod.rs @@ -28,7 +28,7 @@ pub mod utils; #[derive(Debug)] #[allow(clippy::upper_case_acronyms)] -pub(crate) struct NVMeCtlrList<'a> { +pub struct NVMeCtlrList<'a> { entries: RwLock>>>>, } @@ -90,6 +90,16 @@ impl<'a> NVMeCtlrList<'a> { let mut entries = self.write_lock(); entries.insert(cid, ctl); } + + /// Get the names of all available NVMe controllers. + pub fn controllers(&self) -> Vec { + let entries = self.read_lock(); + entries + .keys() + .map(|k| k.to_string()) + .filter(|k| k.contains("nqn")) // Filter out CIDs + .collect::>() + } } impl<'a> Default for NVMeCtlrList<'a> { @@ -102,7 +112,8 @@ impl<'a> Default for NVMeCtlrList<'a> { } } -static NVME_CONTROLLERS: Lazy = Lazy::new(NVMeCtlrList::default); +pub static NVME_CONTROLLERS: Lazy = + Lazy::new(NVMeCtlrList::default); pub fn nvme_bdev_running_config() -> &'static NvmeBdevOpts { &Config::get().nvme_bdev_opts diff --git a/mayastor/src/bin/mayastor-client/controller_cli.rs b/mayastor/src/bin/mayastor-client/controller_cli.rs new file mode 100755 index 000000000..df330e18b --- /dev/null +++ b/mayastor/src/bin/mayastor-client/controller_cli.rs @@ -0,0 +1,95 @@ +//! +//! methods to interact with NVMe controllers + +use super::context::Context; +use crate::{context::OutputFormat, GrpcStatus}; +use ::rpc::mayastor as rpc; +use clap::{App, AppSettings, ArgMatches, SubCommand}; +use colored_json::ToColoredJson; +use snafu::ResultExt; +use tonic::Status; + +pub fn subcommands<'a, 'b>() -> App<'a, 'b> { + let list = + SubCommand::with_name("list").about("List existing NVMe controllers"); + + SubCommand::with_name("controller") + .settings(&[ + AppSettings::SubcommandRequiredElseHelp, + AppSettings::ColoredHelp, + AppSettings::ColorAlways, + ]) + .about("NVMe controllers") + .subcommand(list) +} + +pub async fn handler( + ctx: Context, + matches: &ArgMatches<'_>, +) -> crate::Result<()> { + match matches.subcommand() { + ("list", Some(args)) => list_controllers(ctx, args).await, + (cmd, _) => { + Err(Status::not_found(format!("command {} does not exist", cmd))) + .context(GrpcStatus) + } + } +} + +fn controller_state_to_str(idx: i32) -> String { + match rpc::NvmeControllerState::from_i32(idx).unwrap() { + rpc::NvmeControllerState::New => "new", + rpc::NvmeControllerState::Initializing => "init", + rpc::NvmeControllerState::Running => "running", + rpc::NvmeControllerState::Faulted => "faulted", + rpc::NvmeControllerState::Unconfiguring => "unconfiguring", + rpc::NvmeControllerState::Unconfigured => "unconfigured", + } + .to_string() +} + +async fn list_controllers( + mut ctx: Context, + _matches: &ArgMatches<'_>, +) -> crate::Result<()> { + let response = ctx + .client + .list_nvme_controllers(rpc::Null {}) + .await + .context(GrpcStatus)?; + + match ctx.output { + OutputFormat::Json => { + println!( + "{}", + serde_json::to_string_pretty(&response.get_ref()) + .unwrap() + .to_colored_json_auto() + .unwrap() + ); + } + OutputFormat::Default => { + let controllers = &response.get_ref().controllers; + if controllers.is_empty() { + ctx.v1("No NVMe controllers found"); + return Ok(()); + } + + let table = controllers + .iter() + .map(|c| { + let size = c.size.to_string(); + let blk_size = c.blk_size.to_string(); + let state = controller_state_to_str(c.state); + + vec![c.name.clone(), size, state, blk_size] + }) + .collect(); + + let hdr = vec!["NAMEs", "SIZE", "STATE", "BLKSIZE"]; + ctx.print_list(hdr, table); + } + } + + Ok(()) +} diff --git a/mayastor/src/bin/mayastor-client/main.rs b/mayastor/src/bin/mayastor-client/main.rs index 5a1d67c6e..eb21b44d9 100644 --- a/mayastor/src/bin/mayastor-client/main.rs +++ b/mayastor/src/bin/mayastor-client/main.rs @@ -12,6 +12,7 @@ use ::rpc::mayastor::{ mod bdev_cli; mod context; +mod controller_cli; mod device_cli; mod jsonrpc_cli; mod nexus_child_cli; @@ -108,6 +109,7 @@ async fn main() -> crate::Result<()> { .subcommand(rebuild_cli::subcommands()) .subcommand(snapshot_cli::subcommands()) .subcommand(jsonrpc_cli::subcommands()) + .subcommand(controller_cli::subcommands()) .get_matches(); let ctx = Context::new(&matches).await.context(ContextError)?; @@ -121,6 +123,7 @@ async fn main() -> crate::Result<()> { ("replica", Some(args)) => replica_cli::handler(ctx, args).await, ("rebuild", Some(args)) => rebuild_cli::handler(ctx, args).await, ("snapshot", Some(args)) => snapshot_cli::handler(ctx, args).await, + ("controller", Some(args)) => controller_cli::handler(ctx, args).await, ("jsonrpc", Some(args)) => jsonrpc_cli::json_rpc_call(ctx, args).await, _ => panic!("Command not found"), }; diff --git a/mayastor/src/grpc/controller_grpc.rs b/mayastor/src/grpc/controller_grpc.rs new file mode 100755 index 000000000..0867958ae --- /dev/null +++ b/mayastor/src/grpc/controller_grpc.rs @@ -0,0 +1,72 @@ +use crate::{ + bdev::{ + nexus::nexus_bdev, + NvmeController, + NvmeControllerState, + NVME_CONTROLLERS, + }, + grpc::{rpc_submit, GrpcResult}, +}; + +use ::rpc::mayastor as rpc; +use std::convert::From; +use tonic::{Response, Status}; + +impl<'a> NvmeController<'a> { + fn to_grpc(&self) -> rpc::NvmeController { + let (size, blk_size) = self + .namespace() + .map_or((0, 0), |ns| (ns.size_in_bytes(), ns.block_len() as u32)); + + rpc::NvmeController { + name: self.name.to_string(), + state: rpc::NvmeControllerState::from(self.get_state()) as i32, + size, + blk_size, + } + } +} + +impl From for rpc::NvmeControllerState { + fn from(state: NvmeControllerState) -> Self { + match state { + NvmeControllerState::New => rpc::NvmeControllerState::New, + NvmeControllerState::Initializing => { + rpc::NvmeControllerState::Initializing + } + NvmeControllerState::Running => rpc::NvmeControllerState::Running, + NvmeControllerState::Faulted(_) => { + rpc::NvmeControllerState::Faulted + } + NvmeControllerState::Unconfiguring => { + rpc::NvmeControllerState::Unconfiguring + } + NvmeControllerState::Unconfigured => { + rpc::NvmeControllerState::Unconfigured + } + } + } +} + +pub async fn list_controllers() -> GrpcResult { + let rx = rpc_submit::<_, _, nexus_bdev::Error>(async move { + let controllers = NVME_CONTROLLERS + .controllers() + .iter() + .filter_map(|n| { + NVME_CONTROLLERS + .lookup_by_name(n) + .map(|c| c.lock().to_grpc()) + }) + .collect::>(); + + Ok(rpc::ListNvmeControllersReply { + controllers, + }) + })?; + + rx.await + .map_err(|_| Status::cancelled("cancelled"))? + .map_err(Status::from) + .map(Response::new) +} diff --git a/mayastor/src/grpc/mayastor_grpc.rs b/mayastor/src/grpc/mayastor_grpc.rs index d9ed8ed8e..9a4697717 100644 --- a/mayastor/src/grpc/mayastor_grpc.rs +++ b/mayastor/src/grpc/mayastor_grpc.rs @@ -2,7 +2,7 @@ //! //! The Mayastor gRPC methods serve as a higher abstraction for provisioning //! replicas and targets to be used with CSI. -//! +// //! We want to keep the code here to a minimal, for example grpc/pool.rs //! contains all the conversions and mappings etc to whatever interface from a //! grpc perspective we provide. Also, by doing his, we can test the methods @@ -16,6 +16,7 @@ use crate::{ }, core::{Bdev, BlockDeviceIoStats, CoreError, Protocol, Share}, grpc::{ + controller_grpc::list_controllers, nexus_grpc::{ nexus_add_child, nexus_destroy, @@ -152,6 +153,7 @@ impl From for Replica { } } } + #[tonic::async_trait] impl mayastor_server::Mayastor for MayastorSvc { async fn create_pool( @@ -904,4 +906,11 @@ impl mayastor_server::Mayastor for MayastorSvc { trace!("{:?}", reply); Ok(Response::new(reply)) } + + async fn list_nvme_controllers( + &self, + _request: Request, + ) -> GrpcResult { + list_controllers().await + } } diff --git a/mayastor/src/grpc/mod.rs b/mayastor/src/grpc/mod.rs index 024d93b01..c57353bba 100644 --- a/mayastor/src/grpc/mod.rs +++ b/mayastor/src/grpc/mod.rs @@ -35,6 +35,7 @@ impl From for tonic::Status { } } mod bdev_grpc; +mod controller_grpc; mod json_grpc; mod mayastor_grpc; mod nexus_grpc; diff --git a/rpc/proto/mayastor.proto b/rpc/proto/mayastor.proto index 612dd15cd..b3aa08579 100644 --- a/rpc/proto/mayastor.proto +++ b/rpc/proto/mayastor.proto @@ -75,6 +75,9 @@ service Mayastor { // Obtain resource usage statistics for the current process rpc GetResourceUsage (Null) returns (GetResourceUsageReply) {} + + // NVMe controllers + rpc ListNvmeControllers (Null) returns (ListNvmeControllersReply) {} } // Means no arguments or no return value. @@ -470,6 +473,26 @@ message CreateReply { string name = 1; } +enum NvmeControllerState { + NEW = 0; + INITIALIZING = 1; + RUNNING = 2; + FAULTED = 3; + UNCONFIGURING = 4; + UNCONFIGURED = 5; +} + +message NvmeController { + string name = 1; // NVMe controller name + NvmeControllerState state = 2; // Current state of the NVMe controller + uint64 size = 3; // Size of the controller's namespace (0 if no namespace attached). + uint32 blk_size = 4; // Block size of the namespace (0 if no namespace attached). +} + +message ListNvmeControllersReply { + repeated NvmeController controllers = 1; +} + // SPDK json-rpc proxy service service JsonRpc { diff --git a/test/grpc/test_cli.js b/test/grpc/test_cli.js index f2720b957..44d599f27 100644 --- a/test/grpc/test_cli.js +++ b/test/grpc/test_cli.js @@ -177,6 +177,26 @@ describe('mayastor-client', function () { ] } }, + { + method: 'ListNvmeControllers', + input: {}, + output: { + controllers: [ + { + name: '10.0.0.4:8420/nqn.2019-05.io.openebs:null1n1', + state: 2, + size: 100 * (1024 * 1024), + blkSize: 4096 + }, + { + name: '10.0.0.5:8420/nqn.2019-05.io.openebs:null1n1', + state: 3, + size: 100 * (1024 * 1024), + blkSize: 4096 + } + ] + } + }, { method: 'DestroyNexus', input: { @@ -495,6 +515,42 @@ describe('mayastor-client', function () { }); }); + it('should list nvme controllers for nexus children', function (done) { + const cmd = util.format('%s -q controller list', EGRESS_CMD); + + exec(cmd, (err, stdout, stderr) => { + const controllers = []; + if (err) { return done(err); } + assert.isEmpty(stderr); + + stdout.split('\n').forEach((line) => { + const parts = line.trim().split(' ').filter((s) => s.length !== 0); + if (parts.length <= 1) { return; } + + controllers.push({ + name: parts[0], + size: parts[1], + state: parts[2], + blk_size: parts[3] + }); + }); + + assert.lengthOf(controllers, 2); + + assert.equal(controllers[0].name, '10.0.0.4:8420/nqn.2019-05.io.openebs:null1n1'); + assert.equal(controllers[0].size, '104857600'); + assert.equal(controllers[0].state, 'running'); + assert.equal(controllers[0].blk_size, '4096'); + + assert.equal(controllers[1].name, '10.0.0.5:8420/nqn.2019-05.io.openebs:null1n1'); + assert.equal(controllers[1].size, '104857600'); + assert.equal(controllers[1].state, 'faulted'); + assert.equal(controllers[1].blk_size, '4096'); + + done(); + }); + }); + it('should list nexus children', function (done) { const cmd = util.format('%s -q nexus children %s', EGRESS_CMD, UUID1);