NAS-121797 / 23.10 / CLI for ixdiagnose (by Qubad786)

.github/workflows/test_coverage.yml

@@ -13,15 +13,18 @@ jobs:
   add-coverage:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/truenas/ixdiagnose:master
+      image: ghcr.io/truenas/middleware:master
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
-    - name: Install dependencies
+    - name: Setup dependencies
       run: |
-        pip install -r requirements.txt
-        pip install pytest-cov
-        pip install -U .
+        /usr/bin/install-dev-tools
+    - name: Deps
+      run: |
+        pip install --break-system-packages -r requirements.txt
+        pip install --break-system-packages pytest-cov
+        pip install --break-system-packages -U .
     - name: Run tests
       run: pytest --cov-report xml --cov=ixdiagnose
     - name: Upload coverage to Codecov

docs/filesystem_heirarchy.md

@@ -29,7 +29,7 @@ ixdiagnose/ixdiagnose
 ├── event.py
 ├── exceptions.py
 ├── __init__.py
-├── main.py
+├── cli.py
 ├── plugin.py
 ├── plugins (Directory)
 ├── run.py
@@ -42,7 +42,7 @@ ixdiagnose/ixdiagnose
 - `artifact.py` contains the logic which is used to gather the artifacts.
 - `config.py` contains the logic which is used to parse the configuration file.
 - `event.py` contains the logic which is used to manage the event management used to give real updates on debug generation.
-- `main.py` contains the logic which is used to parse the command line arguments and run the application.
+- `cli.py` contains the logic which is used to parse the command line arguments and run the application.
 - `plugin.py` contains the logic which is used to manage the plugins.
 - `run.py` contains the logic which is used to run the plugins.
 - `utils` contains the utility functions used by the application.

ixdiagnose/artifact.py

@@ -2,16 +2,21 @@
 import traceback
 
 from .artifacts.factory import artifact_factory
+from .config import conf
 from .event import send_event
 from .utils.formatter import dumps
 from .utils.paths import get_artifacts_base_dir
 
 
 def gather_artifacts(percentage: int = 0, total_percentage: int = 100) -> None:
     os.makedirs(get_artifacts_base_dir(), exist_ok=True)
+    to_execute_artifacts = {
+        artifact_name: artifact for artifact_name, artifact in artifact_factory.get_items().items()
+        if artifact_name not in conf.exclude_artifacts
+    }
     artifacts_report = {}
-    artifact_percentage = total_percentage / len(artifact_factory.get_items())
-    for artifact_name, artifact in artifact_factory.get_items().items():
+    artifact_percentage = total_percentage / (len(to_execute_artifacts) or 1)  # We want to handle this quietly
+    for artifact_name, artifact in to_execute_artifacts.items():
         send_event(int(percentage + 0.5), f'Gathering artifact {artifact_name!r}')
 
         try:
@@ -26,5 +31,6 @@ def gather_artifacts(percentage: int = 0, total_percentage: int = 100) -> None:
         artifacts_report[artifact_name] = report
         percentage += artifact_percentage
 
+    send_event(total_percentage, 'Gathered artifacts')
     with open(os.path.join(get_artifacts_base_dir(), 'report.json'), 'w') as f:
         f.write(dumps(artifacts_report))

ixdiagnose/cli.py

@@ -0,0 +1,160 @@
+import click
+import os
+
+from ixdiagnose.event import event_callbacks
+from typing import Callable, List, Optional
+
+from .artifact import gather_artifacts
+from .config import conf
+from .plugin import generate_plugins_debug
+from .run import generate_debug
+
+
+text = ['Specified configuration for debug generation is ']
+
+
+@click.group()
+def main() -> None:
+    pass
+
+
+def list_type(values: str) -> List[str]:
+    return [value.strip() for value in values.split(',')]
+
+
+def update_configuration(
+    timeout: int, compress: Optional[str] = None, debug_path: Optional[str] = None,
+    excluded_artifacts: Optional[List[str]] = None, excluded_plugins: Optional[List[str]] = None,
+) -> None:
+    if compress:
+        conf.compress = True
+        conf.compressed_path = compress
+        text.append(f'- save debug as a compressed folder at: {compress}')
+
+    if timeout:
+        conf.timeout = timeout
+        text.append(f'- timeout for debug generation: {timeout} seconds.')
+
+    if excluded_artifacts:
+        conf.exclude_artifacts = excluded_artifacts
+        text.append(f'- exclude artifacts: {excluded_artifacts}')
+
+    if excluded_plugins:
+        conf.exclude_plugins = excluded_plugins
+        text.append(f'- exclude plugins: {excluded_plugins}')
+
+    if debug_path:
+        conf.debug_path = debug_path
+        text.append(f'- debug path set to {debug_path}')
+
+
+def progress_bar(func: Callable) -> str:
+    bar = click.progressbar(length=100, label='Generating debug')
+    cumulative_progress = 0
+    last_iteration_label = 'Completed debug'
+
+    def callback(progress, desc):
+        nonlocal cumulative_progress
+        progress -= cumulative_progress
+        cumulative_progress += progress
+        if cumulative_progress == 100:
+            bar.label = last_iteration_label
+        else:
+            bar.label = desc
+        bar.update(progress)
+
+    with bar:
+        event_callbacks.register(callback)
+        path = func()
+
+    return path
+
+
+def validate_path(ctx, param, value):
+    if ctx.info_name in ['artifact', 'plugin']:
+        if value is None:
+            raise click.UsageError('Missing option \'--debug-path\'.')
+
+    if value and not os.path.isabs(value):
+        raise click.UsageError('Path must be absolute')
+
+    if ctx.info_name == 'run':
+
+        if value and param.name == 'compress' and os.path.exists(value):
+            raise click.UsageError('Compressed path already exists')
+
+    return value
+
+
+timeout_option = click.option(
+    '-t', '--timeout', type=click.INT, default=20, show_default=True,
+    help='timeout value for middleware client in seconds'
+)
+
+debug_path_option = click.option(
+    '--debug-path', type=click.Path(), callback=validate_path, required=False,
+    help='path where you want to save debug'
+)
+
+
+@main.command(short_help='Generate complete debug')
+@click.option('-s', '--serialized', is_flag=True, default=False, help='generate debug in structured form')
+@click.option(
+    '-c', '--compress', type=str, callback=validate_path,
+    help='get compressed debug, provide file name with complete path'
+)
+@debug_path_option
+@timeout_option
+@click.option(
+    '-Xa', '--exclude-artifacts', type=list_type,
+    help='artifacts you want to exclude in debug (logs,sys_info). A comma separated list without space or in quotes'
+)
+@click.option(
+    '-Xp', '--exclude-plugins', type=list_type,
+    help='plugins you want to exclude in debug (smb,vm,network). A comma separated list without space or in quotes'
+)
+def run(
+    serialized: bool, compress: str, debug_path: str, timeout: int, exclude_artifacts: List[str],
+    exclude_plugins: List[str]
+) -> None:
+    if serialized:
+        conf.structured_data = True
+        text.append('- generate debug in structured form.')
+    else:
+        text.append('- generate debug in default non-structured form.')
+
+    update_configuration(timeout, compress, debug_path, exclude_artifacts, exclude_plugins)
+
+    click.echo('\n'.join(text))
+    path = progress_bar(generate_debug)
+    click.echo(f'Debug saved at {path}')
+
+
+@main.command(short_help='Gather artifacts')
+@debug_path_option
+@timeout_option
+@click.option(
+    '-X', '--exclude', type=list_type,
+    help='artifacts you want to exclude in debug (logs,sys_info). A comma separated list without space or in quotes'
+)
+def artifact(debug_path: str, timeout: int, exclude: List[str]) -> None:
+    update_configuration(timeout, debug_path=debug_path, excluded_artifacts=exclude)
+
+    click.echo('\n'.join(text))
+    progress_bar(gather_artifacts)
+    click.echo(f'Collected artifacts at {debug_path}')
+
+
+@main.command(short_help='Generate plugins\' debug')
+@debug_path_option
+@timeout_option
+@click.option(
+    '-X', '--exclude', type=list_type,
+    help='plugins you want to exclude in debug (smb,vm,network). A comma separated list without space or in quotes'
+)
+def plugin(debug_path: str, timeout: int, exclude: List[str]) -> None:
+    update_configuration(timeout, debug_path=debug_path, excluded_plugins=exclude)
+
+    click.echo('\n'.join(text))
+    progress_bar(generate_plugins_debug)
+    click.echo(f'Generated plugins\' dump at {debug_path}')

ixdiagnose/config.py

@@ -11,6 +11,7 @@ class Configuration:
             'compressed_path': {'type': ['string', 'null']},
             'clean_debug_path': {'type': 'boolean'},
             'debug_path': {'type': ['string', 'null']},
+            'exclude_artifacts': {'type': 'array', 'items': {'type': 'string'}},
             'exclude_plugins': {'type': 'array', 'items': {'type': 'string'}},
             'structured_data': {'type': 'boolean'},
             'timeout': {'type': 'integer'},
@@ -22,6 +23,7 @@ def __init__(self):
         self.compressed_path: Optional[str] = None
         self.clean_debug_path: bool = False
         self.debug_path: Optional[str] = None
+        self.exclude_artifacts: List[str] = []
         self.exclude_plugins: List[str] = []
         self.structured_data: bool = False
         self.timeout: int = 20

ixdiagnose/plugin.py

@@ -12,7 +12,7 @@ def generate_plugins_debug(percentage: int = 0, total_percentage: int = 100) ->
     os.makedirs(get_plugin_base_dir(), exist_ok=True)
 
     to_execute_plugins = {k: v for k, v in plugin_factory.get_items().items() if k not in conf.exclude_plugins}
-    plugin_percentage = total_percentage / len(to_execute_plugins)
+    plugin_percentage = total_percentage / (len(to_execute_plugins) or 1)  # We want to handle this quietly
     plugins_report = {}
     for plugin_name, plugin in to_execute_plugins.items():
         send_event(int(percentage + 0.5), f'Gathering debug information for {plugin_name!r} plugin')
@@ -29,5 +29,6 @@ def generate_plugins_debug(percentage: int = 0, total_percentage: int = 100) ->
         plugins_report[plugin_name] = report
         percentage += plugin_percentage
 
+    send_event(total_percentage, 'Gathered debug information for plugins')
     with open(os.path.join(get_plugin_base_dir(), 'report.json'), 'w') as f:
         f.write(dumps(plugins_report))

ixdiagnose/test/pytest/integration/test_artifacts.py

@@ -132,4 +132,4 @@ def test_truncation_of_file(file_size, file_truncated):
 def test_artifact_event_progress_count():
     event_callbacks.register(callback=event_callback)
     with gather_artifacts():
-        assert len(artifact_factory.get_items()) == len(PROGRESS_DESCRIPTIONS) == len(PROGRESS_TRACK)
+        assert len(artifact_factory.get_items()) + 1 == len(PROGRESS_DESCRIPTIONS) == len(PROGRESS_TRACK)

ixdiagnose/test/pytest/integration/test_plugins.py

@@ -117,4 +117,4 @@ def test_report_schema():
 def test_plugin_event_progress_count():
     event_callbacks.register(callback=event_callback)
     with generate_plugins():
-        assert len(plugin_factory.get_items()) == len(PROGRESS_DESCRIPTIONS) == len(PROGRESS_TRACK)
+        assert len(plugin_factory.get_items()) + 1 == len(PROGRESS_DESCRIPTIONS) == len(PROGRESS_TRACK)