Multi-pargs, pgroupt tags, initial genparsers

* Add missing dependency on six
* Update pylint, flake8 used in CI
* Remove spurious config warning for some unfiltered command line options
* Add tags field to positional argument groups. Assign "file-list" tag to
  file lists from add_library and add_executable
* Remove "sortable" flag from root TreeNode class
* Custom commands can specify if a positional group is a command line
* Custom commands can specify multiple positional groups
* `max_pargs_hwrap` does not apply to `cmdline` groups
* Remove stale members from `TreeNode`
* Format extension.ts with two spaces
* Create a tool to generate parsers from `cmake_parse_args`

cmake_format/BUILD

@@ -12,6 +12,7 @@ py_library(
         "configuration.py",
         "ctest_to.py",
         "formatter.py",
+        "genparsers.py",
         "invocation_tests.py",
         "layout_tests.py",
         "lexer.py",
@@ -38,6 +39,7 @@ py_library(
         "parse_funs/file.py",
         "parse_funs/foreach.py",
         "parse_funs/install.py",
+        "parse_funs/list.py",
         "parse_funs/miscellaneous.py",
         "parse_funs/random.py",
         "parse_funs/set.py",

cmake_format/CMakeLists.txt

@@ -28,6 +28,7 @@ format_and_lint(
   doc/docsources_test.py
   doc/gendoc_sources.py
   formatter.py
+  genparsers.py
   invocation_tests.py
   layout_tests.py
   lexer.py
@@ -54,6 +55,7 @@ format_and_lint(
   parse_funs/file.py
   parse_funs/foreach.py
   parse_funs/install.py
+  parse_funs/list.py
   parse_funs/miscellaneous.py
   parse_funs/random.py
   parse_funs/set.py

cmake_format/__init__.py

@@ -3,4 +3,4 @@
 """
 from __future__ import unicode_literals
 
-VERSION = '0.6.6'
+VERSION = '0.6.7'

cmake_format/__main__.py

@@ -406,13 +406,13 @@ def get_argdict(args):
     if hasattr(configuration.Configuration, key):
       continue
     # Remove common command line arguments
-    if key in ["log_level", "outfile_path", "infilepaths"]:
+    if key in ["log_level", "outfile_path", "infilepaths", "config_files"]:
       continue
     # Remove --dump-config command line arguments
     if key in ["dump_config", "with_help", "with_defaults"]:
       continue
     # Remove cmake-format command line arguments
-    if key in ["in_place", "check"]:
+    if key in ["dump", "check", "in_place"]:
       continue
     if value is None:
       continue

cmake_format/command_tests/__init__.py

@@ -488,6 +488,13 @@ class TestConditional(TestBase):
   kExpectNumSidecarTests = 5
 
 
+class TestCustomCommand(TestBase):
+  """
+  Test some custom command usages
+  """
+  kExpectNumSidecarTests = 6
+
+
 class TestExport(TestBase):
   """
   Test various examples of export()

cmake_format/command_tests/__main__.py

@@ -4,10 +4,11 @@
 from cmake_format.command_tests import (
     TestAddCustomCommand,
     TestConditional,
+    TestCustomCommand,
     TestExport,
     TestExternalProject,
-    TestForeach,
     TestFile,
+    TestForeach,
     TestInstall,
     TestSetTargetProperties,
     TestSet,)

cmake_format/command_tests/custom_command_tests.cmake

@@ -0,0 +1,122 @@
+# test: legacy_spec
+#[=[
+additional_commands = {
+  "foo": {
+    "flags": ["BAR", "BAZ"],
+    "kwargs": {
+      "HEADERS": "*",
+      "SOURCES": "*",
+      "DEPENDS": "*"
+    }
+  }
+}
+]=]
+foo(nonkwarg_a nonkwarg_b
+    HEADERS a.h b.h c.h d.h e.h f.h
+    SOURCES a.cc b.cc d.cc
+    DEPENDS foo
+    bar baz)
+
+# test: new_spec_dict
+#[=[
+additional_commands = {
+  "foo": {
+    "pargs": {
+      "flags": ["BAR", "BAZ"],
+      "legacy": True,
+    },
+    "kwargs": {
+      "HEADERS": "*",
+      "SOURCES": "*",
+      "DEPENDS": "*"
+    }
+  }
+}
+]=]
+foo(nonkwarg_a nonkwarg_b
+    HEADERS a.h b.h c.h d.h e.h f.h
+    SOURCES a.cc b.cc d.cc
+    DEPENDS foo
+    bar baz)
+
+# test: new_spec_list
+#[=[
+additional_commands = {
+  "foo": {
+    "pargs": [{
+      "flags": ["BAR", "BAZ"],
+      "legacy": True,
+    }],
+    "kwargs": {
+      "HEADERS": "*",
+      "SOURCES": "*",
+      "DEPENDS": "*"
+    }
+  }
+}
+]=]
+foo(nonkwarg_a nonkwarg_b bar baz
+    HEADERS a.h b.h c.h d.h e.h f.h
+    SOURCES a.cc b.cc d.cc
+    DEPENDS foo)
+
+# test: new_spec_arg_kwargs
+#[=[
+additional_commands = {
+  "foo": {
+    "pargs": [["*", {
+      "flags": ["BAR", "BAZ"],
+      "legacy": True,
+    }]],
+    "kwargs": {
+      "HEADERS": "*",
+      "SOURCES": "*",
+      "DEPENDS": "*"
+    }
+  }
+}
+]=]
+foo(nonkwarg_a nonkwarg_b
+    HEADERS a.h b.h c.h d.h e.h f.h
+    SOURCES a.cc b.cc d.cc
+    DEPENDS foo
+    bar baz)
+
+# test: new_spec_multiple
+#[=[
+additional_commands = {
+  "foo": {
+    "pargs": ["*", {"flags": ["BAR", "BAZ"]}],
+    "kwargs": {
+      "HEADERS": "*",
+      "SOURCES": "*",
+      "DEPENDS": "*"
+    }
+  }
+}
+]=]
+foo(nonkwarg_a nonkwarg_b
+    HEADERS a.h b.h c.h d.h e.h f.h
+    SOURCES a.cc b.cc d.cc
+    DEPENDS foo
+    bar baz)
+
+# test: custom_cmdline
+#[=[
+additional_commands = {
+  "mything": {
+    "kwargs": {
+      "MYCOMMAND": {
+        "pargs": {
+          "tags": ["cmdline"]
+        }
+      }
+    }
+  }
+}
+]=]
+mything(
+  foo bar baz
+  MYCOMMAND
+    this is a sequence of arguments that are passed to the shell as a command
+    and should not be wrapped even though the sequence is very long)

cmake_format/commands.py

@@ -5,6 +5,9 @@
 from __future__ import unicode_literals
 import sys
 
+from cmake_format.common import UserError
+from cmake_format.parse.util import PositionalSpec
+
 ZERO_OR_MORE = '*'
 ONE_OR_MORE = '+'
 
@@ -72,7 +75,56 @@ def make_conditional_spec(name=None):
                      kwargs={'AND': spec, 'OR': spec})
 
 
-class CommandSpec(dict):
+def parse_pspec(pargs, flags):
+  """
+  Parse a positional argument specification.
+  """
+  out = []
+
+  # Default pargs is "*"
+  if pargs is None:
+    pargs = ZERO_OR_MORE
+
+  # If we only have one scalar specification, return a legacy specification
+  if isinstance(pargs, STRING_TYPES + (int,)):
+    return [PositionalSpec(pargs, flags=flags, legacy=True)]
+
+  if flags:
+    raise UserError(
+        "Illegal use of top-level 'flags' keyword with new-style positional"
+        " argument declaration")
+
+  # If we only have one dictionary specification, then put it in a dictionary
+  # so that we can do the rest consistently
+  if isinstance(pargs, dict):
+    pargs = [pargs]
+
+  for pargdecl in pargs:
+    if isinstance(pargdecl, STRING_TYPES + (int,)):
+      # A scalar declaration is interpreted as npargs
+      out.append(PositionalSpec(pargdecl))
+      continue
+
+    if isinstance(pargdecl, dict):
+        # A dictionary is interpreted as init kwargs
+      if "npargs" not in pargdecl:
+        pargdecl = dict(pargdecl)
+        pargdecl["nargs"] = ZERO_OR_MORE
+      out.append(PositionalSpec(**pargdecl))
+      continue
+
+    if isinstance(pargdecl, (list, tuple)):
+        # A list or tuple is interpreted as (*args, [kwargs])
+      args = list(pargdecl)
+      kwargs = {}
+      if isinstance(args[-1], dict):
+        kwargs = args.pop(-1)
+      out.append(PositionalSpec(*args, **kwargs))
+
+  return out
+
+
+class CommandSpec(object):
   """
   A command specification is primarily a dictionary mapping keyword arguments
   to command specifications. It also includes a command name and number of
@@ -81,47 +133,41 @@ class CommandSpec(dict):
 
   def __init__(self, name, pargs=None, flags=None, kwargs=None):
     super(CommandSpec, self).__init__()
-    if sys.version_info[0] < 3:
-      scalar_types = (int,) + STRING_TYPES
-    else:
-      scalar_types = (int, str)
-
-    if pargs is None:
-      pargs = ZERO_OR_MORE
-
-    if flags is None:
-      flags = []
+    scalar_types = (int,) + STRING_TYPES
 
     self.name = name
-    self.pargs = pargs
 
-    for flagname in flags:
-      self[flagname] = 0
+    try:
+      self.pargs = parse_pspec(pargs, flags)
+    except (TypeError, UserError) as ex:
+      message = (
+          "Invalid user-supplied specification for positional arguments of "
+          "{}:\n{}".format(name, ex))
+      raise UserError(message)
 
-    self.flags = list(flags)
+    self.kwargs = {}
 
     if kwargs is not None:
       if isinstance(kwargs, dict):
         items = kwargs.items()
       elif isinstance(kwargs, (list, tuple)):
         items = list(kwargs)
-
       for keyword, spec in items:
         if isinstance(spec, scalar_types):
-          self[keyword] = spec
+          self.kwargs[keyword] = CommandSpec(name=keyword, pargs=spec)
         elif isinstance(spec, CommandSpec):
-          self[keyword] = spec
+          self.kwargs[keyword] = spec
         elif isinstance(spec, dict):
-          self[keyword] = CommandSpec(name=keyword, **spec)
+          self.kwargs[keyword] = CommandSpec(name=keyword, **spec)
         else:
           raise ValueError("Unexpected type '{}' for kwargs"
                            .format(type(kwargs)))
 
   def is_flag(self, key):
-    return self.get(key, None) == 0
+    return self.kwargs.get(key, None) == 0
 
   def is_kwarg(self, key):
-    subspec = self.get(key, None)
+    subspec = self.kwargs.get(key, None)
     if subspec is None:
       return False
     if isinstance(subspec, int):
@@ -132,10 +178,10 @@ def is_kwarg(self, key):
                      .format(key, type(subspec)))
 
   def add(self, name, pargs=None, flags=None, kwargs=None):
-    self[name.lower()] = CommandSpec(name, pargs, flags, kwargs)
+    self.kwargs[name.lower()] = CommandSpec(name, pargs, flags, kwargs)
 
   def add_conditional(self, name):
-    self[name.lower()] = make_conditional_spec(name)
+    self.kwargs[name.lower()] = make_conditional_spec(name)
 
 
 def get_fn_spec():
@@ -321,20 +367,6 @@ def get_fn_spec():
       "include_directories",
       flags=["AFTER", "BEFORE", "SYSTEM"], kwargs={})
 
-  fn_spec.add(
-      "list", flags=[], kwargs={
-          "APPEND": ZERO_OR_MORE,
-          "FIND": ZERO_OR_MORE,
-          "GET": ZERO_OR_MORE,
-          "INSERT": ZERO_OR_MORE,
-          "LENGTH": ZERO_OR_MORE,
-          "REMOVE_AT": ZERO_OR_MORE,
-          "REMOVE_DUPLICATES": ZERO_OR_MORE,
-          "REMOVE_ITEM": ZERO_OR_MORE,
-          "REVERSE": ZERO_OR_MORE,
-          "SORT": ZERO_OR_MORE
-      })
-
   fn_spec.add(
       "mark_as_advanced", flags=["CLEAR", "FORCE"], kwargs={})
 

cmake_format/common.py

@@ -103,6 +103,9 @@ def __init__(self, msg=None):
     super(UserError, self).__init__()
     self.msg = msg
 
+  def __repr__(self):
+    return self.msg
+
 
 class InternalError(Exception):
   """Raised when we encounter something we do not expect, indicating a problem
@@ -111,3 +114,6 @@ class InternalError(Exception):
   def __init__(self, msg=None):
     super(InternalError, self).__init__()
     self.msg = msg
+
+  def __repr__(self):
+    return self.msg

cmake_format/configuration.py

@@ -211,6 +211,11 @@ class FormattingConfig(ConfigObject):
       "If a positional argument group contains more than this many"
       " arguments, then force it to a vertical layout. "
   )
+  max_rows_cmdline = FieldDescriptor(
+      2,
+      "If a cmdline positional group consumes more than this many lines"
+      " without nesting, then invalidate the layout (and nest)"
+  )
   separate_ctrl_name_with_space = FieldDescriptor(
       False,
       "If true, separate flow control names from their parentheses with a"