Implement configtree, fix bracket comments

* Fix greedy match for bracket comments
* Implement some more readable error messages
* Add source support for sidecar tests
* Overhaul the configuration data structures, dividing configuration up
  among different classes.
* Remove configuration fields from config object __init__
* Add dump-config options to exclude helptext or defaults
* Implement explicit trailing comments
* Implement "include" from config files
* Move logging init into main() functions

Closes: #156

.cmake-format.py

@@ -1,8 +1,8 @@
-additional_commands = {
-  "pkg_find": {
-    "kwargs": {
-      "PKG": "*"
+with section("parse"):
+  additional_commands = {
+    "pkg_find": {
+      "kwargs": {
+        "PKG": "*"
+      }
     }
   }
-}
-

.gitignore

@@ -31,9 +31,8 @@ bazel-*
 # node package manager
 node_modules
 
-# bazel sucks... 
+# bazel sucks...
 WORKSPACE
 
 # Temp/backup files
 *~
-

cmake/codestyle.cmake

@@ -158,4 +158,14 @@ function(format_and_lint module)
       WARNING "The following files will not be linted/formatted because their"
               " extension is not recognized: \n  ${filelist_}")
   endif()
+
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${module}.lint-manifest "")
+  foreach(listname_ cmake_files_ cc_files_ py_files_ js_files_)
+    if(${listname_})
+      string(REPLACE ";" "\n" filenames_ "${${listname_}}")
+      file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/${module}.lint-manifest
+          "${filenames_}\n")
+    endif()
+  endforeach()
+
 endfunction()

cmake/validate_lint_manifest.py

@@ -0,0 +1,133 @@
+"""
+Scan the source tree rooted at a particulary directory and build a list of all
+lintable files. Compare that list against the list of names in any
+`.lint-manifest` in the binary tree. If the two are not identical, exit with
+nonzero status code.
+"""
+
+import argparse
+import io
+import os
+import logging
+import re
+import sys
+
+KNOWN_EXTENSIONS = [
+    ".cmake",
+    ".c",
+    ".cc",
+    ".cxx",
+    ".cpp",
+    ".h",
+    ".hxx",
+    ".hpp",
+    ".js",
+    ".py",
+]
+
+
+def has_known_extension(filename):
+  for extension in KNOWN_EXTENSIONS:
+    if filename.lower().endswith(extension):
+      return True
+  return False
+
+
+SENTINEL_NAMES = [
+    "CMakeLists.txt",
+]
+
+
+def is_sentinel_name(filename):
+  return filename in SENTINEL_NAMES
+
+
+REJECT_NAMES = [
+    # Outputs of cmake in-source configure
+    "CTestTestfile.cmake",
+    "cmake_install.cmake",
+]
+
+
+def is_known_reject(filename):
+  return filename in REJECT_NAMES
+
+
+def get_source_manifest(rootdir, exclude_pattern):
+  manifest = []
+  excluded = []
+  for directory, dirnames, filenames in os.walk(rootdir):
+    dirnames[:] = sorted(dirnames)
+    relpath_dir = os.path.relpath(directory, rootdir)
+    if exclude_pattern and exclude_pattern.match(relpath_dir):
+      continue
+
+    for filename in filenames:
+      if is_known_reject(filename):
+        continue
+
+      relpath_file = os.path.join(relpath_dir, filename)
+      if relpath_dir == ".":
+        relpath_file = filename
+
+      if exclude_pattern and exclude_pattern.match(relpath_file):
+        excluded.append(relpath_file)
+        continue
+      if has_known_extension(filename) or is_sentinel_name(filename):
+        manifest.append(relpath_file)
+        continue
+  return manifest, excluded
+
+
+def get_generated_manifest(rootdir):
+  manifest = []
+  for directory, dirnames, filenames in os.walk(rootdir):
+    dirnames[:] = sorted(dirnames)
+    relpath_dir = os.path.relpath(directory, rootdir)
+    for filename in filenames:
+      relpath_file = os.path.join(relpath_dir, filename)
+      if filename.endswith(".lint-manifest"):
+        fullpath_file = os.path.join(rootdir, relpath_file)
+        with io.open(fullpath_file, "r", encoding="utf-8") as infile:
+          manifest.extend(line.strip() for line in infile)
+  return manifest
+
+
+def main():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument("source_tree", help="path to the source directory")
+  parser.add_argument("binary_tree", help="path to the binary directory")
+  parser.add_argument("--exclude", nargs="*", help="exclusions")
+  args = parser.parse_args()
+
+  exclude_pattern = None
+  if args.exclude:
+    exclude_pattern = "|".join(args.exclude)
+    exclude_pattern = re.compile(exclude_pattern)
+
+  source_manifest, source_excluded = get_source_manifest(
+      args.source_tree, exclude_pattern)
+  gen_manifest = get_generated_manifest(args.binary_tree)
+
+  source_set = set(source_manifest)
+  gen_set = set(gen_manifest)
+
+  missing = source_set.difference(gen_set)
+  extra = gen_set.difference(source_set.union(source_excluded))
+
+  returncode = 0
+  if missing:
+    logging.error(
+        "The following files are missing from the generated manifests:"
+        "\n  %s", "\n  ".join(sorted(missing)))
+    returncode = 1
+  if extra:
+    logging.error(
+        "The following files listed in the lint manifest are not found in"
+        " the source tree:\n  %s", "\n  ".join(sorted(extra)))
+    returncode = 1
+  return returncode
+
+
+if __name__ == "__main__":
+  sys.exit(main())

cmake_format/CMakeLists.txt

@@ -1,6 +1,7 @@
 format_and_lint(
   cmake_format
   # cmake-format: sort
+  CMakeLists.txt
   __init__.py
   __main__.py
   annotate.py
@@ -13,12 +14,16 @@ format_and_lint(
   command_tests/CMakeLists.txt
   command_tests/misc_tests.py
   common.py
+  config_util.py
   configuration.py
   contrib/__init__.py
+  contrib/CMakeLists.txt
   contrib/sign_ca.py
   contrib/validate_database.py
   contrib/validate_pullrequest.py
+  ctest_to.py
   doc/__init__.py
+  doc/CMakeLists.txt
   doc/conf.py
   doc/docsources_test.py
   doc/gendoc_sources.py
@@ -42,11 +47,15 @@ format_and_lint(
   parse_funs/add_executable.py
   parse_funs/add_library.py
   parse_funs/add_xxx.py
+  parse_funs/break.py
+  parse_funs/deprecated.py
   parse_funs/external_project.py
   parse_funs/fetch_content.py
   parse_funs/file.py
   parse_funs/foreach.py
   parse_funs/install.py
+  parse_funs/miscellaneous.py
+  parse_funs/random.py
   parse_funs/set.py
   parse_funs/set_target_properties.py
   parser_tests.py
@@ -59,6 +68,9 @@ format_and_lint(
   test/screw_users_test.py
   test/version_number_test.py
   tests.py
+  tools/__init__.py
+  tools/bump_version.py
+  tools/generate_missing_parsers.py
   tools/parse_cmake_help.py
   tools/usage_lexer.py
   tools/usage_parser.py)
@@ -86,3 +98,11 @@ add_subdirectory(command_tests)
 add_subdirectory(contrib)
 add_subdirectory(doc)
 add_subdirectory(test)
+
+add_test(
+  NAME cmake_format-check-lint-manifest
+  COMMAND
+    python -Bm cmake.validate_lint_manifest #
+    ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} #
+    --exclude "command_tests/" "test/.*" ".*\\.jinja.py$"
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})

cmake_format/__init__.py

@@ -3,4 +3,4 @@
 """
 from __future__ import unicode_literals
 
-VERSION = '0.6.5'
+VERSION = '0.6.6'