Fix comment depth, generate configopt details

* fix RTD push on Travis if the pseudo-release is already up-to-date
* added pattern for at-words to the lexer and new token type
* fix custom parsers didn't set lint spec for PositionalGroupNode
* don't pedantically error out on empty JSON config file
* parser can comprehend comments belonging to higher than the current depth
  in the semantic tree.
* cleanup README, move some stuff to documentation pointers
* generate the config options details page rather than hand writing it

Closes: #109
Closes: #122
Closes: #168

CMakeLists.txt

@@ -47,6 +47,7 @@ add_custom_target(format)
 add_custom_target(gen)
 add_custom_target(lint)
 add_custom_target(test-deps)
+add_custom_target(wheels)
 
 set(ENV{PYTHONPATH} ${CMAKE_SOURCE_DIR})
 set(CTEST_ENVIRONMENT "PYTHONPATH=${CMAKE_SOURCE_DIR}")

cmake_format/CMakeLists.txt

@@ -81,7 +81,10 @@ format_and_lint(
   tools/parse_cmake_help.py
   tools/push_github_release.py
   tools/usage_lexer.py
-  tools/usage_parser.py)
+  tools/usage_parser.py
+  vscode_extension/CMakeLists.txt
+  # EXCLUDE "command_tests/" "doc/stage" "test/.*" ".*\\.jinja.py$"
+)
 
 set(_testnames invocation_tests layout_tests lexer_tests markup_tests
                parser_tests)
@@ -102,43 +105,31 @@ if(NOT IS_TRAVIS_CI)
   endforeach()
 endif()
 
-add_test(
-  NAME cmake_format-check-lint-manifest
-  COMMAND
-    python -Bm cmake.validate_lint_manifest #
-    ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} #
-    --exclude "command_tests/" "doc/stage" "test/.*" ".*\\.jinja.py$"
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-
 set(_genfiles)
-
 add_custom_command(
-  OUTPUT cmake_format/parse/variables.py
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/parse/variables.py
   COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile
           cmake_format/parse/variables.py variables
   DEPENDS tools/parse_cmake_help.py tools/variables.jinja.py
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
   COMMENT "Generating variables.py")
-list(APPEND _genfiles cmake_format/parse/variables.py)
-
+list(APPEND _genfiles ${CMAKE_CURRENT_SOURCE_DIR}/parse/variables.py)
 
 add_custom_command(
-  OUTPUT cmake_format/parse/properties.py
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/parse/properties.py
   COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile
           cmake_format/parse/properties.py properties
   DEPENDS tools/parse_cmake_help.py tools/properties.jinja.py
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
   COMMENT "Generating properties.py")
-list(APPEND _genfiles cmake_format/parse/properties.py)
+list(APPEND _genfiles ${CMAKE_CURRENT_SOURCE_DIR}/parse/properties.py)
 
-add_custom_target(
-  gen-cmake_format
-  DEPENDS ${_genfiles})
+add_custom_target(gen-cmake_format DEPENDS ${_genfiles})
 add_dependencies(gen gen-cmake_format)
 
-# NOTE(josh): this is just here to induce a dependency on the configure step.
-# If we change the version number in __init__.py we need to re-run cmake so we
-# can get out the version number and create rules for the distribution files.
+# NOTE(josh): this is just here to induce a dependency on the configure step. If
+# we change the version number in __init__.py we need to re-run cmake so we can
+# get out the version number and create rules for the distribution files.
 configure_file(__init__.py init.stamp COPYONLY)
 
 execute_process(
@@ -152,16 +143,19 @@ if(NOT _resultcode EQUAL 0)
     FATAL_ERROR "Failed to get cmake-format version number from __init__.py")
 endif()
 
+# NOTE(josh): python uses dot to separate pre-release while node uses dash
+string(REGEX REPLACE "([0-9]+\\.[0-9]+\\.[0-9]+)\\.(.*)" "\\1-\\2"
+                     _extension_version "${_version}")
+
 add_custom_command(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.egg
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/.egg
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
 set(_distdir ${CMAKE_CURRENT_BINARY_DIR}/dist)
 add_custom_command(
-  OUTPUT
-    ${_distdir}/cmake_format-${_version}.tar.gz
-    ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+  OUTPUT ${_distdir}/cmake_format-${_version}.tar.gz
+         ${_distdir}/cmake_format-${_version}-py3-none-any.whl
   COMMAND
     # cmake-format: off
     python cmake_format/pypi/setup.py
@@ -175,23 +169,33 @@ add_custom_command(
   DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/.egg
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
+add_custom_target(cmake_format-wheel
+                  DEPENDS ${_distdir}/cmake_format-${_version}-py3-none-any.whl)
+add_dependencies(wheels cmake_format-wheel)
+
 add_custom_command(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
-  COMMAND python -Bm cmake_format.tools.get_release_notes
-          cmake_format/doc/release_notes.rst \$\${TRAVIS_TAG}
-          -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
-  DEPENDS doc/release_notes.rst
-          tools/get_release_notes.py
+  COMMAND
+    python -Bm cmake_format.tools.get_release_notes
+    cmake_format/doc/release_notes.rst \$\${TRAVIS_TAG} -o
+    ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
+  DEPENDS doc/release_notes.rst tools/get_release_notes.py
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
 add_custom_command(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
-  COMMAND pandoc -s
-          -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+  COMMAND pandoc -s -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
           ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
   DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
+add_subdirectory(command_tests)
+add_subdirectory(contrib)
+add_subdirectory(doc)
+add_subdirectory(test)
+add_subdirectory(vscode_extension)
+
+set(_vscedir ${CMAKE_CURRENT_BINARY_DIR}/vscode_extension)
 add_custom_target(
   push-github-release
   COMMAND
@@ -201,15 +205,17 @@ add_custom_target(
       \$\${TRAVIS_TAG}
       ${_distdir}/cmake_format-${_version}.tar.gz
       ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+      ${_vscedir}/cmake-format-${_extension_version}.vsix
     # cmake-format: on
-  DEPENDS
-    ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
-    ${_distdir}/cmake_format-${_version}.tar.gz
-    ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+          ${_distdir}/cmake_format-${_version}.tar.gz
+          ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+          # ${_vscedir}/cmake-format-${_extension_version}.vsix
   COMMENT "Uploading release artifacts"
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
-add_subdirectory(command_tests)
-add_subdirectory(contrib)
-add_subdirectory(doc)
-add_subdirectory(test)
+# NOTE(josh): cmake is broken. It fails to produce an accurate build system
+# when we correctly specify the .vsix path in the above rule. We must use
+# target level dependencies to get the correct order of operations and then
+# omit the actual file-level dependency.
+add_dependencies(push-github-release vscode-extension)

cmake_format/__init__.py

@@ -3,4 +3,4 @@
 """
 from __future__ import unicode_literals
 
-VERSION = '0.6.8.dev0'
+VERSION = '0.6.9.dev0'

cmake_format/__main__.py

@@ -227,7 +227,15 @@ def get_one_config_dict(configfile_path):
   If the filepath has a known extension then we parse it according to that
   extension. Otherwise we try to parse is using each parser one by one.
   """
+  if not os.path.exists(configfile_path):
+    raise common.UserError(
+        "Desired config file does not exist: {}".format(configfile_path))
+
   if configfile_path.endswith('.json'):
+    # NOTE(josh): an empty file is not valid JSON, but we don't need to be as
+    # pedantic
+    if os.stat(configfile_path).st_size == 0:
+      return {}
     with io.open(configfile_path, 'r', encoding='utf-8') as config_file:
       try:
         return json.load(config_file)
@@ -561,7 +569,7 @@ def main():
     logger.fatal(ex.msg)
     return 1
   except common.InternalError as ex:
-    logger.execption(ex.msg)
+    logger.exception(ex.msg)
     return 1
   except AssertionError as ex:
     logger.exception(

cmake_format/annotate.py

@@ -107,8 +107,6 @@ def setup_argparser(arg_parser):
                           help='path to configuration file')
   arg_parser.add_argument('infilepaths', nargs='*')
 
-  configuration.Configuration().add_to_argparser(arg_parser)
-
 
 def main():
   """Parse arguments, open files, start work."""

cmake_format/command_tests/__init__.py

@@ -485,7 +485,7 @@ class TestComment(TestBase):
   """
   Test various examples involving comments
   """
-  kExpectNumSidecarTests = 3
+  kExpectNumSidecarTests = 5
 
 
 class TestConditional(TestBase):
@@ -548,4 +548,4 @@ class TestSet(TestBase):
   """
   Test various examples of the set() function
   """
-  kExpectNumSidecarTests = 8
+  kExpectNumSidecarTests = 9

cmake_format/command_tests/comment_tests.cmake

@@ -25,3 +25,47 @@ add_custom_target(
   my-target COMMAND foo bar baz # This comment is a trailing comment
                     # But this comment is a line comment
 )
+
+# test: comment attached at correct depth
+ExternalProject_Add(
+  FOO
+  PREFIX ${FOO_PREFIX}
+  TMP_DIR ${TMP_DIR}
+  STAMP_DIR ${FOO_PREFIX}/stamp
+  # Download
+  DOWNLOAD_DIR ${DOWNLOAD_DIR}
+  DOWNLOAD_NAME ${FOO_ARCHIVE_FILE_NAME}
+  URL ${STORAGE_URL}/${FOO_ARCHIVE_FILE_NAME}
+  URL_MD5 ${FOO_MD5}
+  # Patch
+  PATCH_COMMAND ${PATCH_COMMAND} ${PROJECT_SOURCE_DIR}/patch.diff
+  # Configure
+  SOURCE_DIR ${SRC_DIR}
+  CMAKE_ARGS ${CMAKE_OPTS}
+  # Build
+  BUILD_IN_SOURCE 1
+  BUILD_BYPRODUCTS ${CUR_COMPONENT_ARTIFACTS}
+  # Logging
+  LOG_CONFIGURE 1
+  LOG_BUILD 1
+  LOG_INSTALL 1)
+
+# test: comment_depth_doesnt_overbreak
+#[==[
+if(FOO_FOUND)
+  ExternalProject_Add(
+    FOO
+    PREFIX ${FOO_PREFIX}
+    TMP_DIR ${TMP_DIR}
+# This comment belongs to the statement group
+  )
+endif()
+]==]
+if(FOO_FOUND)
+  ExternalProject_Add(
+    FOO
+    PREFIX ${FOO_PREFIX}
+    TMP_DIR ${TMP_DIR}
+    # This comment belongs to the statement group
+  )
+endif()

cmake_format/command_tests/set_tests.cmake

@@ -72,3 +72,9 @@ set(SOURCES
     source_f.cc
     source_g.cc
     source_h.cc)
+
+# test: cmdline_is_hwrapped
+set(CTEST_GIT_COMMAND
+    really-long-argument really-long-argument really-long-argument
+    really-long-argument really-long-argument really-long-argument
+    really-long-argument really-long-argument really-long-argument)

cmake_format/config_util.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import collections
 import contextlib
 import inspect
@@ -179,6 +181,8 @@ def __set__(self, obj, value):
   def __set_name__(self, owner, name):
     # pylint: disable=protected-access
     owner._field_registry.append(self)
+    if sys.version_info < (3, 0, 0):
+      name = name.decode("utf-8")
     self.name = name
 
   def consume_value(self, obj, value_dict):
@@ -225,9 +229,15 @@ def __get__(self, obj, objtype):
   def __set__(self, obj, value):
     setattr(obj, "_" + self.name, value)
 
+  def unset(self, obj):
+    if self.has_override(obj):
+      delattr(obj, "_" + self.name)
+
   def __set_name__(self, owner, name):
     # pylint: disable=protected-access
     owner._field_registry.append(self)
+    if sys.version_info < (3, 0, 0):
+      name = name.decode("utf-8")
     self.name = name
 
   def consume_value(self, obj, value):

cmake_format/configuration.py

@@ -38,11 +38,11 @@ class MarkupConfig(ConfigObject):
   fence_pattern = FieldDescriptor(
       markup.FENCE_PATTERN,
       "Regular expression to match preformat fences in comments"
-      " default=r'{}'".format(markup.FENCE_PATTERN)
+      " default= ``r'{}'``".format(markup.FENCE_PATTERN)
   )
   ruler_pattern = FieldDescriptor(
       markup.RULER_PATTERN,
-      "Regular expression to match rulers in comments default=r'{}'"
+      "Regular expression to match rulers in comments default= ``r'{}'``"
       .format(markup.RULER_PATTERN)
   )
   explicit_trailing_pattern = FieldDescriptor(
@@ -68,14 +68,9 @@ class MarkupConfig(ConfigObject):
       "enable comment markup parsing and reflow"
   )
 
-  def __init__(self, **kwargs):
-    super(MarkupConfig, self).__init__(**kwargs)
-    self.bullet_char = str(self.bullet_char)[0]
-    self.enum_char = str(self.enum_char)[0]
-
 
 class EncodingConfig(ConfigObject):
-  """Options effecting file encoding"""
+  """Options affecting file encoding"""
 
   _field_registry = []
 
@@ -188,7 +183,7 @@ class LinterConfig(ConfigObject):
 
 
 class FormattingConfig(ConfigObject):
-  """Options effecting formatting."""
+  """Options affecting formatting."""
 
   _field_registry = []
 
@@ -391,26 +386,26 @@ class MiscConfig(ConfigObject):
       " Currently only `command_case` is supported."
   )
 
-  def __init__(self, **kwargs):  # pylint: disable=W0613
-    per_command = kwargs.pop("per_command", {})
-    super(MiscConfig, self).__init__(**kwargs)
-
-    self.per_command = commands.get_default_config()
-    for command, cdict in per_command.items():
+  def _update_derived(self):
+    self.per_command_ = commands.get_default_config()
+    for command, cdict in self.per_command.items():
       if not isinstance(cdict, dict):
         logging.warning("Invalid override of type %s for %s",
                         type(cdict), command)
         continue
 
       command = command.lower()
-      if command not in self.per_command:
-        self.per_command[command] = {}
-      self.per_command[command].update(cdict)
+      if command not in self.per_command_:
+        self.per_command_[command] = {}
+      self.per_command_[command].update(cdict)
+
+  def __init__(self, **kwargs):  # pylint: disable=W0613
+    self.per_command_ = {}
+    super(MiscConfig, self).__init__(**kwargs)
 
 
 class Configuration(ConfigObject):
-  """Various configuration options/parameters for formatting
-  """
+  """Various configuration options and parameters"""
   _field_registry = []
 
   parse = SubtreeDescriptor(ParseConfig)
@@ -444,7 +439,7 @@ def resolve_for_command(self, command_name, config_key, default_value=None):
           "in the global configuration ({})".format(config_key))
       default_value = getattr(configobj, fieldname)
 
-    command_dict = self.misc.per_command.get(command_name.lower(), {})
+    command_dict = self.misc.per_command_.get(command_name.lower(), {})
     if config_key in command_dict:
       return command_dict[config_key]