vartags, proptags, docsbuild

* Reduce packaging depependency version numbers
* Add build rules to generate variable and property pattern lists
* Implement lint checks on assignment/use of variables that are "close" to
  builtins except for case.
* Move first_token from configuration object into format context
* Add line, col info to lex error message
* Fix wrong root parser for FetchContent_MakeAvailable
* Fix missing support for string integer npargs
* Fix missing spec for derived classes of PositionalGroupNode
* Fix on/off switch doesn't work inside a statement
* Fix extraneous whitespace inserted before line comment in some statements
* Add more helpful error message on failed configfile parse
* Move documentation build to build time and push documentation artifacts
  to an artifact repository

Closes: #162
Closes: #163
Closes: #164

.gitignore

@@ -2,8 +2,10 @@
 *.o
 *.so
 
-# unified repo tracking
+# sparse-export stuff
 .gitu
+.sparse-export
+tangent/tooling/sparse-exports
 
 # qt creator
 *.qtc.*

.readthedocs.yml

@@ -0,0 +1,20 @@
+# .readthedocs.yml
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+sphinx:
+  configuration: conf.py
+
+build:
+  image: latest
+
+formats: all
+python:
+  version: 3.7
+  install:
+    - requirements: requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - YAML
+        - html-gen

.sparse-export

@@ -0,0 +1 @@
+tangent/tooling/sparse-exports/cmake_format

.travis.yml

@@ -1,13 +1,42 @@
 dist: xenial
 language: python
 python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
+- '2.7'
+- '3.5'
+- '3.6'
+
+before_install:
+- sudo apt-get update
+- sudo apt-get -y install pandoc
 
 install:
-  - pip install -r requirements.txt
+- pip install -r requirements.txt
 
 script:
-  - make better-test
-  - make lint
+- make better-test
+- make lint
+
+env:
+  global:
+    # GITHUB_ACCESS_TOKEN=
+    secure: "e62y3i4FG7KsJ/I8aVKoPk5CG0L7/TW1K4M+BLt5zULsKGyRw6b53I97p5Tc1zcAHqcocFeYL/eC3AucG3VNvyv6LIZuv5Y0upF/MG++tvkfV9Iyok76mP70DXR/ikAJDMgZSdEL/6uTTJ4XfBuH/VvCMMBvj+5ORSL65KIfqXcexj2JWaNtEEfwiVSkDvQZYYbHLRjQdAXzzI7g1xh+8mCmuFAvtn6U1cIZpdQv5u1DNIry8PoQ2asHr+vaCN9iVXvkWcUjoXpJTp96Hd8eO40ts3v3Ja1yJ7GzH09zjbKJS63dcB0LfV5uLjybJCkQ03102ua0V+E5OzPn5VYeIRSRWmdo35e9xZjqqbtNdHqNayOOpew/ik+wf2Nmzva5MCRf154pFfWjBDL4zh0WGBW8FyJ/9NxdYDJsOIznHBPBHCXnQOlrpdInIptuoLnT52V7MOq9XIiuqLlAH8Tr0GzNSJIJfwwhwJuGxgYi2DPDH8PRmkbHbvY/mN2WBxNFfHBSfCQGZJt6kHtEE+xIi0QQXME0h36V/8sOsmQDzArNFl2Um5GCXodtIjZI6Ga3nQhHDRRiQKJz0V/9OsJF4GteGw94xt/l6PkCT0o4D1cmPIELrBTzaEMnXYKRQEOPnvViyV8eJF2RBnaqNk+FqI+hmGtXXUKAdXZWAzyS66I="
+
+jobs:
+  include:
+  - stage: deploy
+    name: deploy-tag
+    if: repo = cheshirekow/cmake_format
+        AND NOT tag IS present
+        AND branch =~ ^((master)|(staging))$
+    python: 3.6
+    script:
+    - make push-github-pseudorelease-tag
+  - name: deploy-artifacts
+    python: 3.6
+    if: repo = cheshirekow/cmake_format
+        AND tag IS present
+        AND (tag =~ ^pseudo-((master)|(staging))$
+             OR tag =~ ^v\d+\.\d+\.\d+)
+    script:
+    - make push-github-release
+    - make push-rtd-repo

CMakeLists.txt

@@ -3,11 +3,13 @@ project(cheshirekow)
 enable_testing()
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 
+include(${CMAKE_SOURCE_DIR}/cmake/environment.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/codestyle.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/ctest_helpers.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/doctools.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/pkgconfig.cmake)
 
+detect_buildenv()
 find_package(Threads REQUIRED)
 pkg_find(
   PKG eigen3
@@ -40,15 +42,18 @@ set(CXX_STANDARD
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=${CXX_STANDARD}")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
 
+add_custom_target(doc)
 add_custom_target(format)
+add_custom_target(gen)
 add_custom_target(lint)
-
-add_custom_target(doc)
+add_custom_target(test-deps)
 
 set(ENV{PYTHONPATH} ${CMAKE_SOURCE_DIR})
 set(CTEST_ENVIRONMENT "PYTHONPATH=${CMAKE_SOURCE_DIR}")
 
-set_property(GLOBAL PROPERTY gloal_doc_files "")
+# Will be populated by subdirectory listfiles with the dependencies of the
+# master sphinx build
+set_property(GLOBAL PROPERTY global_doc_files "")
 
 # NOTE(josh): search through the list of child directories and add any that
 # actually contain a listfile. While globs are evil, this is necessary for
@@ -87,8 +92,37 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/doxy.config.in)
   add_dependencies(doc doxygen)
 endif()
 
+# Validate sparse or update sparse export
+add_custom_target(
+  update-export
+  COMMAND python -Bm tangent.tooling.export_sparse_subrepo update
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  COMMENT "Updataing sparse export")
+
+add_test(
+  NAME verify-export
+  COMMAND python -Bm tangent.tooling.export_sparse_subrepo update
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
 add_custom_target(
   better-test
+  DEPENDS test-deps
   COMMAND ctest --force-new-ctest-process --output-on-failure
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
   COMMENT "Execute ctest")
+
+add_custom_target(
+  buildbot-test
+  DEPENDS test-deps
+  COMMAND ctest
+    # cmake-format: off
+    --label-exclude CI_DISABLED
+    --force-new-ctest-process
+    --output-on-failure
+    --output-log ${CMAKE_BINARY_DIR}/ctestlog.txt
+    # cmake-format: on
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+)
+
+add_custom_target(pre-push)
+add_dependencies(pre-push better-test doc format gen lint)

Makefile

@@ -27,6 +27,6 @@ test: all
 	@echo "Bootstraping build system"
 	mkdir -p .build
 	touch .build/CMakeCache.txt
-	cd .build && env cmake -DIS_TRAVIS_CI=True ../
+	cd .build && env cmake ../
 
 PHONY: all doc test

cmake/codestyle.cmake

@@ -58,13 +58,13 @@ function(format_and_lint module)
         string(TOLOWER ${arg} typename_)
         set(active_list_ ${typename_}_files_)
       else()
-        if(arg MATCHES ".*\.cmake" OR arg MATCHES ".*CMakeLists.txt")
+        if(arg MATCHES ".*\.cmake$" OR arg MATCHES ".*CMakeLists.txt")
           list(APPEND cmake_files_ ${arg})
-        elseif(arg MATCHES ".*\.py")
+        elseif(arg MATCHES ".*\.py$")
           list(APPEND py_files_ ${arg})
-        elseif(arg MATCHES ".*\.(c|cc|h)")
+        elseif(arg MATCHES ".*\.(c|cc|h|hpp)$")
           list(APPEND cc_files_ ${arg})
-        elseif(arg MATCHES ".*\.js(\.tpl)?")
+        elseif(arg MATCHES ".*\.js(\.tpl)?$")
           list(APPEND js_files_ ${arg})
         else()
           list(APPEND unknown_files_ ${arg})

cmake/environment.cmake

@@ -0,0 +1,17 @@
+function(detect_buildenv)
+  if(DEFINED ENV{CI}
+     AND DEFINED ENV{TRAVIS}
+     AND "$ENV{CI}" STREQUAL "true"
+     AND "$ENV{TRAVIS}" STREQUAL "true")
+    set(IS_TRAVIS_CI TRUE PARENT_SCOPE)
+  else()
+    set(IS_TRAVIS_CI FALSE PARENT_SCOPE)
+  endif()
+
+  if(DEFINED ENV{TRAVIS_PULL_REQUEST}
+     AND "$ENV{TRAVIS_PULL_REQUEST}" STREQUAL "true")
+    set(IS_PULL_REQUEST TRUE PARENT_SCOPE)
+  else()
+    set(IS_PULL_REQUEST FALSE PARENT_SCOPE)
+  endif()
+endfunction()

cmake_format/BUILD

@@ -28,6 +28,8 @@ py_library(
         "parse/simple_nodes.py",
         "parse/statement_node.py",
         "parse/util.py",
+        "parse/properties.py",
+        "parse/variables.py",
         "parse_funs/__init__.py",
         "parse_funs/add_executable.py",
         "parse_funs/add_library.py",

cmake_format/CMakeLists.txt

@@ -23,10 +23,10 @@ format_and_lint(
   contrib/validate_pullrequest.py
   ctest_to.py
   doc/__init__.py
+  doc/bits/dump-example.cmake
   doc/CMakeLists.txt
   doc/conf.py
-  doc/docsources_test.py
-  doc/gendoc_sources.py
+  doc/gendoc.py
   formatter.py
   genparsers.py
   invocation_tests.py
@@ -41,9 +41,11 @@ format_and_lint(
   parse/body_nodes.py
   parse/common.py
   parse/printer.py
+  parse/properties.py
   parse/simple_nodes.py
   parse/statement_node.py
   parse/util.py
+  parse/variables.py
   parse_funs/__init__.py
   parse_funs/add_executable.py
   parse_funs/add_library.py
@@ -72,8 +74,12 @@ format_and_lint(
   tests.py
   tools/__init__.py
   tools/bump_version.py
+  tools/create_pseudorelease_tag.py
+  tools/gen_rtd_requirements.py
   tools/generate_missing_parsers.py
+  tools/get_release_notes.py
   tools/parse_cmake_help.py
+  tools/push_github_release.py
   tools/usage_lexer.py
   tools/usage_parser.py)
 
@@ -96,15 +102,114 @@ if(NOT IS_TRAVIS_CI)
   endforeach()
 endif()
 
-add_subdirectory(command_tests)
-add_subdirectory(contrib)
-add_subdirectory(doc)
-add_subdirectory(test)
-
 add_test(
   NAME cmake_format-check-lint-manifest
   COMMAND
     python -Bm cmake.validate_lint_manifest #
     ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} #
-    --exclude "command_tests/" "test/.*" ".*\\.jinja.py$"
+    --exclude "command_tests/" "doc/stage" "test/.*" ".*\\.jinja.py$"
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+set(_genfiles)
+
+add_custom_command(
+  OUTPUT cmake_format/parse/variables.py
+  COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile
+          cmake_format/parse/variables.py variables
+  DEPENDS tools/parse_cmake_help.py tools/variables.jinja.py
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  COMMENT "Generating variables.py")
+list(APPEND _genfiles cmake_format/parse/variables.py)
+
+
+add_custom_command(
+  OUTPUT cmake_format/parse/properties.py
+  COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile
+          cmake_format/parse/properties.py properties
+  DEPENDS tools/parse_cmake_help.py tools/properties.jinja.py
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  COMMENT "Generating properties.py")
+list(APPEND _genfiles cmake_format/parse/properties.py)
+
+add_custom_target(
+  gen-cmake_format
+  DEPENDS ${_genfiles})
+add_dependencies(gen gen-cmake_format)
+
+# NOTE(josh): this is just here to induce a dependency on the configure step.
+# If we change the version number in __init__.py we need to re-run cmake so we
+# can get out the version number and create rules for the distribution files.
+configure_file(__init__.py init.stamp COPYONLY)
+
+execute_process(
+  COMMAND python -Bm cmake_format --version
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  RESULT_VARIABLE _resultcode
+  OUTPUT_VARIABLE _version
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(NOT _resultcode EQUAL 0)
+  message(
+    FATAL_ERROR "Failed to get cmake-format version number from __init__.py")
+endif()
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.egg
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/.egg
   WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+set(_distdir ${CMAKE_CURRENT_BINARY_DIR}/dist)
+add_custom_command(
+  OUTPUT
+    ${_distdir}/cmake_format-${_version}.tar.gz
+    ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+  COMMAND
+    # cmake-format: off
+    python cmake_format/pypi/setup.py
+    build --build-base ${CMAKE_CURRENT_BINARY_DIR}
+    egg_info --egg-base ${CMAKE_CURRENT_BINARY_DIR}/.egg
+    bdist_wheel
+      --bdist-dir ${CMAKE_CURRENT_BINARY_DIR}/bdist
+      --dist-dir ${_distdir}
+    sdist --dist-dir ${_distdir}
+    # cmake-format: on
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/.egg
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
+  COMMAND python -Bm cmake_format.tools.get_release_notes
+          cmake_format/doc/release_notes.rst \$\${TRAVIS_TAG}
+          -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
+  DEPENDS doc/release_notes.rst
+          tools/get_release_notes.py
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+  COMMAND pandoc -s
+          -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+          ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+add_custom_target(
+  push-github-release
+  COMMAND
+    # cmake-format: off
+    python -Bm cmake_format.tools.push_github_release
+      --message ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+      \$\${TRAVIS_TAG}
+      ${_distdir}/cmake_format-${_version}.tar.gz
+      ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+    # cmake-format: on
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md
+    ${_distdir}/cmake_format-${_version}.tar.gz
+    ${_distdir}/cmake_format-${_version}-py3-none-any.whl
+  COMMENT "Uploading release artifacts"
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+add_subdirectory(command_tests)
+add_subdirectory(contrib)
+add_subdirectory(doc)
+add_subdirectory(test)

cmake_format/__init__.py

@@ -3,4 +3,4 @@
 """
 from __future__ import unicode_literals
 
-VERSION = '0.6.7'
+VERSION = '0.6.8.dev0'