diff --git a/.gitignore b/.gitignore index 33b7a46..9a5fc2e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,10 @@ *.o *.so -# unified repo tracking +# sparse-export stuff .gitu +.sparse-export +tangent/tooling/sparse-exports # qt creator *.qtc.* diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..3f3c4db --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,20 @@ +# .readthedocs.yml +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 +sphinx: + configuration: conf.py + +build: + image: latest + +formats: all +python: + version: 3.7 + install: + - requirements: requirements.txt + - method: pip + path: . + extra_requirements: + - YAML + - html-gen diff --git a/.sparse-export b/.sparse-export new file mode 100644 index 0000000..1890643 --- /dev/null +++ b/.sparse-export @@ -0,0 +1 @@ +tangent/tooling/sparse-exports/cmake_format diff --git a/.travis.yml b/.travis.yml index fd360fa..b0da2b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,42 @@ dist: xenial language: python python: - - "2.7" - - "3.5" - - "3.6" +- '2.7' +- '3.5' +- '3.6' + +before_install: +- sudo apt-get update +- sudo apt-get -y install pandoc install: - - pip install -r requirements.txt +- pip install -r requirements.txt script: - - make better-test - - make lint +- make better-test +- make lint + +env: + global: + # GITHUB_ACCESS_TOKEN= + secure: "e62y3i4FG7KsJ/I8aVKoPk5CG0L7/TW1K4M+BLt5zULsKGyRw6b53I97p5Tc1zcAHqcocFeYL/eC3AucG3VNvyv6LIZuv5Y0upF/MG++tvkfV9Iyok76mP70DXR/ikAJDMgZSdEL/6uTTJ4XfBuH/VvCMMBvj+5ORSL65KIfqXcexj2JWaNtEEfwiVSkDvQZYYbHLRjQdAXzzI7g1xh+8mCmuFAvtn6U1cIZpdQv5u1DNIry8PoQ2asHr+vaCN9iVXvkWcUjoXpJTp96Hd8eO40ts3v3Ja1yJ7GzH09zjbKJS63dcB0LfV5uLjybJCkQ03102ua0V+E5OzPn5VYeIRSRWmdo35e9xZjqqbtNdHqNayOOpew/ik+wf2Nmzva5MCRf154pFfWjBDL4zh0WGBW8FyJ/9NxdYDJsOIznHBPBHCXnQOlrpdInIptuoLnT52V7MOq9XIiuqLlAH8Tr0GzNSJIJfwwhwJuGxgYi2DPDH8PRmkbHbvY/mN2WBxNFfHBSfCQGZJt6kHtEE+xIi0QQXME0h36V/8sOsmQDzArNFl2Um5GCXodtIjZI6Ga3nQhHDRRiQKJz0V/9OsJF4GteGw94xt/l6PkCT0o4D1cmPIELrBTzaEMnXYKRQEOPnvViyV8eJF2RBnaqNk+FqI+hmGtXXUKAdXZWAzyS66I=" + +jobs: + include: + - stage: deploy + name: deploy-tag + if: repo = cheshirekow/cmake_format + AND NOT tag IS present + AND branch =~ ^((master)|(staging))$ + python: 3.6 + script: + - make push-github-pseudorelease-tag + - name: deploy-artifacts + python: 3.6 + if: repo = cheshirekow/cmake_format + AND tag IS present + AND (tag =~ ^pseudo-((master)|(staging))$ + OR tag =~ ^v\d+\.\d+\.\d+) + script: + - make push-github-release + - make push-rtd-repo diff --git a/CMakeLists.txt b/CMakeLists.txt index 58be4a4..7f8cbf5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,13 @@ project(cheshirekow) enable_testing() set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +include(${CMAKE_SOURCE_DIR}/cmake/environment.cmake) include(${CMAKE_SOURCE_DIR}/cmake/codestyle.cmake) include(${CMAKE_SOURCE_DIR}/cmake/ctest_helpers.cmake) include(${CMAKE_SOURCE_DIR}/cmake/doctools.cmake) include(${CMAKE_SOURCE_DIR}/cmake/pkgconfig.cmake) +detect_buildenv() find_package(Threads REQUIRED) pkg_find( PKG eigen3 @@ -40,15 +42,18 @@ set(CXX_STANDARD set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=${CXX_STANDARD}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") +add_custom_target(doc) add_custom_target(format) +add_custom_target(gen) add_custom_target(lint) - -add_custom_target(doc) +add_custom_target(test-deps) set(ENV{PYTHONPATH} ${CMAKE_SOURCE_DIR}) set(CTEST_ENVIRONMENT "PYTHONPATH=${CMAKE_SOURCE_DIR}") -set_property(GLOBAL PROPERTY gloal_doc_files "") +# Will be populated by subdirectory listfiles with the dependencies of the +# master sphinx build +set_property(GLOBAL PROPERTY global_doc_files "") # NOTE(josh): search through the list of child directories and add any that # actually contain a listfile. While globs are evil, this is necessary for @@ -87,8 +92,37 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/doxy.config.in) add_dependencies(doc doxygen) endif() +# Validate sparse or update sparse export +add_custom_target( + update-export + COMMAND python -Bm tangent.tooling.export_sparse_subrepo update + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Updataing sparse export") + +add_test( + NAME verify-export + COMMAND python -Bm tangent.tooling.export_sparse_subrepo update + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + add_custom_target( better-test + DEPENDS test-deps COMMAND ctest --force-new-ctest-process --output-on-failure WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Execute ctest") + +add_custom_target( + buildbot-test + DEPENDS test-deps + COMMAND ctest + # cmake-format: off + --label-exclude CI_DISABLED + --force-new-ctest-process + --output-on-failure + --output-log ${CMAKE_BINARY_DIR}/ctestlog.txt + # cmake-format: on + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) + +add_custom_target(pre-push) +add_dependencies(pre-push better-test doc format gen lint) diff --git a/Makefile b/Makefile index 696be0e..a16ffb4 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,6 @@ test: all @echo "Bootstraping build system" mkdir -p .build touch .build/CMakeCache.txt - cd .build && env cmake -DIS_TRAVIS_CI=True ../ + cd .build && env cmake ../ PHONY: all doc test diff --git a/cmake/codestyle.cmake b/cmake/codestyle.cmake index d2449e0..d014c51 100644 --- a/cmake/codestyle.cmake +++ b/cmake/codestyle.cmake @@ -58,13 +58,13 @@ function(format_and_lint module) string(TOLOWER ${arg} typename_) set(active_list_ ${typename_}_files_) else() - if(arg MATCHES ".*\.cmake" OR arg MATCHES ".*CMakeLists.txt") + if(arg MATCHES ".*\.cmake$" OR arg MATCHES ".*CMakeLists.txt") list(APPEND cmake_files_ ${arg}) - elseif(arg MATCHES ".*\.py") + elseif(arg MATCHES ".*\.py$") list(APPEND py_files_ ${arg}) - elseif(arg MATCHES ".*\.(c|cc|h)") + elseif(arg MATCHES ".*\.(c|cc|h|hpp)$") list(APPEND cc_files_ ${arg}) - elseif(arg MATCHES ".*\.js(\.tpl)?") + elseif(arg MATCHES ".*\.js(\.tpl)?$") list(APPEND js_files_ ${arg}) else() list(APPEND unknown_files_ ${arg}) diff --git a/cmake/environment.cmake b/cmake/environment.cmake new file mode 100644 index 0000000..eb017bf --- /dev/null +++ b/cmake/environment.cmake @@ -0,0 +1,17 @@ +function(detect_buildenv) + if(DEFINED ENV{CI} + AND DEFINED ENV{TRAVIS} + AND "$ENV{CI}" STREQUAL "true" + AND "$ENV{TRAVIS}" STREQUAL "true") + set(IS_TRAVIS_CI TRUE PARENT_SCOPE) + else() + set(IS_TRAVIS_CI FALSE PARENT_SCOPE) + endif() + + if(DEFINED ENV{TRAVIS_PULL_REQUEST} + AND "$ENV{TRAVIS_PULL_REQUEST}" STREQUAL "true") + set(IS_PULL_REQUEST TRUE PARENT_SCOPE) + else() + set(IS_PULL_REQUEST FALSE PARENT_SCOPE) + endif() +endfunction() diff --git a/cmake_format/BUILD b/cmake_format/BUILD index e60453e..e53d03e 100644 --- a/cmake_format/BUILD +++ b/cmake_format/BUILD @@ -28,6 +28,8 @@ py_library( "parse/simple_nodes.py", "parse/statement_node.py", "parse/util.py", + "parse/properties.py", + "parse/variables.py", "parse_funs/__init__.py", "parse_funs/add_executable.py", "parse_funs/add_library.py", diff --git a/cmake_format/CMakeLists.txt b/cmake_format/CMakeLists.txt index 891490c..8c065ad 100644 --- a/cmake_format/CMakeLists.txt +++ b/cmake_format/CMakeLists.txt @@ -23,10 +23,10 @@ format_and_lint( contrib/validate_pullrequest.py ctest_to.py doc/__init__.py + doc/bits/dump-example.cmake doc/CMakeLists.txt doc/conf.py - doc/docsources_test.py - doc/gendoc_sources.py + doc/gendoc.py formatter.py genparsers.py invocation_tests.py @@ -41,9 +41,11 @@ format_and_lint( parse/body_nodes.py parse/common.py parse/printer.py + parse/properties.py parse/simple_nodes.py parse/statement_node.py parse/util.py + parse/variables.py parse_funs/__init__.py parse_funs/add_executable.py parse_funs/add_library.py @@ -72,8 +74,12 @@ format_and_lint( tests.py tools/__init__.py tools/bump_version.py + tools/create_pseudorelease_tag.py + tools/gen_rtd_requirements.py tools/generate_missing_parsers.py + tools/get_release_notes.py tools/parse_cmake_help.py + tools/push_github_release.py tools/usage_lexer.py tools/usage_parser.py) @@ -96,15 +102,114 @@ if(NOT IS_TRAVIS_CI) endforeach() endif() -add_subdirectory(command_tests) -add_subdirectory(contrib) -add_subdirectory(doc) -add_subdirectory(test) - add_test( NAME cmake_format-check-lint-manifest COMMAND python -Bm cmake.validate_lint_manifest # ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} # - --exclude "command_tests/" "test/.*" ".*\\.jinja.py$" + --exclude "command_tests/" "doc/stage" "test/.*" ".*\\.jinja.py$" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +set(_genfiles) + +add_custom_command( + OUTPUT cmake_format/parse/variables.py + COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile + cmake_format/parse/variables.py variables + DEPENDS tools/parse_cmake_help.py tools/variables.jinja.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Generating variables.py") +list(APPEND _genfiles cmake_format/parse/variables.py) + + +add_custom_command( + OUTPUT cmake_format/parse/properties.py + COMMAND python -Bm cmake_format.tools.parse_cmake_help --outfile + cmake_format/parse/properties.py properties + DEPENDS tools/parse_cmake_help.py tools/properties.jinja.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Generating properties.py") +list(APPEND _genfiles cmake_format/parse/properties.py) + +add_custom_target( + gen-cmake_format + DEPENDS ${_genfiles}) +add_dependencies(gen gen-cmake_format) + +# NOTE(josh): this is just here to induce a dependency on the configure step. +# If we change the version number in __init__.py we need to re-run cmake so we +# can get out the version number and create rules for the distribution files. +configure_file(__init__.py init.stamp COPYONLY) + +execute_process( + COMMAND python -Bm cmake_format --version + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE _resultcode + OUTPUT_VARIABLE _version + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT _resultcode EQUAL 0) + message( + FATAL_ERROR "Failed to get cmake-format version number from __init__.py") +endif() + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/.egg + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/.egg WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +set(_distdir ${CMAKE_CURRENT_BINARY_DIR}/dist) +add_custom_command( + OUTPUT + ${_distdir}/cmake_format-${_version}.tar.gz + ${_distdir}/cmake_format-${_version}-py3-none-any.whl + COMMAND + # cmake-format: off + python cmake_format/pypi/setup.py + build --build-base ${CMAKE_CURRENT_BINARY_DIR} + egg_info --egg-base ${CMAKE_CURRENT_BINARY_DIR}/.egg + bdist_wheel + --bdist-dir ${CMAKE_CURRENT_BINARY_DIR}/bdist + --dist-dir ${_distdir} + sdist --dist-dir ${_distdir} + # cmake-format: on + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/.egg + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst + COMMAND python -Bm cmake_format.tools.get_release_notes + cmake_format/doc/release_notes.rst \$\${TRAVIS_TAG} + -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst + DEPENDS doc/release_notes.rst + tools/get_release_notes.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + COMMAND pandoc -s + -o ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.rst + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +add_custom_target( + push-github-release + COMMAND + # cmake-format: off + python -Bm cmake_format.tools.push_github_release + --message ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + \$\${TRAVIS_TAG} + ${_distdir}/cmake_format-${_version}.tar.gz + ${_distdir}/cmake_format-${_version}-py3-none-any.whl + # cmake-format: on + DEPENDS + ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + ${_distdir}/cmake_format-${_version}.tar.gz + ${_distdir}/cmake_format-${_version}-py3-none-any.whl + COMMENT "Uploading release artifacts" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +add_subdirectory(command_tests) +add_subdirectory(contrib) +add_subdirectory(doc) +add_subdirectory(test) diff --git a/cmake_format/__init__.py b/cmake_format/__init__.py index 62aea45..652529b 100644 --- a/cmake_format/__init__.py +++ b/cmake_format/__init__.py @@ -3,4 +3,4 @@ """ from __future__ import unicode_literals -VERSION = '0.6.7' +VERSION = '0.6.8.dev0' diff --git a/cmake_format/__main__.py b/cmake_format/__main__.py index f30fb09..afa5e4b 100644 --- a/cmake_format/__main__.py +++ b/cmake_format/__main__.py @@ -116,7 +116,7 @@ def process_file(config, infile_content, dump=None): for token in tokens: outfile.write("{}\n".format(token)) return outfile.getvalue(), True - config.first_token = lexer.get_first_non_whitespace_token(tokens) + first_token = lexer.get_first_non_whitespace_token(tokens) parse_db = parse_funs.get_parse_db() parse_db.update(parse_funs.get_legacy_parse(config.parse.fn_spec).kwargs) ctx = parse.ParseContext(parse_db, config=config) @@ -128,7 +128,7 @@ def process_file(config, infile_content, dump=None): dump_markup([parse_tree], config, outfile) return outfile.getvalue(), True - box_tree = formatter.layout_tree(parse_tree, config) + box_tree = formatter.layout_tree(parse_tree, config, first_token=first_token) if dump == "layout": formatter.dump_tree([box_tree], outfile) return outfile.getvalue(), True @@ -229,14 +229,32 @@ def get_one_config_dict(configfile_path): """ if configfile_path.endswith('.json'): with io.open(configfile_path, 'r', encoding='utf-8') as config_file: - return json.load(config_file) + try: + return json.load(config_file) + except ValueError as ex: + message = ( + "Failed to parse json config file {}: {}" + .format(configfile_path, ex)) + raise common.UserError(message) if configfile_path.endswith('.yaml'): with io.open(configfile_path, 'r', encoding='utf-8') as config_file: - return load_yaml(config_file) + try: + return load_yaml(config_file) + except ValueError as ex: + message = ( + "Failed to parse yaml config file {}: {}" + .format(configfile_path, ex)) + raise common.UserError(message) if configfile_path.endswith('.py'): - return exec_pyconfig(configfile_path) + try: + return exec_pyconfig(configfile_path) + except Exception as ex: + message = ( + "Failed to parse python config file {}: {}" + .format(configfile_path, ex)) + raise common.UserError(message) return try_get_configdict(configfile_path) @@ -248,6 +266,7 @@ def get_configdict(configfile_paths): configfile_path = include_queue.pop(0) configfile_path = os.path.expanduser(configfile_path) increment_dict = get_one_config_dict(configfile_path) + for include_path in increment_dict.pop("include", []): if not os.path.isabs(include_path): include_path = os.path.join( diff --git a/cmake_format/annotate.py b/cmake_format/annotate.py index e7ae7ec..a752670 100644 --- a/cmake_format/annotate.py +++ b/cmake_format/annotate.py @@ -26,6 +26,21 @@ from cmake_format import render +EMBED_TPL = """ + + + +""" + + def annotate_file(config, infile, outfile, outfmt=None): """ Parse the input cmake file, re-format it, and print to the output file. @@ -37,7 +52,6 @@ def annotate_file(config, infile, outfile, outfmt=None): config = config.clone() config.format.set_line_ending(detected) tokens = lexer.tokenize(infile_content) - config.first_token = lexer.get_first_non_whitespace_token(tokens) parse_db = parse_funs.get_parse_db() parse_db.update(parse_funs.get_legacy_parse(config.parse.fn_spec).kwargs) ctx = parse.ParseContext(parse_db) @@ -51,6 +65,18 @@ def annotate_file(config, infile, outfile, outfmt=None): html_content = render.get_html(parse_tree, fullpage=False) outfile.write(html_content) return + if outfmt == "iframe": + html_content = render.get_html(parse_tree, fullpage=True) + wrap_lines = EMBED_TPL.split("\n") + for line in wrap_lines[:2]: + outfile.write(line) + outfile.write("\n") + outfile.write(html_content) + for line in wrap_lines[3:]: + outfile.write(line) + outfile.write("\n") + return + raise ValueError("Invalid output format: {}".format(outfmt)) @@ -70,7 +96,7 @@ def setup_argparser(arg_parser): arg_parser.add_argument('-v', '--version', action='version', version=cmake_format.VERSION) arg_parser.add_argument( - "-f", "--format", choices=["page", "stub"], default="stub", + "-f", "--format", choices=["page", "stub", "iframe"], default="stub", help="whether to output a standalone `page` complete with " "tags, or just the annotated content") diff --git a/cmake_format/command_tests/CMakeLists.txt b/cmake_format/command_tests/CMakeLists.txt index d5fe454..38aa2ff 100644 --- a/cmake_format/command_tests/CMakeLists.txt +++ b/cmake_format/command_tests/CMakeLists.txt @@ -1,19 +1,17 @@ set(MODPREFIX cmake_format.command_tests) -set( - _testnames - TestAddCustomCommand - TestAddExecutableCommand - TestAddLibraryCommand - TestConditional - TestExport - TestFile - TestForeach - TestInstall - TestMiscFormatting - TestSetTargetProperties - TestSet -) +set(_testnames + TestAddCustomCommand + TestAddExecutableCommand + TestAddLibraryCommand + TestConditional + TestExport + TestFile + TestForeach + TestInstall + TestMiscFormatting + TestSetTargetProperties + TestSet) foreach(_testname ${_testnames}) add_test( diff --git a/cmake_format/command_tests/__init__.py b/cmake_format/command_tests/__init__.py index 1c70f90..b7ef772 100644 --- a/cmake_format/command_tests/__init__.py +++ b/cmake_format/command_tests/__init__.py @@ -481,6 +481,13 @@ class TestAddCustomCommand(TestBase): kExpectNumSidecarTests = 4 +class TestComment(TestBase): + """ + Test various examples involving comments + """ + kExpectNumSidecarTests = 3 + + class TestConditional(TestBase): """ Test various examples of commands that take conditional statements diff --git a/cmake_format/command_tests/__main__.py b/cmake_format/command_tests/__main__.py index 01634be..f851f39 100644 --- a/cmake_format/command_tests/__main__.py +++ b/cmake_format/command_tests/__main__.py @@ -4,6 +4,7 @@ from cmake_format.command_tests import ( TestAddCustomCommand, TestConditional, + TestComment, TestCustomCommand, TestExport, TestExternalProject, @@ -11,7 +12,7 @@ TestForeach, TestInstall, TestSetTargetProperties, - TestSet,) + TestSet) from cmake_format.command_tests.add_executable_tests \ import TestAddExecutableCommand diff --git a/cmake_format/command_tests/comment_tests.cmake b/cmake_format/command_tests/comment_tests.cmake new file mode 100644 index 0000000..c3d0aa5 --- /dev/null +++ b/cmake_format/command_tests/comment_tests.cmake @@ -0,0 +1,27 @@ +# test: format_off_code +# No, I really want this to look ugly +# cmake-format: off +add_library(a b.cc + c.cc d.cc + e.cc) +# cmake-format: on + +# test: format_off_nested +add_custom_target( + push-github-release + COMMAND + # cmake-format: off + python -Bm cmake_format.tools.push_github_release + --message ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + $ENV{TRAVIS_TAG} + ${_distdir}/cmake-format-${_version}.tar.gz + ${_distdir}/cmake-format-${_version}-py3-none-any.whl + # cmake-format: on + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/release_notes-${_version}.md + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +# test: multiline comment in statement +add_custom_target( + my-target COMMAND foo bar baz # This comment is a trailing comment + # But this comment is a line comment +) diff --git a/cmake_format/command_tests/misc_tests.cmake b/cmake_format/command_tests/misc_tests.cmake index f9623f0..e71ba72 100644 --- a/cmake_format/command_tests/misc_tests.cmake +++ b/cmake_format/command_tests/misc_tests.cmake @@ -67,14 +67,6 @@ foo(some_arg some_arg " This string is on multiple lines ") -# test: format_off_code -# No, I really want this to look ugly -# cmake-format: off -add_library(a b.cc - c.cc d.cc - e.cc) -# cmake-format: on - # test: multiline_statment_comment_idempotent set(HELLO hello world!) # TODO(josh): fix this bad code with some change that # takes mutiple lines to explain diff --git a/cmake_format/command_tests/misc_tests.py b/cmake_format/command_tests/misc_tests.py index 213b1d0..83c0912 100644 --- a/cmake_format/command_tests/misc_tests.py +++ b/cmake_format/command_tests/misc_tests.py @@ -15,7 +15,7 @@ class TestMiscFormatting(TestBase): """ Ensure that various inputs format the way we want them to """ - kExpectNumSidecarTests = 82 + kExpectNumSidecarTests = 81 def test_config_hashruler_minlength(self): diff --git a/cmake_format/configuration.py b/cmake_format/configuration.py index b088a33..c6334d2 100644 --- a/cmake_format/configuration.py +++ b/cmake_format/configuration.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import logging +import re from cmake_format import commands from cmake_format import markup @@ -183,8 +184,6 @@ class LinterConfig(ConfigObject): 'DEPENDS': '*' } } - - } @@ -197,39 +196,47 @@ class FormattingConfig(ConfigObject): 80, "How wide to allow formatted cmake files" ) + tab_size = FieldDescriptor( 2, "How many spaces to tab for indent" ) + max_subgroups_hwrap = FieldDescriptor( 2, "If an argument group contains more than this many sub-groups " "(parg or kwarg groups) then force it to a vertical layout. " ) + max_pargs_hwrap = FieldDescriptor( 6, "If a positional argument group contains more than this many" " arguments, then force it to a vertical layout. " ) + max_rows_cmdline = FieldDescriptor( 2, "If a cmdline positional group consumes more than this many lines" " without nesting, then invalidate the layout (and nest)" ) + separate_ctrl_name_with_space = FieldDescriptor( False, "If true, separate flow control names from their parentheses with a" " space" ) + separate_fn_name_with_space = FieldDescriptor( False, "If true, separate function names from parentheses with a space" ) + dangle_parens = FieldDescriptor( False, "If a statement is wrapped to more than one line, than dangle the" " closing parenthesis on its own line." ) + dangle_align = FieldDescriptor( "prefix", "If the trailing parenthesis must be 'dangled' on its on line, then" @@ -238,51 +245,61 @@ class FormattingConfig(ConfigObject): " level, `child`: align to the column of the arguments", ["prefix", "prefix-indent", "child", "off"], ) + min_prefix_chars = FieldDescriptor( 4, "If the statement spelling length (including space and parenthesis)" " is smaller than this amount, then force reject nested layouts." ) + max_prefix_chars = FieldDescriptor( 10, "If the statement spelling length (including space and parenthesis)" " is larger than the tab width by more than this amount, then" " force reject un-nested layouts." ) + max_lines_hwrap = FieldDescriptor( 2, "If a candidate layout is wrapped horizontally but it exceeds this" " many lines, then reject the layout." ) + line_ending = FieldDescriptor( "unix", "What style line endings to use in the output.", ['windows', 'unix', 'auto'] ) + command_case = FieldDescriptor( "canonical", "Format command names consistently as 'lower' or 'upper' case", ['lower', 'upper', 'canonical', 'unchanged'] ) + keyword_case = FieldDescriptor( "unchanged", "Format keywords consistently as 'lower' or 'upper' case", ['lower', 'upper', 'unchanged'] ) + always_wrap = FieldDescriptor( [], "A list of command names which should always be wrapped" ) + enable_sort = FieldDescriptor( True, "If true, the argument lists which are known to be sortable will be " "sorted lexicographicall" ) + autosort = FieldDescriptor( False, "If true, the parsers may infer whether or not an argument list is" " sortable (without annotation)." ) + require_valid_layout = FieldDescriptor( False, "By default, if cmake-format cannot successfully fit everything into" @@ -322,6 +339,15 @@ def _update_derived(self): }[self.line_ending] +BUILTIN_VARTAGS = [ + (".*_COMMAND", ["cmdline"]) +] + +BUILTIN_PROPTAGS = [ + (".*_DIRECTORIES", ["file-list"]) +] + + class ParseConfig(ConfigObject): """Options affecting listfile parsing""" @@ -332,13 +358,25 @@ class ParseConfig(ConfigObject): "Specify structure for custom cmake functions" ) + vartags = FieldDescriptor([], "Specify variable tags.") + proptags = FieldDescriptor([], "Specify property tags.") + def _update_derived(self): if self.additional_commands is not None: for command_name, spec in self.additional_commands.items(): self.fn_spec.add(command_name, **spec) + self.vartags_ = [ + (re.compile(pattern, re.IGNORECASE), tags) for pattern, tags in + BUILTIN_VARTAGS + self.vartags] + self.proptags_ = [ + (re.compile(pattern, re.IGNORECASE), tags) for pattern, tags in + BUILTIN_PROPTAGS + self.proptags] + def __init__(self, **kwargs): # pylint: disable=W0613 self.fn_spec = commands.get_fn_spec() + self.vartags_ = [] + self.proptags_ = [] super(ParseConfig, self).__init__(**kwargs) @@ -382,14 +420,6 @@ class Configuration(ConfigObject): encode = SubtreeDescriptor(EncodingConfig) misc = SubtreeDescriptor(MiscConfig) - def __init__(self, **kwargs): # pylint: disable=W0613 - super(Configuration, self).__init__(**kwargs) - - # TODO(josh): this does not belong here! We need some global state for - # formatting and the only thing we have accessible through the whole format - # stack is this config object... so I'm abusing it by adding this field here - self.first_token = None - def resolve_for_command(self, command_name, config_key, default_value=None): """ Check for a per-command value or override of the given configuration key diff --git a/cmake_format/doc/.readthedocs.yml b/cmake_format/doc/.readthedocs.yml new file mode 100644 index 0000000..192a878 --- /dev/null +++ b/cmake_format/doc/.readthedocs.yml @@ -0,0 +1,15 @@ +# .readthedocs.yml +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 +sphinx: + configuration: conf.py + +build: + image: latest + +formats: all +python: + version: 3.7 + install: + - requirements: rtd-requirements.txt diff --git a/cmake_format/doc/BUILD b/cmake_format/doc/BUILD index 4227881..ad72a6a 100644 --- a/cmake_format/doc/BUILD +++ b/cmake_format/doc/BUILD @@ -4,26 +4,7 @@ py_library( name="doc_tests", srcs=[ "__init__.py", - "conf.py", - "docsources_test.py", - "gendoc_sources.py"], + "conf.py"], data=[]) -# -- Python 2 -- - -py_test( - name="docsources_test", - srcs=["docsources_test.py"], - deps=["//cmake_format:cmake_format", ":doc_tests"], - python_version="PY2", - ) - -# -- Python 3 -- - -py_test( - name="docsources_test_py3", - srcs=["docsources_test.py"], - main="docsources_test.py", - deps=["//cmake_format:cmake_format", ":doc_tests"], - python_version="PY3", - ) +# TODO(josh): add genfile rules diff --git a/cmake_format/doc/CMakeLists.txt b/cmake_format/doc/CMakeLists.txt index 23e2bf0..fe6593e 100644 --- a/cmake_format/doc/CMakeLists.txt +++ b/cmake_format/doc/CMakeLists.txt @@ -1,75 +1,382 @@ -add_custom_target( - scanrst-cmake_format_docs +# +# Generated source bits for sphinx documentation +# +set(_stage ${CMAKE_CURRENT_BINARY_DIR}/stage) +set(_genbits) + +add_custom_command( + OUTPUT ${_stage}/bits + COMMAND ${CMAKE_COMMAND} -E make_directory ${_stage}/bits + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/format-usage.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/format-usage.txt + COMMAND python -Bm cmake_format --help > ${_stage}/bits/format-usage.txt + DEPENDS ${_stage}/bits # + ../__main__.py # + ../configuration.py # + ../config_util.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/format-usage.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/annotate-usage.txt + COMMAND python -Bm cmake_format.annotate --help > + ${_stage}/bits/annotate-usage.txt + DEPENDS ${_stage}/bits # + ../__main__.py # + ../annotate.py # + ../configuration.py # + ../config_util.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/annotate-usage.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/ctest-to-usage.txt + COMMAND python -Bm cmake_format.ctest_to --help > + ${_stage}/bits/ctest-to-usage.txt + DEPENDS ${_stage}/bits # + ../__main__.py # + ../ctest_to.py # + ../configuration.py # + ../config_util.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/ctest-to-usage.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/lint-usage.txt + COMMAND python -Bm cmake_lint --help > ${_stage}/bits/lint-usage.txt + DEPENDS ${_stage}/bits # + ../../cmake_lint/__main__.py # + ../__main__.py # + ../ctest_to.py # + ../configuration.py # + ../config_util.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/lint-usage.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/configbits.py + COMMAND python -Bm cmake_format --dump-config > ${_stage}/bits/configbits.py + DEPENDS ${_stage}/bits # + ../__main__.py # + ../configuration.py # + ../config_util.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/configbits.py) + +add_custom_command( + OUTPUT ${_stage}/bits/lint-in.cmake + COMMAND ${CMAKE_COMMAND} -E copy cmake_lint/test/expect_lint.cmake + ${_stage}/bits/lint-in.cmake + DEPENDS ${_stage}/bits # + ${CMAKE_SOURCE_DIR}/cmake_lint/test/expect_lint.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/lint-in.cmake) + +add_custom_command( + OUTPUT ${_stage}/bits/lint-out.cmake + COMMAND python -Bm cmake_lint cmake_lint/test/expect_lint.cmake > + ${_stage}/bits/lint-out.cmake || echo "OK" + DEPENDS ${_stage}/bits # + ../../cmake_lint/__main__.py # + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/lint-out.cmake) + +add_custom_command( + OUTPUT ${_stage}/bits/dump-example-src.cmake + COMMAND ${CMAKE_COMMAND} -E copy cmake_format/doc/bits/dump-example.cmake + ${_stage}/bits/dump-example-src.cmake + DEPENDS ${_stage}/bits # + bits/dump-example.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/dump-example-src.cmake) + +add_custom_command( + OUTPUT ${_stage}/bits/dump-example-lex.txt COMMAND - python -B ${CMAKE_SOURCE_DIR}/doc/find_rst.py --manifest-path - ${CMAKE_CURRENT_BINARY_DIR}/rst_manifest.txt --touch - ${CMAKE_SOURCE_DIR}/cmake_format - DEPENDS ${CMAKE_SOURCE_DIR}/doc/find_rst.py - BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/rst_manifest.txt - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - COMMENT "Scanning RST for cmake_format") + python -Bm cmake_format --dump lex ${_stage}/bits/dump-example-src.cmake > + ${_stage}/bits/dump-example-lex.txt + DEPENDS ${_stage}/bits # + ../../cmake_format/__main__.py # + ${_stage}/bits/dump-example-src.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/dump-example-lex.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/dump-example-parse.txt + COMMAND + python -Bm cmake_format --dump parse + cmake_format/doc/bits/dump-example.cmake > + ${_stage}/bits/dump-example-parse.txt + DEPENDS ${_stage}/bits # + ../../cmake_format/__main__.py # + ${_stage}/bits/dump-example-src.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/dump-example-parse.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/dump-example-layout.txt + COMMAND + python -Bm cmake_format --dump layout + cmake_format/doc/bits/dump-example.cmake > + ${_stage}/bits/dump-example-layout.txt + DEPENDS ${_stage}/bits # + ../../cmake_format/__main__.py # + ${_stage}/bits/dump-example-src.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/dump-example-layout.txt) + +add_custom_command( + OUTPUT ${_stage}/bits/example-in.cmake + COMMAND ${CMAKE_COMMAND} -E copy cmake_format/test/test_in.cmake + ${_stage}/bits/example-in.cmake + DEPENDS ${_stage}/bits # + ../test/test_in.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/example-in.cmake) + +add_custom_command( + OUTPUT ${_stage}/bits/example-out.cmake + COMMAND ${CMAKE_COMMAND} -E copy cmake_format/test/test_out.cmake + ${_stage}/bits/example-out.cmake + DEPENDS ${_stage}/bits # + ../test/test_out.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/example-out.cmake) + +add_custom_command( + OUTPUT ${_stage}/bits/lintimpl-table.rst + COMMAND python -Bm cmake_lint.gendocs table > + ${_stage}/bits/lintimpl-table.rst + DEPENDS ${_stage}/bits # + ../../cmake_lint/lintdb.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/lintimpl-table.rst) + +add_custom_command( + OUTPUT ${_stage}/bits/features.rst + COMMAND tail -n +5 cmake_format/doc/format-features.rst > + ${_stage}/bits/features.rst + DEPENDS ${_stage}/bits # + format-features.rst + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/bits/features.rst) + +add_custom_command( + OUTPUT ${_stage}/lint-implemented.rst + COMMAND python -Bm cmake_lint.gendocs reference > + ${_stage}/lint-implemented.rst + DEPENDS ${_stage}/bits # + ../../cmake_lint/lintdb.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/lint-implemented.rst) + +add_custom_command( + OUTPUT ${_stage}/example_rendered.html + COMMAND python -Bm cmake_format.annotate --format iframe + cmake_format/test/test_out.cmake > ${_stage}/example_rendered.html + DEPENDS ${_stage}/bits # + ../annotate.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +list(APPEND _genbits ${_stage}/example_rendered.html) -if(NOT CMAKE_GENERATOR STREQUAL "Ninja") +# TODO(josh): remove this iteration for just one file. +set(_genfiles) +set(_insource_genfiles README.rst) +foreach(_genfile ${_insource_genfiles}) add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/rst_manifest.txt - DEPENDS scanrst-cmake_format_docs - COMMENT "Stubbing RST scan for cmake_format_doc") -endif() + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${_genfile} + COMMAND + python -Bm cmake_format.doc.gendoc --bits ${_stage}/bits + ${CMAKE_CURRENT_SOURCE_DIR}/${_genfile} -set(gendoc_sources_ - ${CMAKE_CURRENT_SOURCE_DIR}/README.rst - ${CMAKE_CURRENT_SOURCE_DIR}/configuration.rst - ${CMAKE_CURRENT_SOURCE_DIR}/cmake-annotate.rst - ${CMAKE_CURRENT_SOURCE_DIR}/cmake-format.rst - ${CMAKE_CURRENT_SOURCE_DIR}/cmake-lint.rst - ${CMAKE_CURRENT_SOURCE_DIR}/ctest-to.rst - ${CMAKE_CURRENT_SOURCE_DIR}/example.rst - ${CMAKE_CURRENT_SOURCE_DIR}/example_rendered.html - ${CMAKE_CURRENT_SOURCE_DIR}/lint-example.rst - ${CMAKE_CURRENT_SOURCE_DIR}/lint-implemented.rst - ${CMAKE_CURRENT_SOURCE_DIR}/lint-usage.rst - ${CMAKE_CURRENT_SOURCE_DIR}/parse_tree.rst - ${CMAKE_CURRENT_SOURCE_DIR}/usage.rst) + # TODO(josh): not every file depends on every bit, so this is overkill, + # but whatever... it's fine for now. We'll need more complicated code if + # we want to actually declare which bits each file depends on. + DEPENDS ${_genbits} gendoc.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + list(APPEND _genfiles ${CMAKE_CURRENT_SOURCE_DIR}/${_genfile}) -add_custom_target( - gendoc_sources-cmake_format - COMMAND python -Bm cmake_format.doc.gendoc_sources - DEPENDS gendoc_sources.py ${CMAKE_CURRENT_BINARY_DIR}/rst_manifest.txt - BYPRODUCTS ${gendoc_sources_} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - COMMENT "Generating files for cmake-format") + add_test( + NAME cmake_format-doc-verify-${_genfile} + COMMAND python -Bm cmake_format.doc.gendoc --verify --bits ${_stage}/bits + ${CMAKE_CURRENT_SOURCE_DIR}/${_genfile} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +endforeach() +add_custom_target(genbits-cmake_format DEPENDS ${_genbits}) +add_dependencies(test-deps genbits-cmake_format) +add_custom_target(genfiles-cmake_format DEPENDS ${_genfiles}) +add_dependencies(gen genfiles-cmake_format) -if(NOT CMAKE_GENERATOR STREQUAL "Ninja") +# copy sourcefiles from the sourcetree into the build tree so that they're all +# accessible from one tree +set(_copyfiles) +set(_docsources + .readthedocs.yml + bits/dump-example.cmake + changelog.rst + cmake-annotate.rst + cmake-format.rst + cmake-lint.rst + configopts.rst + configuration.rst + conf.py + contributing.rst + ctest-to.rst + custom_parsers.rst + format-algorithm.rst + format-case_studies.rst + format-example.rst + format-features.rst + format-usage.rst + index.rst + installation.rst + lint-example.rst + lint-summary.rst + lint-usage.rst + modules.rst + parse-algorithm.rst + parse-automatic.rst + parse-tree.rst + release_notes.rst) +foreach(_docsource ${_docsources}) add_custom_command( - OUTPUT ${gendoc_sources_} - DEPENDS gendoc_sources-cmake_format - COMMENT "Stubbing gendocs for cmake_format_doc") -endif() + OUTPUT ${_stage}/${_docsource} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/${_docsource} + ${_stage}/${_docsource} + DEPENDS ${_stage}/bits) + list(APPEND _copyfiles ${_stage}/${_docsource}) +endforeach() +# For the subproject build, also copy some of the common files into the build +# tree. add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake_format_doc.stamp + OUTPUT ${_stage}/conf_common.py + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/doc/conf.py + ${_stage}/conf_common.py) +list(APPEND _copyfiles ${_stage}/conf_common.py) + +add_custom_command( + OUTPUT ${_stage}/sphinx-static/css/cheshire_theme.css COMMAND - env PYTHONPATH=${CMAKE_SOURCE_DIR} sphinx-build -M html - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} - COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/cmake_format_doc.stamp - DEPENDS conf.py - ${CMAKE_CURRENT_BINARY_DIR}/rst_manifest.txt - ${CMAKE_SOURCE_DIR}/doc/conf.py - ${CMAKE_SOURCE_DIR}/doc/sphinx-static/css/cheshire_theme.css - ${gendoc_sources_} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/doc/sphinx-static/css/cheshire_theme.css + ${_stage}/sphinx-static/css/cheshire_theme.css + DEPENDS ${_stage}/bits) +list(APPEND _copyfiles ${_stage}/sphinx-static/css/cheshire_theme.css) -add_custom_target(cmake_format-doc - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cmake_format_doc.stamp) +set(_sphinx_manifest ${_genbits} ${_copyfiles}) + +# A list of files that should be exported to the read-the-docs repository. This +# is used to clean the directory prior to export +set(_rtd_manifest ${_sphinx_manifest} + README.rst .gitignore .readthedocs.yml rtd-requirements.txt) +list(SORT _sphinx_manifest) + +# NOTE(josh): cmake 3.10 does not have list(JOIN) +string(REPLACE ";" "\n" _manifest_text "${_sphinx_manifest}") + +# NOTE(josh): don't use file(WRITE) because that wont create a dependency on the +# configure step to rerun cmake if the file is missing +configure_file(MANIFEST.in ${CMAKE_CURRENT_BINARY_DIR}/MANIFEST + NEWLINE_STYLE UNIX) add_custom_target( - show-cmake_format-doc + clean-sphinx-stage + DEPENDS ${_sphinx_manifest} ${CMAKE_CURRENT_BINARY_DIR}/MANIFEST + COMMAND python -Bm tangent.tooling.clean_stage + ${CMAKE_CURRENT_BINARY_DIR}/MANIFEST ${_stage} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Cleaning cmake_format/doc/stage") + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/html/index.html + COMMAND env PYTHONPATH=${CMAKE_SOURCE_DIR} sphinx-build -M html ${_stage} + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/html/index.html + DEPENDS clean-sphinx-stage ${_sphinx_manifest} + ${CMAKE_CURRENT_BINARY_DIR}/MANIFEST + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Building sphinx documentation for cmake_format") + +add_custom_target(doc-cmake_format + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/html/index.html) + +add_custom_target( + showdoc-cmake_format COMMAND xdg-open ${CMAKE_CURRENT_BINARY_DIR}/html/index.html - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cmake_format_doc.stamp) + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/html/index.html) -add_dependencies(doc cmake_format-doc) +add_dependencies(doc doc-cmake_format) +set_property(GLOBAL APPEND PROPERTY global_doc_files ${_sphinx_manifest}) -add_test( - NAME cmake_format-docsources-test - COMMAND python -Bm cmake_format.doc.gendoc_sources --verify - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES + ${_copyfiles} ${_genbits}) + + +if(IS_TRAVIS_CI) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake-format.deploy_key + COMMAND + # cmake-format: off + openssl aes-256-cbc + -K $ENV{encrypted_bbb694bc6401_key} + -iv $ENV{encrypted_bbb694bc6401_iv} + -in ${CMAKE_SOURCE_DIR}/tangent/tooling/deploy_keys/cmake-format.enc + -out ${CMAKE_CURRENT_BINARY_DIR}/cmake-format.deploy_key + -d + # cmake-format: on + COMMAND chmod 0600 ${CMAKE_CURRENT_BINARY_DIR}/cmake-format.deploy_key + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Decrypting cmake-format.deploy_key") + + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.deploy_key + COMMAND + # cmake-format: off + openssl aes-256-cbc + -K $ENV{encrypted_4bf912706780_key} + -iv $ENV{encrypted_4bf912706780_iv} + -in ${CMAKE_SOURCE_DIR}/tangent/tooling/deploy_keys/cmake-tools-rtd.enc + -out ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.deploy_key + -d + # cmake-format: on + COMMAND chmod 0600 ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.deploy_key + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Decrypting cmake-tools-rtd.deploy_key") +endif() + +set(_ssh_cmd + "ssh -i ${CMAKE_CURRENT_BINARY_DIR}/cmake-format.deploy_key -F /dev/null") + +add_custom_target( + push-github-pseudorelease-tag + COMMAND + python -Bm cmake_format.tools.create_pseudorelease_tag \$\${TRAVIS_BRANCH} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cmake-format.deploy_key + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Pushing doc artifacts to the RTD repository") + + +add_custom_target( + push-rtd-repo + COMMAND + python -Bm cmake_format.tools.gen_rtd_requirements \$\${TRAVIS_TAG} + ${_version} -o ${_stage}/rtd-requirements.txt + COMMAND + # cmake-format: off + python -Bm tangent.tooling.sync_doc_artifacts + --doc-repo git@github.com:cheshirekow/cmake-tools-rtd.git + --repo-dir ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.git + --scratch-tree ${CMAKE_CURRENT_BINARY_DIR}/scratch-tree + --stage ${_stage} + --deploy-key ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.deploy_key + --tag \$\${TRAVIS_TAG} + # cmake-format: on + DEPENDS clean-sphinx-stage ${_sphinx_manifest} + ${CMAKE_CURRENT_BINARY_DIR}/cmake-tools-rtd.deploy_key + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Pushing doc artifacts to the RTD repository") diff --git a/cmake_format/doc/MANIFEST.in b/cmake_format/doc/MANIFEST.in new file mode 100644 index 0000000..006eee0 --- /dev/null +++ b/cmake_format/doc/MANIFEST.in @@ -0,0 +1 @@ +${_manifest_text} diff --git a/cmake_format/doc/README.rst b/cmake_format/doc/README.rst index 3049f57..47dcd19 100644 --- a/cmake_format/doc/README.rst +++ b/cmake_format/doc/README.rst @@ -39,7 +39,7 @@ Integrations Usage ----- -.. dynamic: usage-begin +.. dynamic: format-usage-begin .. code:: text @@ -87,6 +87,12 @@ Usage Various configuration options/parameters for formatting: + Options affecting listfile parsing: + --vartags [VARTAGS [VARTAGS ...]] + Specify variable tags. + --proptags [PROPTAGS [PROPTAGS ...]] + Specify property tags. + Options effecting formatting.: --line-width LINE_WIDTH How wide to allow formatted cmake files @@ -241,7 +247,7 @@ Usage utf-8. Note that cmake only claims to support utf-8 so be careful when using anything else -.. dynamic: usage-end +.. dynamic: format-usage-end ------------- Configuration @@ -252,9 +258,9 @@ An example configuration file is given here. Additional flags and additional kwargs will help ``cmake-format`` to break up your custom commands in a pleasant way. -.. dynamic: configuration-begin +.. dynamic: configbits-begin -.. code:: text +.. code:: python # ---------------------------------- # Options affecting listfile parsing @@ -264,6 +270,12 @@ pleasant way. # Specify structure for custom cmake functions additional_commands = {'pkg_find': {'kwargs': {'PKG': '*'}}} + # Specify variable tags. + vartags = [] + + # Specify property tags. + proptags = [] + # ----------------------------- # Options effecting formatting. # ----------------------------- @@ -468,7 +480,7 @@ pleasant way. per_command = {} -.. dynamic: configuration-end +.. dynamic: configbits-end You may specify a path to a configuration file with the ``--config-file`` command line option. Otherwise, ``cmake-format`` will search the ancestry @@ -680,7 +692,6 @@ custom command would look something like this: .. __: https://cmake-format.rtfd.io/custom_parsers - .. dynamic: features-end --------------------------------- diff --git a/cmake_format/doc/bits/dump-example.cmake b/cmake_format/doc/bits/dump-example.cmake new file mode 100644 index 0000000..b511857 --- /dev/null +++ b/cmake_format/doc/bits/dump-example.cmake @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.5) +project(demo) +if(FOO AND (BAR OR BAZ)) + add_library(hello hello.cc) +endif() diff --git a/cmake_format/doc/changelog.rst b/cmake_format/doc/changelog.rst index 3e36e50..68ba1ce 100644 --- a/cmake_format/doc/changelog.rst +++ b/cmake_format/doc/changelog.rst @@ -6,6 +6,35 @@ Changelog v0.6 series ----------- +v0.6.8 +------ + +* Reduce packaging depependency version numbers +* Add build rules to generate variable and property pattern lists +* Implement lint checks on assignment/use of variables that are "close" to + builtins except for case. +* Move first_token from configuration object into format context +* Add line, col info to lex error message +* Fix wrong root parser for FetchContent_MakeAvailable +* Fix missing support for string integer npargs +* Fix missing spec for derived classes of PositionalGroupNode +* Fix on/off switch doesn't work inside a statement +* Fix extraneous whitespace inserted before line comment in some statements +* Add more helpful error message on failed configfile parse +* Move documentation build to build time and push documentation artifacts + to an artifact repository + +* Closes `#162`: cmake-lint crashes when evaluating `math` +* Closes `#163`: cmake-lint crashes when using `VERBATIM` in + `add_custom_target` +* Closes `#164`: Internal error FetchContent_MakeAvailable +* Closes: 000bf9a, 6e4ef70, 85a3985, 9a3afa6, cf4570e + +.. _#162: https://github.com/cheshirekow/cmake_format/issues/162 +.. _#163: https://github.com/cheshirekow/cmake_format/issues/163 +.. _#164: https://github.com/cheshirekow/cmake_format/issues/164 + + v0.6.7 ------ @@ -24,7 +53,8 @@ v0.6.7 * Closes `#139`_: Disable wrap for custom functions * Closes `#159`_: Missing dependency on six -* Closes: 6ef7d0d, cf7ac49, cfa3c02, eefbde3, e75513a, f704714 +* Closes: 6ef7d0d, 9669d02, cc60267, cf7ac49, cfa3c02, eefbde3, e75513a, +* Closes: f704714 .. _#139: https://github.com/cheshirekow/cmake_format/issues/139 .. _#159: https://github.com/cheshirekow/cmake_format/issues/159 diff --git a/cmake_format/doc/cmake-annotate.rst b/cmake_format/doc/cmake-annotate.rst index af57b78..078299d 100644 --- a/cmake_format/doc/cmake-annotate.rst +++ b/cmake_format/doc/cmake-annotate.rst @@ -9,198 +9,7 @@ program which can create semantic HTML documents from parsed listfiles. This enables, in particular, semantic highlighting for your code documentation. -.. dynamic: annotate-usage-begin - -.. code:: text - - usage: - cmake-annotate [-h] - [--format {page,stub}] - [-o OUTFILE_PATH] - [-c CONFIG_FILE] - infilepath [infilepath ...] - - Parse cmake listfiles and re-emit them with semantic annotations in HTML. - - Some options regarding parsing are configurable by providing a configuration - file. The configuration file format is the same as that used by cmake-format, - and the same file can be used for both programs. - - cmake-format can spit out the default configuration for you as starting point - for customization. Run with `--dump-config [yaml|json|python]`. - - positional arguments: - infilepaths - - optional arguments: - -h, --help show this help message and exit - -v, --version show program's version number and exit - -f {page,stub}, --format {page,stub} - whether to output a standalone `page` complete with - tags, or just the annotated content - -o OUTFILE_PATH, --outfile-path OUTFILE_PATH - Where to write the formatted file. Default is stdout. - -c CONFIG_FILE, --config-file CONFIG_FILE - path to configuration file - - Various configuration options/parameters for formatting: - - - Options effecting formatting.: - --line-width LINE_WIDTH - How wide to allow formatted cmake files - --tab-size TAB_SIZE How many spaces to tab for indent - --max-subgroups-hwrap MAX_SUBGROUPS_HWRAP - If an argument group contains more than this many sub- - groups (parg or kwarg groups) then force it to a - vertical layout. - --max-pargs-hwrap MAX_PARGS_HWRAP - If a positional argument group contains more than this - many arguments, then force it to a vertical layout. - --max-rows-cmdline MAX_ROWS_CMDLINE - If a cmdline positional group consumes more than this - many lines without nesting, then invalidate the layout - (and nest) - --separate-ctrl-name-with-space [SEPARATE_CTRL_NAME_WITH_SPACE] - If true, separate flow control names from their - parentheses with a space - --separate-fn-name-with-space [SEPARATE_FN_NAME_WITH_SPACE] - If true, separate function names from parentheses with - a space - --dangle-parens [DANGLE_PARENS] - If a statement is wrapped to more than one line, than - dangle the closing parenthesis on its own line. - --dangle-align {prefix,prefix-indent,child,off} - If the trailing parenthesis must be 'dangled' on its - on line, then align it to this reference: `prefix`: - the start of the statement, `prefix-indent`: the start - of the statement, plus one indentation level, `child`: - align to the column of the arguments - --min-prefix-chars MIN_PREFIX_CHARS - If the statement spelling length (including space and - parenthesis) is smaller than this amount, then force - reject nested layouts. - --max-prefix-chars MAX_PREFIX_CHARS - If the statement spelling length (including space and - parenthesis) is larger than the tab width by more than - this amount, then force reject un-nested layouts. - --max-lines-hwrap MAX_LINES_HWRAP - If a candidate layout is wrapped horizontally but it - exceeds this many lines, then reject the layout. - --line-ending {windows,unix,auto} - What style line endings to use in the output. - --command-case {lower,upper,canonical,unchanged} - Format command names consistently as 'lower' or - 'upper' case - --keyword-case {lower,upper,unchanged} - Format keywords consistently as 'lower' or 'upper' - case - --always-wrap [ALWAYS_WRAP [ALWAYS_WRAP ...]] - A list of command names which should always be wrapped - --enable-sort [ENABLE_SORT] - If true, the argument lists which are known to be - sortable will be sorted lexicographicall - --autosort [AUTOSORT] - If true, the parsers may infer whether or not an - argument list is sortable (without annotation). - --require-valid-layout [REQUIRE_VALID_LAYOUT] - By default, if cmake-format cannot successfully fit - everything into the desired linewidth it will apply - the last, most agressive attempt that it made. If this - flag is True, however, cmake-format will print error, - exit with non-zero status code, and write-out nothing - - Options affecting comment reflow and formatting.: - --bullet-char BULLET_CHAR - What character to use for bulleted lists - --enum-char ENUM_CHAR - What character to use as punctuation after numerals in - an enumerated list - --first-comment-is-literal [FIRST_COMMENT_IS_LITERAL] - If comment markup is enabled, don't reflow the first - comment block in each listfile. Use this to preserve - formatting of your copyright/license statements. - --literal-comment-pattern LITERAL_COMMENT_PATTERN - If comment markup is enabled, don't reflow any comment - block which matches this (regex) pattern. Default is - `None` (disabled). - --fence-pattern FENCE_PATTERN - Regular expression to match preformat fences in - comments default=r'^\s*([`~]{3}[`~]*)(.*)$' - --ruler-pattern RULER_PATTERN - Regular expression to match rulers in comments - default=r'^\s*[^\w\s]{3}.*[^\w\s]{3}$' - --explicit-trailing-pattern EXPLICIT_TRAILING_PATTERN - If a comment line matches starts with this pattern - then it is explicitly a trailing comment for the - preceeding argument. Default is '#<' - --hashruler-min-length HASHRULER_MIN_LENGTH - If a comment line starts with at least this many - consecutive hash characters, then don't lstrip() them - off. This allows for lazy hash rulers where the first - hash char is not separated by space - --canonicalize-hashrulers [CANONICALIZE_HASHRULERS] - If true, then insert a space between the first hash - char and remaining hash chars in a hash ruler, and - normalize its length to fill the column - --enable-markup [ENABLE_MARKUP] - enable comment markup parsing and reflow - - Options affecting the linter: - --disabled-codes [DISABLED_CODES [DISABLED_CODES ...]] - a list of lint codes to disable - --function-pattern FUNCTION_PATTERN - regular expression pattern describing valid function - names - --macro-pattern MACRO_PATTERN - regular expression pattern describing valid macro - names - --global-var-pattern GLOBAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with global scope - --internal-var-pattern INTERNAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with global scope (but internal semantic) - --local-var-pattern LOCAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with local scope - --private-var-pattern PRIVATE_VAR_PATTERN - regular expression pattern describing valid names for - privatedirectory variables - --public-var-pattern PUBLIC_VAR_PATTERN - regular expression pattern describing valid names for - publicdirectory variables - --keyword-pattern KEYWORD_PATTERN - regular expression pattern describing valid names for - keywords used in functions or macros - --max-conditionals-custom-parser MAX_CONDITIONALS_CUSTOM_PARSER - In the heuristic for C0201, how many conditionals to - match within a loop in before considering the loop a - parser. - --min-statement-spacing MIN_STATEMENT_SPACING - Require at least this many newlines between statements - --max-statement-spacing MAX_STATEMENT_SPACING - Require no more than this many newlines between - statements - --max-returns MAX_RETURNS - --max-branches MAX_BRANCHES - --max-arguments MAX_ARGUMENTS - --max-localvars MAX_LOCALVARS - --max-statements MAX_STATEMENTS - - Options effecting file encoding: - --emit-byteorder-mark [EMIT_BYTEORDER_MARK] - If true, emit the unicode byte-order mark (BOM) at the - start of the file - --input-encoding INPUT_ENCODING - Specify the encoding of the input file. Defaults to - utf-8 - --output-encoding OUTPUT_ENCODING - Specify the encoding of the output file. Defaults to - utf-8. Note that cmake only claims to support utf-8 so - be careful when using anything else - -.. dynamic: annotate-usage-end +.. literalinclude:: bits/annotate-usage.txt ``--format stub`` will output just the marked-up listfile content. The markup is done as ```` elements with different css classes for each diff --git a/cmake_format/doc/cmake-format.rst b/cmake_format/doc/cmake-format.rst index 0b0ce06..2edc8fa 100644 --- a/cmake_format/doc/cmake-format.rst +++ b/cmake_format/doc/cmake-format.rst @@ -8,11 +8,8 @@ like crap. .. toctree:: :maxdepth: 2 - features - usage - example - parse_tree - parser_algorithm - format_algorithm - case_studies - automatic_parsers + format-features + format-usage + format-example + format-algorithm + format-case_studies diff --git a/cmake_format/doc/conf.py b/cmake_format/doc/conf.py index 36bd306..8d5660e 100644 --- a/cmake_format/doc/conf.py +++ b/cmake_format/doc/conf.py @@ -1,20 +1,15 @@ import importlib import os -this_file = os.path.realpath(__file__) -this_dir = os.path.dirname(this_file) -_ = os.path.dirname(this_dir) -root_dir = os.path.dirname(_) - -with open(os.path.join(root_dir, "doc/conf.py")) as infile: +# Source the common stuff +with open(os.path.join("./conf_common.py")) as infile: exec(infile.read()) # pylint: disable=W0122 -project = "cmake_format" -module = importlib.import_module(project) +_module = importlib.import_module("cmake_format") +# Override the project-specific stuff +project = "cmake-tools" docname = project + u'doc' title = project + ' Documentation' -version = module.VERSION -release = module.VERSION - -html_static_path = [os.path.join(root_dir, "doc/sphinx-static")] +version = _module.VERSION +release = _module.VERSION diff --git a/cmake_format/doc/configopts.rst b/cmake_format/doc/configopts.rst index e588e2b..d4e646b 100644 --- a/cmake_format/doc/configopts.rst +++ b/cmake_format/doc/configopts.rst @@ -11,12 +11,56 @@ include Configurations are merged and individual variables follow a latest-wins semantic. +-------------- +Parser options +-------------- + additional_commands =================== Use this variable to specify how to parse custom cmake functions. See :ref:`additional-cmd`. +vartags +======= + +Specify a mapping of variable patterns (python regular expression) to a list +of tags. Any time a a variable matching this pattern is encountered the tags +can be used to affect the parsing/formatting. For example: + +.. code:: + + vartags = [ + (".*_COMMAND", ["cmdline"]) + ] + +Specifies that any variable ending in ``_COMMAND`` be tagged as ``cmdline``. +This will affect the formatting by preventing the arguments from being +vertically wrapped. + +Note: this particular rule is builtin so you do not need to include this in +your configuration. Use the configuration variable to add new rules. + +proptags +======== + +Specify a mapping of property patterns (python regular expression) to a list +of tags. Any time a a property matching this pattern is encountered the tags +can be used to affect the parsing/formatting. For example: + +.. code:: + + proptags = [ + (".*_DIRECTORIES", ["file-list"]) + ] + +Specifies that any property ending in ``_DIRECTORIES`` be tagged as +``file-list``. In the future this may affect formatting by allowing arguments +to be sorted (but currently has no effect). + +Note: this particular rule is builtin so you do not need to include this in +your configuration. Use the configuration variable to add new rules. + -------------------------- General Formatting Options -------------------------- diff --git a/cmake_format/doc/configuration.rst b/cmake_format/doc/configuration.rst index 0489a44..4fb4510 100644 --- a/cmake_format/doc/configuration.rst +++ b/cmake_format/doc/configuration.rst @@ -34,220 +34,4 @@ configuration ``stdout`` and use that as a starting point. Here is an example python-style configuration file with the default options and help-text. Some detailed examples can be found at :ref:`configopts`. -.. dynamic: configuration-begin - -.. code:: text - - # ---------------------------------- - # Options affecting listfile parsing - # ---------------------------------- - with section("parse"): - - # Specify structure for custom cmake functions - additional_commands = {'pkg_find': {'kwargs': {'PKG': '*'}}} - - # ----------------------------- - # Options effecting formatting. - # ----------------------------- - with section("format"): - - # How wide to allow formatted cmake files - line_width = 80 - - # How many spaces to tab for indent - tab_size = 2 - - # If an argument group contains more than this many sub-groups (parg or kwarg - # groups) then force it to a vertical layout. - max_subgroups_hwrap = 2 - - # If a positional argument group contains more than this many arguments, then - # force it to a vertical layout. - max_pargs_hwrap = 6 - - # If a cmdline positional group consumes more than this many lines without - # nesting, then invalidate the layout (and nest) - max_rows_cmdline = 2 - - # If true, separate flow control names from their parentheses with a space - separate_ctrl_name_with_space = False - - # If true, separate function names from parentheses with a space - separate_fn_name_with_space = False - - # If a statement is wrapped to more than one line, than dangle the closing - # parenthesis on its own line. - dangle_parens = False - - # If the trailing parenthesis must be 'dangled' on its on line, then align it - # to this reference: `prefix`: the start of the statement, `prefix-indent`: - # the start of the statement, plus one indentation level, `child`: align to - # the column of the arguments - dangle_align = 'prefix' - - # If the statement spelling length (including space and parenthesis) is - # smaller than this amount, then force reject nested layouts. - min_prefix_chars = 4 - - # If the statement spelling length (including space and parenthesis) is larger - # than the tab width by more than this amount, then force reject un-nested - # layouts. - max_prefix_chars = 10 - - # If a candidate layout is wrapped horizontally but it exceeds this many - # lines, then reject the layout. - max_lines_hwrap = 2 - - # What style line endings to use in the output. - line_ending = 'unix' - - # Format command names consistently as 'lower' or 'upper' case - command_case = 'canonical' - - # Format keywords consistently as 'lower' or 'upper' case - keyword_case = 'unchanged' - - # A list of command names which should always be wrapped - always_wrap = [] - - # If true, the argument lists which are known to be sortable will be sorted - # lexicographicall - enable_sort = True - - # If true, the parsers may infer whether or not an argument list is sortable - # (without annotation). - autosort = False - - # By default, if cmake-format cannot successfully fit everything into the - # desired linewidth it will apply the last, most agressive attempt that it - # made. If this flag is True, however, cmake-format will print error, exit - # with non-zero status code, and write-out nothing - require_valid_layout = False - - # A dictionary mapping layout nodes to a list of wrap decisions. See the - # documentation for more information. - layout_passes = {} - - # ------------------------------------------------ - # Options affecting comment reflow and formatting. - # ------------------------------------------------ - with section("markup"): - - # What character to use for bulleted lists - bullet_char = '*' - - # What character to use as punctuation after numerals in an enumerated list - enum_char = '.' - - # If comment markup is enabled, don't reflow the first comment block in each - # listfile. Use this to preserve formatting of your copyright/license - # statements. - first_comment_is_literal = False - - # If comment markup is enabled, don't reflow any comment block which matches - # this (regex) pattern. Default is `None` (disabled). - literal_comment_pattern = None - - # Regular expression to match preformat fences in comments - # default=r'^\s*([`~]{3}[`~]*)(.*)$' - fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$' - - # Regular expression to match rulers in comments - # default=r'^\s*[^\w\s]{3}.*[^\w\s]{3}$' - ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$' - - # If a comment line matches starts with this pattern then it is explicitly a - # trailing comment for the preceeding argument. Default is '#<' - explicit_trailing_pattern = '#<' - - # If a comment line starts with at least this many consecutive hash - # characters, then don't lstrip() them off. This allows for lazy hash rulers - # where the first hash char is not separated by space - hashruler_min_length = 10 - - # If true, then insert a space between the first hash char and remaining hash - # chars in a hash ruler, and normalize its length to fill the column - canonicalize_hashrulers = True - - # enable comment markup parsing and reflow - enable_markup = True - - # ---------------------------- - # Options affecting the linter - # ---------------------------- - with section("lint"): - - # a list of lint codes to disable - disabled_codes = [] - - # regular expression pattern describing valid function names - function_pattern = '[0-9a-z_]+' - - # regular expression pattern describing valid macro names - macro_pattern = '[0-9A-Z_]+' - - # regular expression pattern describing valid names for variables with global - # scope - global_var_pattern = '[0-9A-Z][0-9A-Z_]+' - - # regular expression pattern describing valid names for variables with global - # scope (but internal semantic) - internal_var_pattern = '_[0-9A-Z][0-9A-Z_]+' - - # regular expression pattern describing valid names for variables with local - # scope - local_var_pattern = '[0-9a-z_]+' - - # regular expression pattern describing valid names for privatedirectory - # variables - private_var_pattern = '_[0-9a-z_]+' - - # regular expression pattern describing valid names for publicdirectory - # variables - public_var_pattern = '[0-9A-Z][0-9A-Z_]+' - - # regular expression pattern describing valid names for keywords used in - # functions or macros - keyword_pattern = '[0-9A-Z_]+' - - # In the heuristic for C0201, how many conditionals to match within a loop in - # before considering the loop a parser. - max_conditionals_custom_parser = 2 - - # Require at least this many newlines between statements - min_statement_spacing = 1 - - # Require no more than this many newlines between statements - max_statement_spacing = 1 - max_returns = 6 - max_branches = 12 - max_arguments = 5 - max_localvars = 15 - max_statements = 50 - - # ------------------------------- - # Options effecting file encoding - # ------------------------------- - with section("encode"): - - # If true, emit the unicode byte-order mark (BOM) at the start of the file - emit_byteorder_mark = False - - # Specify the encoding of the input file. Defaults to utf-8 - input_encoding = 'utf-8' - - # Specify the encoding of the output file. Defaults to utf-8. Note that cmake - # only claims to support utf-8 so be careful when using anything else - output_encoding = 'utf-8' - - # ------------------------------------- - # Miscellaneous configurations options. - # ------------------------------------- - with section("misc"): - - # A dictionary containing any per-command configuration overrides. Currently - # only `command_case` is supported. - per_command = {} - - -.. dynamic: configuration-end +.. literalinclude:: bits/configbits.py diff --git a/cmake_format/doc/ctest-to.rst b/cmake_format/doc/ctest-to.rst index 0cff3f7..135393c 100644 --- a/cmake_format/doc/ctest-to.rst +++ b/cmake_format/doc/ctest-to.rst @@ -9,27 +9,7 @@ generate a more structured representation of the test spec. Usage ----- -.. dynamic: ctest-to-usage-begin - -.. code:: text - - usage: ctest-to [-h] [--log-level {debug,info,warning,error}] [--json | --xml] - [directory] - - Parse ctest testfiles and re-emit the test specification in a more structured - format. - - positional arguments: - directory - - optional arguments: - -h, --help show this help message and exit - --log-level {debug,info,warning,error} - --json - --xml - -.. dynamic: ctest-to-usage-end - +.. literalinclude:: bits/ctest-to-usage.txt ------- Example diff --git a/cmake_format/doc/docsources_test.py b/cmake_format/doc/docsources_test.py deleted file mode 100644 index 2aacdd5..0000000 --- a/cmake_format/doc/docsources_test.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import unittest -import subprocess -import sys -import tempfile - - -class TestDocSources(unittest.TestCase): - """ - Ensure that dynamic documentation content is up-to-date - """ - - def test_docsources_uptodate(self): - rootdir = os.path.realpath(__file__) - for _ in range(3): - rootdir = os.path.dirname(rootdir) - - with tempfile.NamedTemporaryFile(delete=False) as errlog: - errlogpath = errlog.name - result = subprocess.call( - [sys.executable, "-Bm", "cmake_format.doc.gendoc_sources", - "--verify"], - cwd=rootdir, stderr=errlog) - with open(errlogpath, "r") as infile: - errmsg = "Error log: \n" + infile.read() - os.unlink(errlogpath) - self.assertEqual(0, result, errmsg) - - -if __name__ == '__main__': - unittest.main() diff --git a/cmake_format/doc/example_rendered.html b/cmake_format/doc/example_rendered.html index 3f10c46..4a7c914 100644 --- a/cmake_format/doc/example_rendered.html +++ b/cmake_format/doc/example_rendered.html @@ -1,4 +1,3 @@ - - - + \ No newline at end of file diff --git a/cmake_format/doc/format-algorithm.rst b/cmake_format/doc/format-algorithm.rst new file mode 100644 index 0000000..21fc744 --- /dev/null +++ b/cmake_format/doc/format-algorithm.rst @@ -0,0 +1,255 @@ +.. _formatting-algorithm: + +==================== +Formatting Algorithm +==================== + +The formatter works by attempting to select an appropriate ``position`` and +``wrap`` (collectively referred to as a "layout") for each node in the layout +tree. Positions are represented by ``(row, col)`` pairs and the wrap dictates +how childen of that node are positioned. + +-------- +Wrapping +-------- + +``cmake-format`` implements three styles of wrapping. +The default wrapping for all nodes is horizontal wrapping. If horizontal +wrapping fails to emit an admissible layout, then a node will advance to +either vertical wrapping or nested wrapping (which one depends on the type of +node). + +Horizontal Wrapping +=================== + +Horizontal wrapping is like "word wrap". Each child is assigned a position +immediately following it's predecessor, so long as that child fits in the +remaining space up to the column limit. Otherwise the child is moved to the +next line:: + + | |<- col-limit + | ██████ ███ ██ █████ | + | ███████████████ ████ | + | █████████ ████ | + +Note that a line comment can force an early newline:: + + | |<- col-limit + | ██████ ███ # | + | ██ █████ | + | ███████████████ ████ | + | █████████ ████ | + +Note that wrapping happens at the depth of the layout tree, so if we have +multiple groups of multiple arguments each, then each group will be placed +as if it were a single unit:: + + | |<- col-limit + | (██████ ███) (██ █████) | + | (███ ██ ███████████████ ████) | + +Groups may be parenthetical groups (as above) or keyword groups:: + + | |<- col-limit + | ▒▒▒▒▒▒▒ ███ ▒▒▒ █████ | + | ▒▒▒▒▒ ██ ███████████████ ████ | + +or any other grouping assigned by the parser. + +In the event that a subgroup cannot be packed within a single line of full +column width, it will be wrapped internally, and the next group placed on +the next line:: + + | |<- col-limit + | ▒▒▒▒▒ ███ ▒▒▒▒ █████ | + | ▒▒▒▒ ██ ███████████████ ████ | + | ██ █████ | + | ▒▒▒▒▒▒▒ ██ ██ █ ▒▒ █ | + +In particular the following is never a valid packing (where the two groups are +siblings) in the layout tree:: + + | |<- col-limit + | ▒▒▒ █████ ▒▒▒ ██ | + | ███████████████ | + | ████ ██ █████ | + +Vertical Wrapping +================= + +Vertical wrapping assigns each child to the next row:: + + ██████ + ███ + ██ + █████ + ███████████████ + ████ + +Again, note that this happens at the depth of the layout tree. In particular +children may be wrapped horizontally within the subtrees:: + + | ▒▒▒▒▒▒ ███ ██████ |<- col-limit + | ▒▒▒ ██████ ██ | + | ▒▒▒▒ ████ █████ ██████ | + | ██████ ██████ | + | ████ ██████████ | + | ▒▒ ███ ████ | + + +Nesting +======= + +Nesting places children in a column which is one ``tab_width`` to the +right of the parent node's position, and one line below. For example:: + + | |<- col-limit + | ▒▒▒▒▒ | + | ██ ███ ██ █████ | + | ████████████████ | + | █████████ ████ | + +In a more deeply nested layout tree, we might see the following:: + + | |<- col-limit + | ▓▓▓▓▓ | + | ▒▒▒▒▒ | + | ██ ███ ██ █████ | + | ████████████████ | + | █████████ ████ | + | ▒▒▒ | + | ████ ███ █ | + | ▒▒▒▒▒▒ | + | ████ ███ █ | + +Depending on how ``cmake-format`` is configured, elements at different depths +may be nested differently. For example:: + + | |<- col-limit + | ▓▓▓▓▓ | + | ▒▒▒▒▒ ██ ███ ██ █████ | + | ████████████████ | + | █████████ ████ | + | ▒▒▒ ████ ███ █ | + | ▒▒▒▒▒▒ ████ ███ █ | + +Note that the only nodes that can nest are ``STATEMENT`` and ``KWARGGROUP`` +nodes. These nodes necessarily only have one child, an ``ARGGROUP`` node. +Therefore there really isn't a notion of "wrapping" for these nodes. + +-------------------- +Formatting algorithm +-------------------- + +For top-level nodes in the layout tree (i.e. ``COMMENT``, ``STATEMENT``, +``BODY``, ``FLOW_CONTROL``, etc...) the positioning is straight forward and +these nodes are laid out in a single pass. Each child is positioned on the +first line after the output cursor of it's predecessor, and at a column +``config.format.tab_size`` to the right of it's parent. + +``STATEMENTS`` however, are laid out over several passes until the +text for that subtree is accepted. Each pass is governed by a +specification mapping pass number to a wrap decision (i.e. a +boolean indicating whether or not to wrap vertical or nest children) + +Layout Passes +============= + +The current algorithm works in a kind of top-down refinement. When a node is +laid out by calling it's ``reflow()`` method, it is informed of its parent's +current pass number (``passno``). It then iterates through its own ``passno`` +from zero up to it's parent's ``passno`` and terminates at the first admissible +layout. Note that within the layout of the node itself, it's current +``passno`` can only affect its ``wrap`` decision. However, because each of its +children will advance through their own passes, the overall layout of a subtree +between two different passes may change, even if the node at the subtree root +didn't change it's ``wrap`` decision between those passes. + +This approach seems to work well even for +:ref:`deeply nested ` or +:ref:`complex ` statements. + +Newline decision +================ + +When a node is in horizontal layout mode (``wrap=False``), there are a couple +of reasons why the algorithm might choose to insert a newline between two +of it's children. + +1. If a token would overflow the column limit, insert a newline (e.g. the + usual notion of wrapping) +2. If the token is the last token before a closing parenthesis, and the + token plus the parenthesis would overflow the column limit, then insert a + newline. +3. If a token is preceeded by a line comment, then the token cannot be placed + on the same line as the comment (or it will become part of the comment) so + a newline is inserted between them. +4. If a token is a line comment which is not associated with an argument (e.g. + it is a "free" comment at the current scope) then it will not be placed + on the same line as a preceeding argument token. If it was, then subsequent + parses would associate this comment with that argument. In such a case, a + newline is inserted between the preceeding argument and the line comment. +5. If the node is an interior node, and one of it's children is internally + wrapped (i.e. consumes more than two lines) then it will not be placed + on the same line as another node. In such a case a newlines is inserted. +6. If the node is an interior node and a child fails to find an admissible + layout at the current cursor, a newline is inserted and a new layout attempt + is made for the child. + +Admissible layouts +================== + +There are a couple of reasons why a layout may be deemed inadmissible: + +1. If the bounding box of a node overflows the column limit +2. If a node is horizontally wrapped at the current ``passno`` but consumes + more than ``max_lines_hwrap`` lines +3. If the node is horizontally wrapped at the current ``passno`` but the node + path is marked as ``always_wrap`` + +Comments +======== + +A (multi-line) comment on the last row does not contribute to the height for +the purposes of this thresholding, but one on any other line does. Another way +to say this is that comments are excluded from the size computation, but +their influence on other argument is not:: + + # This content is 3 lines tall + foobarbaz_hello(▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + ▏argument_one argument_two # this comment is two lines long and it ▕ + ▏ # forces the next argument onto line three▕ + ▏argument_three argument_four) ▕ + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + # This is only 2 lines tall + foobarbaz_hello(▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + ▏argument_one argument_two argument_three▕ + ▏argument_four # this comment is two lines long and wraps but it + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔# has no contribution to the size of the content. + +Dealing with comments during horizontal wrapping can be a little tricky. +They definitely induce a newline at their termination, but they may also +predicate a newline in front of the commented argument. See the examples in +:ref:`Case Studies/Comments `. We don't necessarily need +to deal with this right now. The user can always force the issue by adding some +comment strings that force a comment width, like this:: + + set(HEADERS header_a.h header_b.h header_c.h + header_d.h # This comment is pretty long and if it's argument is close + # to the edge of the column then the comment gets wrapped + # very poorly ------------------------ + header_e.h header_f.h) + +The string of dashes ``------------------------`` is long enough that the +minimum width of the comment block is given by:: + + # This comment is pretty + # long and if it's + # argument is close to the + # edge of the column then + # the comment gets wrapped + # very poorly + # ------------------------ + +Which would preclude it from being crammed into the right-most slot. + diff --git a/cmake_format/doc/format-case_studies.rst b/cmake_format/doc/format-case_studies.rst new file mode 100644 index 0000000..78023e8 --- /dev/null +++ b/cmake_format/doc/format-case_studies.rst @@ -0,0 +1,579 @@ +============ +Case Studies +============ + +This is a collection of interesting cases that are illustrative of different +concepts for formatting options. + +-------------------- +Positional Arguments +-------------------- + +Lots of short args, looks good all on one line:: + + add_subdirectories(foo bar baz foo2 bar2 baz2) + + +Also doesn't look too bad when wrapped horizontally:: + + add_subdirectories( + foo bar baz foo2 bar2 baz2 foo3 bar3 baz3 foo4 bar4 baz4 foo5 bar5 baz5 + foo6 bar6 baz6 foo7 bar7 baz7 foo8 bar8 baz8 foo9 bar9 baz9) + +Though probably matches expectations better if it is wrapped vertically, +even if it does look like shit:: + + add_subdirectories( + foo + bar + baz + foo2 + bar2 + baz2 + foo3 + bar3 + baz3 + foo4 + bar4 + baz4 + foo5 + bar5 + baz5 + foo6 + bar6 + baz6 + foo7 + bar7 + baz7 + foo8 + bar8 + baz8 + foo9 + bar9 + baz9) + +Just a couple of long args, looks bad wrapped horizontally:: + + set(HEADERS very_long_header_name_a.h very_long_header_name_b.h + very_long_header_name_c.h) + +and looks better wrapped vertically, horizontally nested:: + + set(HEADERS + very_long_header_name_a.h + very_long_header_name_b.h + very_long_header_name_c.h) + +also looks pretty good packed after the first argument:: + + set(HEADERS very_long_header_name_a.h + very_long_header_name_b.h + very_long_header_name_c.h) + +or possibly nested:: + + set(HEADERS + very_long_header_name_a.h + very_long_header_name_b.h + very_long_header_name_c.h) + + set( + HEADERS + very_long_header_name_a.h + very_long_header_name_b.h + very_long_header_name_c.h) + + but this starts to look a little inconsistent when other arguments are + used:: + + set( + HEADERS PARENT_SCOPE + very_long_header_name_a.h + very_long_header_name_b.h + very_long_header_name_c.h) + +Lots of medium-length args, looks good vertical, horizontally nested:: + + set(SOURCES + source_a.cc + source_b.cc + source_d.cc + source_e.cc + source_f.cc + source_g.cc) + +Interestingly, if the PARGGROUP list is one level deeper, the distinction +between what looks good is a little blurrier. With lots of short names, +I think it looks good both ways:: + + # This very long command should be broken up along keyword arguments + foo(nonkwarg_a nonkwarg_b + HEADERS a.h b.h c.h d.h e.h f.h + SOURCES a.cc b.cc d.cc + DEPENDS foo + bar baz) + +versus:: + + # This very long command should be broken up along keyword arguments + foo(nonkwarg_a nonkwarg_b + HEADERS a.h + b.h + c.h + d.h + e.h + f.h + SOURCES a.cc b.cc d.cc + DEPENDS foo + bar baz) + +though it does seems like the same rules can be applied here if we include +some configuration for both number of arguments and length of arguments. + +----------------- +Keyword Arguments +----------------- + +When chidren include both positionals and keyword arguments, +it looks good with each group wrapped vertically, while grand children +are wrapped horizontally:: + + set_target_properties( + foo bar baz + PROPERTIES COMPILE_FLAGS "-std=c++11 -Wall -Wextra") + +I think it makes sense even with just a single positional, even if it is a +little sparsish:: + + set_target_properties( + foo + PROPERTIES COMPILE_FLAGS "-std=c++11 -Wall -Wextra -Wfoobarbazoption") + +Another option is to put the "short" positional arguments on the first line, +but while this might look good in some cases I think the cost of inconstency +is high. This also introduces some readability issues in that the +positional arguments are easy to overlook in this layout:: + + set_target_properties(foo + PROPERTIES COMPILE_FLAGS "-std=c++11 -Wall -Wextra -Wfoobarbazoption") + +Though in the case that it could all be on one line so we should do that:: + + set_target_properties( + foo PROPERTIES COMPILE_FLAGS "-std=c++11 -Wall -Wextra") + +And ``set_target_properties`` is actually kind of special because +``PROPERITES`` doesn't act much like a keyword. It's more of a separator after +which we parse thins in pairs. This might look nicer, but I'm not sure we can +really make it fit with other rules:: + + set_target_properties( + foo PROPERTIES + COMPILE_FLAGS "-std=c++11 -Wall -Wextra -Wfoobarbazoption" + LINKER_FLAGS "-fpic -someotheroption") + +Though with custom parse logic we might be able to do so. The custom parser +would include ``PROPERTIES`` in the positional arguments and would label the +first half of each pair as a ``KEYWORD`` node. + +----- +set() +----- + +``set()`` is somewhat of a special case due to things like this:: + + set(args + OUTPUT fizz.txt + COMMAND foo --bar --baz + DEPENDS foo.txt bar.txt baz.txt + COMMENT "This is my rule" + BYPRODUCTS buzz.txt) + add_custom_command(${arg}) + +I suppose it's not the worse case that we just horizontally wrap it by default +in which case the user can enforce wrapping with line comments. It would be +nice if they could somehow annotate it though, like with a comment +``# cmf: as=add_custom_command``. That sounds complicated though. One really +fancy solution would be to scan for potential kwargs, then try to match against +a known command based on the registry. + +Note that ``set()`` isn't the only command like this. There are likely to be +other commands, specifically wrapper commands, that might take an unstructured +argument list which becomes structured under the hood. + +.. _comments-case-study: + +-------- +Comments +-------- + +Argument comments can get a little tricky, because this looks bad:: + + set(HEADERS header_a.h header_b.h header_c.h header_d.h # This comment is + # pretty long and + # if it's argument + # is close to the + # edge of the column + # then the comment + # gets wrapped very + # poorly + header_e.h header_f.h) + +and this looks good:: + + set(HEADERS + header_a.h + header_b.h + header_c.h + header_d.h # This comment is pretty long and if it's argument is close + # to the edge of the column then the comment gets wrapped + # very poorly + header_e.h + header_f.h) + +but this also looks acceptable and I could imagine some organization choosing +to go this route with their style configuration:: + + set(HEADERS header_a.h header_b.h header_c.h + header_d.h # This comment is pretty long and if it's argument is close + # to the edge of the column then the comment gets wrapped + # very poorly + header_e.h header_f.h) + +So I'm not sure that the presence of a line comment should necessarily +predicate a vertical wrapping. Rather, I think the choice of wrapping strategy +should be independant of the presence of a comment. In the case of horizontal +wrapping though, we need some kind of threshold or score to determine when +a comment has gotten "too smooshed" and the whole thing should move to the +next line. In the example above:: + + # option A: ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + set(HEADERS header_a.h header_b.h header_c.h header_d.h ▏# This comment is ▕ + ▏# pretty long and ▕ + ▏# if it's argument ▕ + ▏# is close to the ▕ + ▏# edge of the column▕ + ▏# then the comment ▕ + ▏# gets wrapped very ▕ + ▏# poorly ▕ + header_e.h header_f.h) ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + + # option B: + set(HEADERS header_a.h header_b.h header_c.h▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + header_d.h ▏# This comment is pretty long and if it's argument is close▕ + ▏# to the edge of the column then the comment gets wrapped ▕ + ▏# very poorly ▕ + header_e.h header_f.h)▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ + +"Option A" lays out the comment on eight lines while "option B" lays out the +comment in three lines. I'm not sure what the threshold should be for choosing +one over the other. Should it be based on how many lines the comment is, or +how much whitespace we introduce due to it? In "Option A" we introduce seven +lines of whitespace between consecutive rows of arguments whereas in "Option B" +we only add two. Should it be based on aspect ratio? + +And, honestly, "Option A" isn't all that bad. I'm not sure it would cross +everyones threshold for inducing a wrap. + +For this particular example I think the best looking layout is the vertical +wrapping, but we don't want the presence of a line comment to automatically +indluce vertical wrapping. For instance in this example, we definitely want +to keep horizontal wrapping, we just want the line comment to induce an early +wrap:: + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + COMMAND sphinx-build -M html # + ${CMAKE_CURRENT_SOURCE_DIR} # + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + DEPENDS ${foobar_docs} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +One comprimise solution is to change the behavior of the line comment +depending on the nature of the PARGGROUP. The parser can tag each PARGGROUP +with it's ``default_wrap`` (either "horizontal" or "vertical"). Then, +when a wrap is required the default wrap can be used. A wrap might be required +due to: + +* arguments overflow the column width +* exceed threshold in number or size of arguments +* presence of a line comment + +This comprimise is the reason the previous version of ``cmake-format`` had a +distinct ``HPACK`` wrapping algorithm. It allowed us a configuration where +all wrapping would be vertical wrapping. + +A second comprimise solution, which is compatible with the previous solution, +is to make the wrapping tunable by an annotation comment. For instance:: + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + COMMAND sphinx-build -M html # cmf:hwrap + ${CMAKE_CURRENT_SOURCE_DIR} # + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + DEPENDS ${foobar_docs} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +where the ``hwrap`` annotation would change the default behavior of the +line comment from inducing vertical wrapping to inducing a newline within +vertical wrapping. If the annotation syntax requires too many characters, we +could use something like double-hash ``##``, hash-h ``#h`, or hash-v (``#v``) +for this purpose. This could be a sandard "microtag" format including the +ability to set the list sortable. For example: ``#v,s`` would be +"vertical, sortable" + +Another interesting case is if we have an argument comment on a keyword +argument, or a prefix group. For example:: + + set(foobarbaz # comment about foobarbaz + value_one value_two value_three value_four value_five value_six + value_seven value_eight) + +Should that be formatted as above, or as:: + + set(foobarbaz # comment about foobarbaz + value_one value_two value_three value_four value_five + value_six value_seven value_eight) + +If we're already formatting set as:: + + set(foobarbaz value_one value_two value_three value_four value_five + value_six value_seven value_eight) + +------- +Nesting +------- + +When logic get's nested, the need to nest after long command names becomes +more apparent:: + + if(foo) + if(sbar) + # This comment is in-scope. + add_library( + foo_bar_baz + foo.cc + bar.cc # this is a comment for arg2 this is more comment for + # arg2, it should be joined with the first. + baz.cc) # This comment is part of add_library + + other_command( + some_long_argument some_long_argument) # this comment is very + # long and gets split + # across some lines + + other_command(some_long_argument some_long_argument some_long_argument) + # this comment is even longer and wouldn't make sense to pack at the + # end of the command so it gets it's own lines + endif() + endif() + +Another good example is ``add_custom_comand()``:: + + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + COMMAND sphinx-build -M html ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND touch ${CMAKE_CURRENT_BINARY_DIR}/foobar_doc.stamp + DEPENDS ${foobar_docs} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +But note the tricky bit here. I think we definitely want the COMMAND +ARGGOUP (which is a single PARGGROUP) to be horizontally wrapped. + +.. _install-case-study: + +There are also some commands with second (or more) levels of keyword +arguments, and it's not clear if the nesting rules are best applied +top-down:: + + install( + TARGETS foo bar baz + ARCHIVE DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP + LIBRARY DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP + RUNTIME DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP) + +Or bottom-up:: + + install( + TARGETS foo bar baz + ARCHIVE + DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP + LIBRARY + DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP + RUNTIME + DESTINATION + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + CONFIGURATIONS Debug Release + COMPONENT foo-component + OPTIONAL EXCLUDE_FROM_ALL NAMELINK_SKIP) + +.. _conditionals-case-study: + +------------ +Conditionals +------------ + +Treating boolean operators as keyword arguments works pretty well, so long +as we treat parenthetical groups as a single unit:: + + set(matchme "_DATA_\\|_CMAKE_\\|INTRA_PRED\\|_COMPILED\\|_HOSTING\\|_PERF_\\|CODER_") + if(("${var}" MATCHES "_TEST_" AND NOT "${var}" MATCHES "${matchme}") + OR (CONFIG_AV1_ENCODER + AND CONFIG_ENCODE_PERF_TESTS + AND "${var}" MATCHES "_ENCODE_PERF_TEST_") + OR (CONFIG_AV1_DECODER + AND CONFIG_DECODE_PERF_TESTS + AND "${var}" MATCHES "_DECODE_PERF_TEST_") + OR (CONFIG_AV1_ENCODER AND "${var}" MATCHES "_TEST_ENCODER_") + OR (CONFIG_AV1_DECODER AND "${var}" MATCHES "_TEST_DECODER_")) + list(APPEND aom_test_source_vars ${var}) + endif() + +I don't think there's any reason to add structure for the internal operators +like ``MATCHES``. In particular children of a boolean operator can be simple +positional argument groups (horizontally-wrapped). We can tag the internal +operator as a keyword but we don't need to create a KWARGGROUP for it. + +------------------------------ +Internally Wrapped Positionals +------------------------------ + +The third kwarg (AND) in this statement looks bad because it is Internally +wrapped. The second option looks better: + +.. code:: cmake + + set(matchme "_DATA_\|_CMAKE_\|INTRA_PRED\|_COMPILED\|_HOSTING\|_PERF_\|CODER_") + if(("${var}" MATCHES "_TEST_" AND NOT "${var}" MATCHES "${matchme}") + OR (CONFIG_AV1_ENCODER AND CONFIG_ENCODE_PERF_TESTS AND "${var}" MATCHES + "_ENCODE_PERF_TEST_" + )) + list(APPEND aom_test_source_vars ${var}) + endif() + + set(matchme "_DATA_\|_CMAKE_\|INTRA_PRED\|_COMPILED\|_HOSTING\|_PERF_\|CODER_") + if(("${var}" MATCHES "_TEST_" AND NOT "${var}" MATCHES "${matchme}") + OR (CONFIG_AV1_ENCODER + AND CONFIG_ENCODE_PERF_TESTS + AND "${var}" MATCHES "_ENCODE_PERF_TEST_")) + list(APPEND aom_test_source_vars ${var}) + endif() + +However, this short :code:`set()` statement looks better if we don't push the +internally wrapped argument to the next line: + +.. code:: cmake + + set(sources # cmake-format: sortable + bar.cc baz.cc foo.cc) + +Perhaps the difference is that in the latter case it's going to consume two +lines anyway... whereas in the former case it would only consume one +line. + +-------------------- +Columnized arguments +-------------------- + +Some very long statements with a large number of keywords might look nice +and organized if we columize the child argument groups. For example: + +.. code:: cmake + + ExternalProject_Add( + FOO + PREFIX ${FOO_PREFIX} + TMP_DIR ${TMP_DIR} + STAMP_DIR ${FOO_PREFIX}/stamp + # Download + DOWNLOAD_DIR ${DOWNLOAD_DIR} + DOWNLOAD_NAME ${FOO_ARCHIVE_FILE_NAME} + URL ${STORAGE_URL}/${FOO_ARCHIVE_FILE_NAME} + URL_MD5 ${FOO_MD5} + # Patch + PATCH_COMMAND ${PATCH_COMMAND} ${PROJECT_SOURCE_DIR}/patch.diff + # Configure + SOURCE_DIR ${SRC_DIR} + CMAKE_ARGS ${CMAKE_OPTS} + # Build + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${CUR_COMPONENT_ARTIFACTS} + # Logging + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + ) + +Note what :code:`clang-format` does for these cases. If two consecutive +keywords are more than :code:`n` characters different in length, then break +columns, which might come out something like this: + +.. code:: cmake + + ExternalProject_Add( + FOO + PREFIX ${FOO_PREFIX} + TMP_DIR ${TMP_DIR} + STAMP_DIR ${FOO_PREFIX}/stamp + # Download + DOWNLOAD_DIR ${DOWNLOAD_DIR} + DOWNLOAD_NAME ${FOO_ARCHIVE_FILE_NAME} + URL ${STORAGE_URL}/${FOO_ARCHIVE_FILE_NAME} + URL_MD5 ${FOO_MD5} + # Patch + PATCH_COMMAND ${PATCH_COMMAND} ${PROJECT_SOURCE_DIR}/patch.diff + # Configure + SOURCE_DIR ${SRC_DIR} + CMAKE_ARGS ${CMAKE_OPTS} + # Build + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${CUR_COMPONENT_ARTIFACTS} + # Logging + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + ) + +As an experimental feature, we could require a tag :code:`# cmf: columnize` +to enable this formatting. + +------------------------- +Algorithm Ideas and Notes +------------------------- + +Layout Passes +============= + +Up through version 0.5.2 each node would lay itself out using pass numbers +``[0, ]``. This worked pretty well, but actually I would like +the nesting to be a little more depth dependant. For example I would like +depth 0 (statement) to nest rather early, while I would like higher depths +(i.e. KWARGS) to nest later, but go vertical earlier. + +One alternative is to have a global ``passno`` and apply different rules at +each pass until things fit, but the probem with this option is that two +subtrees might require fastly different passes. We don't want to +vertically wrap one all kwargs just because one needs to. diff --git a/cmake_format/doc/format-example.rst b/cmake_format/doc/format-example.rst new file mode 100644 index 0000000..1911286 --- /dev/null +++ b/cmake_format/doc/format-example.rst @@ -0,0 +1,11 @@ +------- +Example +------- + +Will turn this: + +.. literalinclude:: bits/example-in.cmake + +into this: + +.. literalinclude:: bits/example-out.cmake diff --git a/cmake_format/doc/format-features.rst b/cmake_format/doc/format-features.rst new file mode 100644 index 0000000..01317d7 --- /dev/null +++ b/cmake_format/doc/format-features.rst @@ -0,0 +1,199 @@ +======== +Features +======== + +------- +Markup +------- + +``cmake-format`` is for the exceptionally lazy. It will even format your +comments for you. It will reflow your comment text to within the configured +line width. It also understands a very limited markup format for a couple of +common bits. + +**rulers**: A ruler is a line which starts with and ends with three or more +non-alphanum or space characters:: + + # ---- This is a Ruler ---- + # cmake-format will know to keep the ruler separated from the + # paragraphs around it. So it wont try to reflow this text as + # a single paragraph. + # ---- This is also a Ruler --- + + +**list**: A list is started on the first encountered list item, which starts +with a bullet character (``*``) followed by a space followed by some text. +Subsequent lines will be included in the list item until the next list item +is encountered (the bullet must be at the same indentation level). The list +must be surrounded by a pair of empty lines. Nested lists will be formatted in +nested text:: + + # here are some lists: + # + # * item 1 + # * item 2 + # + # * subitem 1 + # * subitem 2 + # + # * second list item 1 + # * second list item 2 + +**enumerations**: An enumeration is similar to a list but the bullet character +is some integers followed by a period. New enumeration items are detected as +long as either the first digit or the punctuation lines up in the same column +as the previous item. ``cmake-format`` will renumber your items and align their +labels for you:: + + # This is an enumeration + # + # 1. item + # 2. item + # 3. item + +**fences**: If you have any text which you do not want to be formatted you can +guard it with a pair of fences. Fences are three or more tilde characters:: + + # ~~~ + # This comment is fenced + # and will not be formatted + # ~~~ + +Note that comment fences guard reflow of *comment text*, and not cmake code. +If you wish to prevent formatting of cmake, code, see below. In addition to +fenced-literals, there are three other ways to preserve comment text from +markup and/or reflow processing: + +* The ``--first-comment-is-literal`` configuration option will exactly preserve + the first comment in the file. This is intended to preserve copyright or + other formatted header comments. +* The ``--literal-comment-pattern`` configuration option allows for a more + generic way to identify comments which should be preserved literally. This + configuration takes a regular expression pattern. +* The ``--enable-markup`` configuration option globally enables comment markup + processing. It defaults to true so set it to false if you wish to globally + disable comment markup processing. Note that trailing whitespace is still + chomped from comments. + +-------------------------- +Disable Formatting Locally +-------------------------- + +You can locally disable and enable code formatting by using the special +comments ``# cmake-format: off`` and ``# cmake-format: on``. + +------------------- +Sort Argument Lists +------------------- + +Starting with version `0.5.0`, ``cmake-format`` can sort your argument lists +for you. If the configuration includes ``autosort=True`` (the default), it +will replace:: + + add_library(foobar STATIC EXCLUDE_FROM_ALL + sourcefile_06.cc + sourcefile_03.cc + sourcefile_02.cc + sourcefile_04.cc + sourcefile_07.cc + sourcefile_01.cc + sourcefile_05.cc) + +with:: + + add_library(foobar STATIC EXCLUDE_FROM_ALL + sourcefile_01.cc + sourcefile_02.cc + sourcefile_03.cc + sourcefile_04.cc + sourcefile_05.cc + sourcefile_06.cc + sourcefile_07.cc) + +This is implemented for any argument lists which the parser knows are +inherently sortable. This includes the following cmake commands: + +* ``add_library`` +* ``add_executable`` + +For most other cmake commands, you can use an annotation comment to hint to +``cmake-format`` that the argument list is sortable. For instance:: + + set(SOURCES + # cmake-format: sortable + bar.cc + baz.cc + foo.cc) + +Annotations can be given in a line-comment or a bracket comment. There is a +long-form and a short-form for each. The acceptable formats are: + ++-----------------+-------+------------------------------+ +| Line Comment | long | ``# cmake-format: `` | ++-----------------+-------+------------------------------+ +| Line Comment | short | ``# cmf: `` | ++-----------------+-------+------------------------------+ +| Bracket Comment | long | ``#[[cmake-format: ]]`` | ++-----------------+-------+------------------------------+ +| Bracket Comment | short | ``#[[cmf: ]]`` | ++-----------------+-------+------------------------------+ + +In order to annotate a positional argument list as sortable, the acceptable +tags are: ``sortable`` or ``sort``. For the commands listed above where +the positinal argument lists are inherently sortable, you can locally disable +sorting by annotating them with ``unsortable`` or ``unsort``. For example:: + + add_library(foobar STATIC + # cmake-format: unsort + sourcefile_03.cc + sourcefile_01.cc + sourcefile_02.cc) + +Note that this is only needed if your configuration has enabled ``autosort``, +and you can globally disable sorting by making setting this configuration to +``False``. + + +--------------- +Custom Commands +--------------- + +Due to the fact that cmake is a macro language, `cmake-format` is, by +necessity, a *semantic* source code formatter. In general it tries to make +smart formatting decisions based on the meaning of arguments in an otherwise +unstructured list of arguments in a cmake statement. `cmake-format` can +intelligently format your custom commands, but you will need to tell it how +to interpret your arguments. + +Currently, you can do this by adding your command specifications to the +`additional_commands` configuration variables, e.g.: + +.. code:: + + # Additional FLAGS and KWARGS for custom commands + additional_commands = { + "foo": { + "pargs": 2, + "flags": ["BAR", "BAZ"], + "kwargs": { + "HEADERS": '*', + "SOURCES": '*', + "DEPENDS": '*', + } + } + } + +The format is a nested dictionary mapping statement names (dictionary keys) +to `argument specifications`__. For the example specification above, the +custom command would look something like this: + +.. code:: + + foo(hello world + HEADERS a.h b.h c.h d.h + SOURCES a.cc b.cc c.cc d.cc + DEPENDS flub buzz bizz + BAR BAZ) + + +.. __: https://cmake-format.rtfd.io/custom_parsers diff --git a/cmake_format/doc/format-usage.rst b/cmake_format/doc/format-usage.rst new file mode 100644 index 0000000..ccff60f --- /dev/null +++ b/cmake_format/doc/format-usage.rst @@ -0,0 +1,5 @@ +===== +Usage +===== + +.. literalinclude:: bits/format-usage.txt diff --git a/cmake_format/doc/gendoc.py b/cmake_format/doc/gendoc.py new file mode 100644 index 0000000..3ff785c --- /dev/null +++ b/cmake_format/doc/gendoc.py @@ -0,0 +1,136 @@ +""" +Update dynamic parts of documentation files. +""" + +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import difflib +import io +import os +import re +import sys + + +def format_directive(content, codetag=None): + # TODO(josh): change the indentation to 2 for some more compact files + if codetag is None: + codetag = "" + if codetag == "table": + outlines = [".. table:: ", " :align: center", ""] + else: + outlines = [".. code:: {}".format(codetag), ""] + for line in content.split("\n"): + outlines.append((" " + line).rstrip()) + outlines.append("") + return "\n".join(outlines) + + +EXTENSION_TO_TAG = { + ".txt": "text", + ".py": "python", + ".cmake": "cmake" +} + + +def get_dynamic_text(bitsdir, bittag): + for filename in os.listdir(bitsdir): + if not filename.startswith(bittag): + continue + + inpath = os.path.join(bitsdir, filename) + with io.open(inpath, "r", encoding="utf-8") as infile: + content = infile.read() + + extension = filename[len(bittag):] + if filename.endswith("-table.rst"): + # TODO(josh): replace the following line with + # codetag = "table" + # so that tables are center aligned in the output + return content + if filename.endswith(".rst"): + return content + + codetag = EXTENSION_TO_TAG.get(extension, None) + return format_directive(content, codetag) + raise RuntimeError("No dynamic text for bit {}".format(bittag)) + + +def process_file(filepath, nextpath, bitsdir): + """ + Write a copy of `filepath` to `nextpath` with substitutions from dynamic_text + """ + tag_pattern = re.compile("^.. dynamic: (.*)-(begin|end)$") + active_section = None + + with io.open(filepath, "r", encoding="utf-8") as infile: + with io.open(nextpath, "w", encoding="utf-8") as outfile: + for line in infile: + match = tag_pattern.match(line.strip()) + if active_section is None: + outfile.write(line) + + if match: + if match.group(2) == "begin": + active_section = match.group(1) + elif match.group(2) == "end": + assert active_section == match.group(1), "Unexpected end tag" + outfile.write("\n") + content = get_dynamic_text(bitsdir, active_section) + outfile.write(content) + active_section = None + outfile.write(line) + else: + raise RuntimeError("Unexpected tag") + + +def update_file(filepath, bitsdir): + """ + Create a copy of the file replacing any dynamic sections with updated + text. Then replace the original with the updated copy. + """ + + nextpath = filepath + ".next" + process_file(filepath, nextpath, bitsdir) + os.rename(nextpath, filepath) + + +def verify_file(filepath, bitsdir): + """ + Create a copy of the file replacing any dynamic sections with updated + text. Then verify that the copy is not different than the original + """ + + nextpath = filepath + ".tmp" + process_file(filepath, nextpath, bitsdir) + with io.open(filepath, "r", encoding="utf-8") as infile: + lines_a = infile.read().split("\n") + with io.open(nextpath, "r", encoding="utf-8") as infile: + lines_b = infile.read().split("\n") + os.unlink(nextpath) + diff = list(difflib.unified_diff(lines_a, lines_b)) + if diff: + raise RuntimeError("{} is out of date\n{}" + .format(filepath, "\n".join(diff))) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--verify", action="store_true") + parser.add_argument("--bits") + parser.add_argument("filepath") + args = parser.parse_args() + + if args.verify and sys.version_info < (3, 6, 0): + print("Skipping verification on python < 3.6") + sys.exit(0) + + if args.verify: + verify_file(args.filepath, args.bits) + else: + update_file(args.filepath, args.bits) + + +if __name__ == "__main__": + main() diff --git a/cmake_format/doc/gendoc_sources.py b/cmake_format/doc/gendoc_sources.py deleted file mode 100644 index 26e3b9f..0000000 --- a/cmake_format/doc/gendoc_sources.py +++ /dev/null @@ -1,270 +0,0 @@ -""" -Update dynamic parts of README.rst - -Execute from the `cmake_format` project directory with something like: - -python -B doc/update_readme.py -""" - -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import difflib -import io -import os -import re -import subprocess -import sys -import tempfile - -import cmake_lint.test.genfiles - - -def format_directive(content, codetag=None): - if codetag is None: - codetag = "" - outlines = [".. code:: {}".format(codetag), ""] - for line in content.split("\n"): - outlines.append((" " + line).rstrip()) - outlines.append("") - return "\n".join(outlines) - - -def process_file(filepath, nextpath, dynamic_text): - """ - Write a copy of `filepath` to `nextpath` with substitutions from dynamic_text - """ - tag_pattern = re.compile("^.. dynamic: (.*)-(begin|end)$") - active_section = None - - with io.open(filepath, "r", encoding="utf-8") as infile: - with io.open(nextpath, "w", encoding="utf-8") as outfile: - for line in infile: - match = tag_pattern.match(line.strip()) - if active_section is None: - outfile.write(line) - - if match: - if match.group(2) == "begin": - active_section = match.group(1) - elif match.group(2) == "end": - assert active_section == match.group(1), "Unexpected end tag" - outfile.write("\n") - outfile.write(dynamic_text[active_section]) - active_section = None - outfile.write(line) - else: - raise RuntimeError("Unexpected tag") - - -def update_file(filepath, dynamic_text): - """ - Create a copy of the file replacing any dynamic sections with updated - text. Then replace the original with the updated copy. - """ - - nextpath = filepath + ".next" - process_file(filepath, nextpath, dynamic_text) - os.rename(nextpath, filepath) - - -def verify_file(filepath, dynamic_text): - """ - Create a copy of the file replacing any dynamic sections with updated - text. Then verify that the copy is not different than the original - """ - - basename = os.path.basename(filepath) - if "." in basename: - prefix, suffix = basename.split(".", 1) - else: - prefix = basename - suffix = None - - fd, nextpath = tempfile.mkstemp(prefix=prefix, suffix=suffix) - os.close(fd) - process_file(filepath, nextpath, dynamic_text) - with io.open(filepath, "r", encoding="utf-8") as infile: - lines_a = infile.read().split("\n") - with io.open(nextpath, "r", encoding="utf-8") as infile: - lines_b = infile.read().split("\n") - - diff = list(difflib.unified_diff(lines_a, lines_b)) - if diff: - raise RuntimeError("{} is out of date\n{}" - .format(filepath, "\n".join(diff))) - - -DUMP_EXAMPLE = """ -cmake_minimum_required(VERSION 3.5) -project(demo) -if(FOO AND (BAR OR BAZ)) - add_library(hello hello.cc) -endif() -""" - - -def generate_annotate_output(outfile, argv, **kwargs): - outfile.write(""" - - -""".encode("utf-8")) - - -def generate_docsources(verify): - """ - Process dynamic text sources and perform text substitutions - """ - - docdir = os.path.dirname(os.path.realpath(__file__)) - projectdir = os.path.dirname(docdir.rstrip(os.sep)) - repodir = os.path.dirname(projectdir.rstrip(os.sep)) - env = os.environ.copy() - env["PYTHONPATH"] = repodir - - dynamic_text = {} - dynamic_text["usage"] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_format", "--help"], env=env - ).decode("utf-8"), - "text") - - dynamic_text["annotate-usage"] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_format.annotate", "--help"], env=env - ).decode("utf-8"), - "text") - - dynamic_text["ctest-to-usage"] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_format.ctest_to", "--help"], env=env - ).decode("utf-8"), - "text") - - with io.open(os.path.join( - repodir, "cmake_lint/test/expect_lint.cmake")) as infile: - dynamic_text["lint-in"] = format_directive(infile.read(), "cmake") - - with tempfile.NamedTemporaryFile(delete=False) as outfile: - subprocess.call( - [sys.executable, "-Bm", "cmake_lint", - "cmake_lint/test/expect_lint.cmake"], - env=env, cwd=repodir, stdout=outfile) - - lintout = outfile.name - with io.open(lintout, "r") as infile: - dynamic_text["lint-out"] = format_directive(infile.read(), "text") - - dynamic_text["lint-usage"] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_lint", "--help"], env=env - ).decode("utf-8"), - "text") - - dynamic_text["configuration"] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_format", "--dump-config"], env=env - ).decode("utf-8"), - "text") - - with tempfile.NamedTemporaryFile( - mode="w", delete=False) as outfile: - outfile.write(DUMP_EXAMPLE) - dump_example_src = outfile.name - - dynamic_text["dump-example-src"] = format_directive(DUMP_EXAMPLE, "cmake") - for step in ("lex", "parse", "layout"): - dynamic_text["dump-example-" + step] = format_directive( - subprocess.check_output( - [sys.executable, "-Bm", "cmake_format", - "--dump", step, dump_example_src], env=env - ).decode("utf-8"), - "text" - ) - - with io.open(os.path.join(docdir, "features.rst"), - encoding="utf-8") as infile: - copylines = [] - for idx, line in enumerate(infile): - if idx > 3: - copylines.append(line) - copylines.append("\n") - dynamic_text["features"] = "".join(copylines) - - with io.open( - os.path.join(projectdir, "test/test_in.cmake"), - encoding="utf-8") as infile: - dynamic_text["example-in"] = format_directive(infile.read(), "cmake") - with io.open( - os.path.join(projectdir, "test/test_out.cmake"), - encoding="utf-8") as infile: - dynamic_text["example-out"] = format_directive(infile.read(), "cmake") - - dynamic_text["lintimpl-table"] = subprocess.check_output( - [sys.executable, "-Bm", "cmake_lint.gendocs", "table"], - cwd=repodir, env=env).decode("utf-8") - - if verify: - handle_file = verify_file - else: - handle_file = update_file - - for filename in ( - "doc/README.rst", - "doc/configuration.rst", - "doc/cmake-annotate.rst", - "doc/cmake-format.rst", - "doc/cmake-lint.rst", - "doc/ctest-to.rst", - "doc/example.rst", - "doc/lint-example.rst", - "doc/lint-usage.rst", - "doc/lint-summary.rst", - "doc/parse_tree.rst", - "doc/usage.rst", - ): - handle_file(os.path.join(projectdir, filename), dynamic_text) - - if not verify: - with io.open(os.path.join( - projectdir, "doc/example_rendered.html"), "wb") as outfile: - generate_annotate_output( - outfile, [sys.executable, "-Bm", "cmake_format.annotate", - "--format", "page", "test/test_out.cmake"], - env=env, cwd=projectdir) - - with io.open( - os.path.join(projectdir, "doc/lint-implemented.rst"), "wb") as outfile: - subprocess.check_call( - [sys.executable, "-Bm", "cmake_lint.gendocs", "reference"], - cwd=repodir, stdout=outfile) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--verify", action="store_true") - args = parser.parse_args() - - if args.verify and sys.version_info < (3, 6, 0): - print("Skipping verification on python < 3.6") - sys.exit(0) - - generate_docsources(args.verify) - if not args.verify: - cmake_lint.test.genfiles.genfiles() - - -if __name__ == "__main__": - main() diff --git a/cmake_format/doc/index.rst b/cmake_format/doc/index.rst index 73aa1b1..b25bc8b 100644 --- a/cmake_format/doc/index.rst +++ b/cmake_format/doc/index.rst @@ -33,6 +33,9 @@ for ``cmake``: ctest-to configopts custom_parsers + parse-tree + parse-algorithm + parse-automatic contributing release_notes diff --git a/cmake_format/doc/installation.rst b/cmake_format/doc/installation.rst index e1283dc..560c670 100644 --- a/cmake_format/doc/installation.rst +++ b/cmake_format/doc/installation.rst @@ -44,7 +44,7 @@ Install from source You can also install from source with pip. You can download a release package from github__ or pypi__ and then install it directly with pip. For example:: - pip install v0.6.7.tar.gz + pip install v0.6.8.tar.gz .. __: https://github.com/cheshirekow/cmake_format/releases .. __: https://pypi.org/project/cmake-format/#files @@ -90,6 +90,6 @@ formatter, ``cmake-format``, to your hooks with the following addition to your repos: - repo: https://github.com/cheshirekow/cmake-format-precommit - rev: v0.6.7 + rev: v0.6.8 hooks: - id: cmake-format diff --git a/cmake_format/doc/lint-example.rst b/cmake_format/doc/lint-example.rst index da94c9c..6b41047 100644 --- a/cmake_format/doc/lint-example.rst +++ b/cmake_format/doc/lint-example.rst @@ -4,234 +4,8 @@ Example Given the following linty file: -.. dynamic: lint-in-begin - -.. code:: cmake - - # The line is too long and exceeds the default 80 character column limit enforced cmake-lint - - # This line has trailing whitespace - - # This line has the wrong line endings - - - function(BAD_FUNCTION_NAME badArgName good_arg_name) - # Function names should be lower case - endfunction() - - # - macro(bad_macro_name badArgName good_arg_name) - # Macro names should be upper case - endmacro() - - if(FOOBAR) - foreach(loopvar a b c d) - # cmake-lint: disable=C0103 - foreach(LOOPVAR2 a b c d) - # pass - endforeach() - endforeach() - foreach(LOOPVAR3 a b c d) - # pass - endforeach() - - endif() - - set(VARNAME varvalue CACHE STRING) - - cmake_minimum_required_version(VERSION 2.8.11 VERSION 3.16) - - add_custom_command() - - set(_form TARGET PRE_BUILD) - add_custom_command( - ${form} - COMMAND echo "hello" - COMMENT "echo hello") - - add_custom_command( - TARGRET PRE_BUILD - COMMAND echo "hello" - COMMENT "echo hello") - - add_custom_command(OUTPUT foo) - - add_custom_target(foo ALL) - - file() - - set(_form TOUCH) - file(${form} foo.py) - - file(TOUCHE foo.py) - - break() - - continue() - - # foo: docstring - function(foo) - foreach(arg ${ARGN}) - if(arg STREQUAL "FOO") - # parse FOO - elseif(arg STREQUAL "BAR") - # parse BAR - elseif(arg MATCHES "BAZ.*") - # parse BAZ - endif() - endforeach() - endfunction() - - set(_foo "hello") set(_bar "goodbye") - - set(_foo "hello") - - - set(_bar "goodbye") - - # foo: docstring - function(foo "arg-name") - # pass - endfunction() - - # foo: docstring - function(foo arg arg) - # pass - endfunction() - - # foo: docstring - function(foo arg ARG) - # pass - endfunction() - - return() - message("This code is unreachable") - - # cmake-lint: disable=E0109,C0321 - # foo:docstring, too many arguments - function(foo, a0, a1, a2, a3, a4, a5) - - # Too many branches, too many returns - if(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - elseif(blah) return() - else() return() - endif() - - # too many statements - message(foo) message(foo) message(foo) message(foo) message(foo) message(foo) - message(foo) message(foo) message(foo) message(foo) message(foo) message(foo) - message(foo) message(foo) message(foo) message(foo) message(foo) message(foo) - message(foo) message(foo) message(foo) message(foo) message(foo) message(foo) - message(foo) message(foo) message(foo) message(foo) message(foo) message(foo) - endfunction() - - # cmake-lint: disable=C0111 - # cache (global) variables should be upper snake - set(MyGlobalVar CACHE STRING "my var") - # internal variables are treated as private and should be upper snake with an - # underscore prefix - set(MY_INTERNAL_VAR CACHE INTERNAL "my var") - # directory-scope variables should be upper-snake (public) or lower-snake with - # underscore prefix - set(_INVALID_PRIVATE_NAME "foo") - set(invalid_public_name "foo") - function(foo) - set(INVALID_LOCAL_NAME "foo") - endfunction() - - # This file is missing a final newline -.. dynamic: lint-in-end +.. literalinclude:: bits/lint-in.txt The output is: -.. dynamic: lint-out-begin - -.. code:: text - - cmake_lint/test/expect_lint.cmake - ================================= - cmake_lint/test/expect_lint.cmake:00: [C0301] Line too long (92/80) - cmake_lint/test/expect_lint.cmake:02: [C0303] Trailing whitespace - cmake_lint/test/expect_lint.cmake:04: [C0327] Wrong line ending (windows) - cmake_lint/test/expect_lint.cmake:08,00: [C0111] Missing docstring on function or macro declaration - cmake_lint/test/expect_lint.cmake:08,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:08,09: [C0103] Invalid function name "BAD_FUNCTION_NAME" - cmake_lint/test/expect_lint.cmake:13,00: [C0112] Empty docstring on function or macro declaration - cmake_lint/test/expect_lint.cmake:13,06: [C0103] Invalid function name "bad_macro_name" - cmake_lint/test/expect_lint.cmake:17,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:24,10: [C0103] Invalid loopvar name "LOOPVAR3" - cmake_lint/test/expect_lint.cmake:30,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:30,27: [E1120] Missing required positional argument - cmake_lint/test/expect_lint.cmake:30,33: [E1120] Missing required positional argument - cmake_lint/test/expect_lint.cmake:32,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:32,46: [E1122] Duplicate keyword argument VERSION - cmake_lint/test/expect_lint.cmake:34,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:34,19: [E1120] Missing required positional argument - cmake_lint/test/expect_lint.cmake:36,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:38,02: [C0114] Form descriminator hidden behind variable dereference - cmake_lint/test/expect_lint.cmake:42,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:43,02: [E1126] Invalid form descriminator - cmake_lint/test/expect_lint.cmake:47,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:47,19: [C0113] Missing COMMENT in statement which allows it - cmake_lint/test/expect_lint.cmake:47,19: [E1125] Missing required keyword argument COMMAND - cmake_lint/test/expect_lint.cmake:49,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:49,18: [C0113] Missing COMMAND in statement which allows it - cmake_lint/test/expect_lint.cmake:49,18: [C0113] Missing COMMENT in statement which allows it - cmake_lint/test/expect_lint.cmake:51,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:51,05: [E1120] Missing required positional argument - cmake_lint/test/expect_lint.cmake:53,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:54,05: [C0114] Form descriminator hidden behind variable dereference - cmake_lint/test/expect_lint.cmake:56,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:56,05: [E1126] Invalid form descriminator - cmake_lint/test/expect_lint.cmake:58,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:58,00: [E0103] break outside of loop - cmake_lint/test/expect_lint.cmake:58,00: [W0101] Unreachable code - cmake_lint/test/expect_lint.cmake:60,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:60,00: [E0103] continue outside of loop - cmake_lint/test/expect_lint.cmake:60,00: [W0101] Unreachable code - cmake_lint/test/expect_lint.cmake:64,02: [C0201] Consider replacing custom parser logic with cmake_parse_arguments - cmake_lint/test/expect_lint.cmake:75,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:75,18: [C0321] Multiple statements on a single line - cmake_lint/test/expect_lint.cmake:77,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:80,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:83,13: [E0109] Invalid argument name "arg-name" in function/macro definition - cmake_lint/test/expect_lint.cmake:88,17: [E0108] Duplicate argument name arg in function/macro definition - cmake_lint/test/expect_lint.cmake:93,17: [C0202] Argument name ARG differs from existing argument only in case - cmake_lint/test/expect_lint.cmake:97,00: [C0305] too many newlines between statements - cmake_lint/test/expect_lint.cmake:97,00: [W0101] Unreachable code - cmake_lint/test/expect_lint.cmake:102,00: [R0911] Too many return statements 17/6 - cmake_lint/test/expect_lint.cmake:102,00: [R0912] Too many branches 17/12 - cmake_lint/test/expect_lint.cmake:102,00: [R0913] Too many named arguments 6/5 - cmake_lint/test/expect_lint.cmake:102,00: [R0915] Too many statements 65/50 - cmake_lint/test/expect_lint.cmake:134,04: [C0103] Invalid CACHE variable name "MyGlobalVar" - cmake_lint/test/expect_lint.cmake:137,04: [C0103] Invalid INTERNAL variable name "MY_INTERNAL_VAR" - cmake_lint/test/expect_lint.cmake:140,04: [C0103] Invalid directory variable name "_INVALID_PRIVATE_NAME" - cmake_lint/test/expect_lint.cmake:141,04: [C0103] Invalid directory variable name "invalid_public_name" - cmake_lint/test/expect_lint.cmake:143,06: [C0103] Invalid local variable name "INVALID_LOCAL_NAME" - cmake_lint/test/expect_lint.cmake:146: [C0304] Final newline missing - - Summary - ======= - files scanned: 1 - found lint: - Convention: 40 - Error: 12 - Refactor: 4 - Warning: 3 - - -.. dynamic: lint-out-end +.. literalinclude:: bits/lint-out.txt diff --git a/cmake_format/doc/lint-implemented.rst b/cmake_format/doc/lint-implemented.rst index 351bac5..a3387be 100644 --- a/cmake_format/doc/lint-implemented.rst +++ b/cmake_format/doc/lint-implemented.rst @@ -611,3 +611,30 @@ message Use of deprecated command {:s} +----- +W0105 +----- + +message +------- + +.. code:: + + {:s} variable '{:s}' which matches a built-in except for case + + +description +----------- + + +This warning means that you are using a variable such as, +for example, `cmake_cxx_standard` which matches a builtin variable +(`CMAKE_CXX_STANDARD`) except for the case. If this was intentional, then it's +bad practice as it causes confusion (there are two variables in the namespace +with identical name except for case), though it was probably not intentional +and you probably aren't assigning to the correct variable. + +This warning may be emitted for assignment (e.g. `set()` or `list()`) as +well as for variable expansion in an argument (e.g. `"${CMAKE_Cxx_STANDARD}"`). + + diff --git a/cmake_format/doc/lint-summary.rst b/cmake_format/doc/lint-summary.rst index c4efe1e..1b258c6 100644 --- a/cmake_format/doc/lint-summary.rst +++ b/cmake_format/doc/lint-summary.rst @@ -13,73 +13,7 @@ each check is available in :ref:`lint-checks`. .. default-role:: ref -.. dynamic: lintimpl-table-begin - -+-------+--------------------------------------------------------------------+ -|`C0102`| Black listed name "{:s}" | -+-------+--------------------------------------------------------------------+ -|`C0103`| Invalid {:s} name "{:s}" | -+-------+--------------------------------------------------------------------+ -|`C0111`| Missing docstring on function or macro declaration | -+-------+--------------------------------------------------------------------+ -|`C0112`| Empty docstring on function or macro declaration | -+-------+--------------------------------------------------------------------+ -|`C0113`| Missing {:s} in statement which allows it | -+-------+--------------------------------------------------------------------+ -|`C0114`| Form descriminator hidden behind variable dereference | -+-------+--------------------------------------------------------------------+ -|`C0201`| Consider replacing custom parser logic with cmake_parse_arguments | -+-------+--------------------------------------------------------------------+ -|`C0202`| Argument name {:s} differs from existing argument only in case | -+-------+--------------------------------------------------------------------+ -|`C0301`| Line too long ({:d}/{:d}) | -+-------+--------------------------------------------------------------------+ -|`C0303`| Trailing whitespace | -+-------+--------------------------------------------------------------------+ -|`C0304`| Final newline missing | -+-------+--------------------------------------------------------------------+ -|`C0305`| {:s} newlines between statements | -+-------+--------------------------------------------------------------------+ -|`C0321`| Multiple statements on a single line | -+-------+--------------------------------------------------------------------+ -|`C0327`| Wrong line ending ({:s}) | -+-------+--------------------------------------------------------------------+ -|`E0011`| Unrecognized file option {:s} | -+-------+--------------------------------------------------------------------+ -|`E0012`| Bad option value {:s} | -+-------+--------------------------------------------------------------------+ -|`E0103`| {:s} outside of loop | -+-------+--------------------------------------------------------------------+ -|`E0108`| Duplicate argument name {:s} in function/macro definition | -+-------+--------------------------------------------------------------------+ -|`E0109`| Invalid argument name {:s} in function/macro definition | -+-------+--------------------------------------------------------------------+ -|`E1120`| Missing required positional argument | -+-------+--------------------------------------------------------------------+ -|`E1121`| Too many positional arguments | -+-------+--------------------------------------------------------------------+ -|`E1122`| Duplicate keyword argument {:s} | -+-------+--------------------------------------------------------------------+ -|`E1125`| Missing required keyword argument {:s} | -+-------+--------------------------------------------------------------------+ -|`E1126`| Invalid form descriminator | -+-------+--------------------------------------------------------------------+ -|`R0911`| Too many return statements {:d}/{:d} | -+-------+--------------------------------------------------------------------+ -|`R0912`| Too many branches {:d}/{:d} | -+-------+--------------------------------------------------------------------+ -|`R0913`| Too many named arguments {:d}/{:d} | -+-------+--------------------------------------------------------------------+ -|`R0914`| Too many local variables {:d}/{:d} | -+-------+--------------------------------------------------------------------+ -|`R0915`| Too many statements {:d}/{:d} | -+-------+--------------------------------------------------------------------+ -|`W0101`| Unreachable code | -+-------+--------------------------------------------------------------------+ -|`W0104`| Use of deprecated command {:s} | -+-------+--------------------------------------------------------------------+ - -.. dynamic: lintimpl-table-end +.. include:: bits/lintimpl-table.rst ------- Planned diff --git a/cmake_format/doc/lint-usage.rst b/cmake_format/doc/lint-usage.rst index fd67cb9..c0d2efd 100644 --- a/cmake_format/doc/lint-usage.rst +++ b/cmake_format/doc/lint-usage.rst @@ -2,189 +2,4 @@ Usage ===== -.. dynamic: lint-usage-begin - -.. code:: text - - usage: - cmake-lint [-h] - [--dump-config {yaml,json,python} | -o OUTFILE_PATH] - [-c CONFIG_FILE] - infilepath [infilepath ...] - - Check cmake listfile for lint - - positional arguments: - infilepaths - - optional arguments: - -h, --help show this help message and exit - -v, --version show program's version number and exit - -l {error,warning,info,debug}, --log-level {error,warning,info,debug} - --dump-config [{yaml,json,python}] - If specified, print the default configuration to - stdout and exit - -o OUTFILE_PATH, --outfile-path OUTFILE_PATH - Write errors to this file. Default is stdout. - -c CONFIG_FILES [CONFIG_FILES ...], --config-files CONFIG_FILES [CONFIG_FILES ...] - path to configuration file(s) - - Various configuration options/parameters for formatting: - - - Options effecting formatting.: - --line-width LINE_WIDTH - How wide to allow formatted cmake files - --tab-size TAB_SIZE How many spaces to tab for indent - --max-subgroups-hwrap MAX_SUBGROUPS_HWRAP - If an argument group contains more than this many sub- - groups (parg or kwarg groups) then force it to a - vertical layout. - --max-pargs-hwrap MAX_PARGS_HWRAP - If a positional argument group contains more than this - many arguments, then force it to a vertical layout. - --max-rows-cmdline MAX_ROWS_CMDLINE - If a cmdline positional group consumes more than this - many lines without nesting, then invalidate the layout - (and nest) - --separate-ctrl-name-with-space [SEPARATE_CTRL_NAME_WITH_SPACE] - If true, separate flow control names from their - parentheses with a space - --separate-fn-name-with-space [SEPARATE_FN_NAME_WITH_SPACE] - If true, separate function names from parentheses with - a space - --dangle-parens [DANGLE_PARENS] - If a statement is wrapped to more than one line, than - dangle the closing parenthesis on its own line. - --dangle-align {prefix,prefix-indent,child,off} - If the trailing parenthesis must be 'dangled' on its - on line, then align it to this reference: `prefix`: - the start of the statement, `prefix-indent`: the start - of the statement, plus one indentation level, `child`: - align to the column of the arguments - --min-prefix-chars MIN_PREFIX_CHARS - If the statement spelling length (including space and - parenthesis) is smaller than this amount, then force - reject nested layouts. - --max-prefix-chars MAX_PREFIX_CHARS - If the statement spelling length (including space and - parenthesis) is larger than the tab width by more than - this amount, then force reject un-nested layouts. - --max-lines-hwrap MAX_LINES_HWRAP - If a candidate layout is wrapped horizontally but it - exceeds this many lines, then reject the layout. - --line-ending {windows,unix,auto} - What style line endings to use in the output. - --command-case {lower,upper,canonical,unchanged} - Format command names consistently as 'lower' or - 'upper' case - --keyword-case {lower,upper,unchanged} - Format keywords consistently as 'lower' or 'upper' - case - --always-wrap [ALWAYS_WRAP [ALWAYS_WRAP ...]] - A list of command names which should always be wrapped - --enable-sort [ENABLE_SORT] - If true, the argument lists which are known to be - sortable will be sorted lexicographicall - --autosort [AUTOSORT] - If true, the parsers may infer whether or not an - argument list is sortable (without annotation). - --require-valid-layout [REQUIRE_VALID_LAYOUT] - By default, if cmake-format cannot successfully fit - everything into the desired linewidth it will apply - the last, most agressive attempt that it made. If this - flag is True, however, cmake-format will print error, - exit with non-zero status code, and write-out nothing - - Options affecting comment reflow and formatting.: - --bullet-char BULLET_CHAR - What character to use for bulleted lists - --enum-char ENUM_CHAR - What character to use as punctuation after numerals in - an enumerated list - --first-comment-is-literal [FIRST_COMMENT_IS_LITERAL] - If comment markup is enabled, don't reflow the first - comment block in each listfile. Use this to preserve - formatting of your copyright/license statements. - --literal-comment-pattern LITERAL_COMMENT_PATTERN - If comment markup is enabled, don't reflow any comment - block which matches this (regex) pattern. Default is - `None` (disabled). - --fence-pattern FENCE_PATTERN - Regular expression to match preformat fences in - comments default=r'^\s*([`~]{3}[`~]*)(.*)$' - --ruler-pattern RULER_PATTERN - Regular expression to match rulers in comments - default=r'^\s*[^\w\s]{3}.*[^\w\s]{3}$' - --explicit-trailing-pattern EXPLICIT_TRAILING_PATTERN - If a comment line matches starts with this pattern - then it is explicitly a trailing comment for the - preceeding argument. Default is '#<' - --hashruler-min-length HASHRULER_MIN_LENGTH - If a comment line starts with at least this many - consecutive hash characters, then don't lstrip() them - off. This allows for lazy hash rulers where the first - hash char is not separated by space - --canonicalize-hashrulers [CANONICALIZE_HASHRULERS] - If true, then insert a space between the first hash - char and remaining hash chars in a hash ruler, and - normalize its length to fill the column - --enable-markup [ENABLE_MARKUP] - enable comment markup parsing and reflow - - Options affecting the linter: - --disabled-codes [DISABLED_CODES [DISABLED_CODES ...]] - a list of lint codes to disable - --function-pattern FUNCTION_PATTERN - regular expression pattern describing valid function - names - --macro-pattern MACRO_PATTERN - regular expression pattern describing valid macro - names - --global-var-pattern GLOBAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with global scope - --internal-var-pattern INTERNAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with global scope (but internal semantic) - --local-var-pattern LOCAL_VAR_PATTERN - regular expression pattern describing valid names for - variables with local scope - --private-var-pattern PRIVATE_VAR_PATTERN - regular expression pattern describing valid names for - privatedirectory variables - --public-var-pattern PUBLIC_VAR_PATTERN - regular expression pattern describing valid names for - publicdirectory variables - --keyword-pattern KEYWORD_PATTERN - regular expression pattern describing valid names for - keywords used in functions or macros - --max-conditionals-custom-parser MAX_CONDITIONALS_CUSTOM_PARSER - In the heuristic for C0201, how many conditionals to - match within a loop in before considering the loop a - parser. - --min-statement-spacing MIN_STATEMENT_SPACING - Require at least this many newlines between statements - --max-statement-spacing MAX_STATEMENT_SPACING - Require no more than this many newlines between - statements - --max-returns MAX_RETURNS - --max-branches MAX_BRANCHES - --max-arguments MAX_ARGUMENTS - --max-localvars MAX_LOCALVARS - --max-statements MAX_STATEMENTS - - Options effecting file encoding: - --emit-byteorder-mark [EMIT_BYTEORDER_MARK] - If true, emit the unicode byte-order mark (BOM) at the - start of the file - --input-encoding INPUT_ENCODING - Specify the encoding of the input file. Defaults to - utf-8 - --output-encoding OUTPUT_ENCODING - Specify the encoding of the output file. Defaults to - utf-8. Note that cmake only claims to support utf-8 so - be careful when using anything else - -.. dynamic: lint-usage-end - +.. literalinclude:: bits/lint-usage.rst diff --git a/cmake_format/doc/parse-algorithm.rst b/cmake_format/doc/parse-algorithm.rst new file mode 100644 index 0000000..84ac52e --- /dev/null +++ b/cmake_format/doc/parse-algorithm.rst @@ -0,0 +1,9 @@ +================ +Parser Algorithm +================ + +TODO(josh): add notes regarding the standard parse function and what it does +with regard to positionals and kwargs. Then add notes about the new style +parser which uses nested function calls. Note that the motivation for this is +commands that take multiple forms. We need computational logic to look at the +first argument to determine how to parse the rest of it. diff --git a/cmake_format/doc/parse-automatic.rst b/cmake_format/doc/parse-automatic.rst new file mode 100644 index 0000000..7bfe7e3 --- /dev/null +++ b/cmake_format/doc/parse-automatic.rst @@ -0,0 +1,217 @@ +================= +Automatic Parsers +================= + +cmake provides help commands which can print out the usage information for all +of the builtin statements that it supports. You can get a list of commands +with + +.. code:: + + cmake --help-command-list + +And you can get the help text for a command with (for example): + +.. code:: + + cmake --help-command add_custom_command + +In general (but not always) the usage string is given an a restructured-text +block that looks like this: + +.. code:: + + :: + + add_custom_command(TARGET + PRE_BUILD | PRE_LINK | POST_BUILD + COMMAND command1 [ARGS] [args1...] + [COMMAND command2 [ARGS] [args2...] ...] + [BYPRODUCTS [files...]] + [WORKING_DIRECTORY dir] + [COMMENT comment] + [VERBATIM] [USES_TERMINAL]) + +The syntax of these usage strings isn't 100% consistent but if we could +generate a parser that even understands *most* of these strings then that +would greatly reduce the maintenance load. + + +-------------- +Expect Objects +-------------- + +The output of the specification parser is an "Expect Tree". A tree of +objects representing what is expected from a statement. If a sequence +of tokens satisfies an expected subtree then the a corresponding parse +tree is generated. If a mandatory expected subtree is not satisfied +then an error is generated. If an optional expected subtree is not +satisfied then the next sibling is tried. + + +------------ +Case studies +------------ + + +Inconsistent usage of angle brackets +==================================== + +Sometimes mandatory arguments are shown in angle brackets, sometimes not. I +can't really figure a pattern for when they are used and when they are not. + + +Ellipses +======== + +Whether or not there is a space between an elipsis and the preceeding token +seems to imply something about what is repeated. + +:: + + add_custom_command(TARGET + PRE_BUILD | PRE_LINK | POST_BUILD + COMMAND command1 [ARGS] [args1...] + [COMMAND command2 [ARGS] [args2...] ...] + [BYPRODUCTS [files...]] + [WORKING_DIRECTORY dir] + [COMMENT comment] + [VERBATIM] [USES_TERMINAL]) + + +Note that the elipsis for "COMMAND" is inside the bracket above, but is +outside the bracket here: + +:: + + add_dependencies( []...) + + +Choices +======= + +Pipe character is used to separate choices. + +:: + + configure_file( + [COPYONLY] [ESCAPE_QUOTES] [@ONLY] + [NEWLINE_STYLE [UNIX|DOS|WIN32|LF|CRLF] ]) + +Sometimes it's a mandatory choice + +:: + + ctest_test([BUILD ] [APPEND] + [START ] + [END ] + [STRIDE ] + [EXCLUDE ] + [INCLUDE ] + [EXCLUDE_LABEL ] + [INCLUDE_LABEL ] + [EXCLUDE_FIXTURE ] + [EXCLUDE_FIXTURE_SETUP ] + [EXCLUDE_FIXTURE_CLEANUP ] + [PARALLEL_LEVEL ] + [TEST_LOAD ] + [SCHEDULE_RANDOM ] + [STOP_TIME ] + [RETURN_VALUE ] + [CAPTURE_CMAKE_ERROR ] + [QUIET] + ) + +Sometimes the choice is among literals, in which case there are no surrounding +brackets. + +:: + + file(GLOB + [LIST_DIRECTORIES true|false] [RELATIVE ] + [...]) + + +Manditory Sequence +================== + +In this case the literal pattern is listed inside the mandatory group pattern +(angle brackets). + +:: + + file(GENERATE OUTPUT output-file + + [CONDITION expression]) + +This one is pretty complex, and also demonstrates the nested bracket usage. +I think the indication here is that "you must have one of these choices. + +:: + + get_property( + | + SOURCE | + INSTALL | + TEST | + CACHE | + VARIABLE> + PROPERTY + [SET | DEFINED | BRIEF_DOCS | FULL_DOCS]) + +Nested Optionals +================ + +:: + + install(TARGETS targets... [EXPORT ] + [[ARCHIVE|LIBRARY|RUNTIME|OBJECTS|FRAMEWORK|BUNDLE| + PRIVATE_HEADER|PUBLIC_HEADER|RESOURCE] + [DESTINATION ] + [PERMISSIONS permissions...] + [CONFIGURATIONS [Debug|Release|...]] + [COMPONENT ] + [OPTIONAL] [EXCLUDE_FROM_ALL] + [NAMELINK_ONLY|NAMELINK_SKIP] + ] [...] + [INCLUDES DESTINATION [ ...]] + ) + + +Multiple Forms +============== + +:: + + string(SUBSTRING ) + string(STRIP ) + string(GENEX_STRIP ) + string(COMPARE LESS ) + + +---------- +Conclusion +---------- + +After implementing a prototype parser and testing it on some of the above cases +it is clear that the help text is not very consistent and is likely to be very +challenging to get an implementation that works reliabily and knows when it +fails. For example: + +:: + + add_custom_command(TARGET + PRE_BUILD | PRE_LINK | POST_BUILD + COMMAND command1 [ARGS] [args1...] + [COMMAND command2 [ARGS] [args2...] ...] + [BYPRODUCTS [files...]] + [WORKING_DIRECTORY dir] + [COMMENT comment] + [VERBATIM] [USES_TERMINAL]) + +In this form of the command, the `PRE_BUILD` `PRE_LINK` or `POST_BUILD` +argument is required. Normally it seems like they would put this in angle +brackets as `` but they do not. So it's +ambiguous where the pipes are splitting and what the groups are. diff --git a/cmake_format/doc/parse-tree.rst b/cmake_format/doc/parse-tree.rst new file mode 100644 index 0000000..d82ff30 --- /dev/null +++ b/cmake_format/doc/parse-tree.rst @@ -0,0 +1,412 @@ +================ +CMake Parse Tree +================ + +This document is intended to describe the high level organization of how +cmake listfiles are parsed and organized into an abstract syntax tree. + +Digestion and formatting of a listfile is done in four phases: + + * tokenization + * parsing + * layout tree construction + * layout / reflow + +--------- +Tokenizer +--------- + +Listfiles are first digested into a sequence of tokens. The tokenizer is +implemented in `lexer.py` an defines the following types of tokens: + ++------------------+--------------------------------------+-------------------+ +| Token Type | Description | Example | ++------------------+--------------------------------------+-------------------+ +| QUOTED_LITERAL | A single or double quoted string, | ``"foo"`` | +| | from the first quote to the first | ``'bar'`` | +| | subsequent un-escaped quote | | ++------------------+--------------------------------------+-------------------+ +| BRACKET_ARGUMENT | A bracket-quoted argument of a |``[=[hello foo]=]``| +| | cmake-statement | | ++------------------+--------------------------------------+-------------------+ +| NUMBER | Unquoted numeric literal | ``1234`` | ++------------------+--------------------------------------+-------------------+ +| LEFT_PAREN | A left parenthesis | ``(`` | ++------------------+--------------------------------------+-------------------+ +| RIGHT_PAREN | A right parenthesis | ``)`` | ++------------------+--------------------------------------+-------------------+ +| WORD | An unquoted literal string which | ``foo`` | +| | matches lexical rules such that it | ``foo_bar`` | +| | could be a cmake entity name, such | | +| | as the name of a function or | | +| | variable | | ++------------------+--------------------------------------+-------------------+ +| DEREF | A variable dereference expression, | ``${foo}`` | +| | from the dollar sign up to the outer | ``${foo_${bar}}`` | +| | most right curly brace | | ++------------------+--------------------------------------+-------------------+ +| NEWLINE | A single carriage return, newline or | | +| | (carriage-return, newline) pair | | ++------------------+--------------------------------------+-------------------+ +| WHITESPACE | A continuous sequence of space, tab | | +| | or other ascii whitespace | | ++------------------+--------------------------------------+-------------------+ +| BRACKET_COMMENT | A bracket-quoted comment string |``#[=[hello]=]`` | ++------------------+--------------------------------------+-------------------+ +| COMMENT | A single line starting with a hash |``# hello world`` | ++------------------+--------------------------------------+-------------------+ +| UNQUOTED_LITERAL | A sequence of non-whitespace | ``--verbose`` | +| | characters used as a cmake argument | | +| | but not satisfying the requirements | | +| | of a cmake name | | ++------------------+--------------------------------------+-------------------+ +| FORMAT_OFF | A special comment disabling | ``cmake-format:`` | +| | cmake-format temporarily | `` off`` | ++------------------+--------------------------------------+-------------------+ +| FORMAT_OFF | A special comment re-enabling | ``cmake-format:`` | +| | | `` on`` | ++------------------+--------------------------------------+-------------------+ + +Each token covers a continuous sequence of characters of the input file. +Futhermore, the sequence of tokens digest from the file covers the entire range +of infile offsets. The ``Token`` object stores information about the input file +byte offset, line number, and column number of it's start location. Note that +for ``utf-8`` input where a character may be composed of more than one byte, +the ``(row, col)`` location is the location of the character while the +``offset`` is the index of the first byte of the character. + +You can inspect the tokenization of a listfile by executing ``cmake-format`` +with ``--dump lex``. For example: + +.. dynamic: dump-example-lex-begin + +.. code:: text + + Token(type=NEWLINE, content='\n', line=1, col=0) + Token(type=WORD, content='cmake_minimum_required', line=2, col=0) + Token(type=LEFT_PAREN, content='(', line=2, col=22) + Token(type=WORD, content='VERSION', line=2, col=23) + Token(type=WHITESPACE, content=' ', line=2, col=30) + Token(type=UNQUOTED_LITERAL, content='3.5', line=2, col=31) + Token(type=RIGHT_PAREN, content=')', line=2, col=34) + Token(type=NEWLINE, content='\n', line=2, col=35) + Token(type=WORD, content='project', line=3, col=0) + Token(type=LEFT_PAREN, content='(', line=3, col=7) + Token(type=WORD, content='demo', line=3, col=8) + Token(type=RIGHT_PAREN, content=')', line=3, col=12) + Token(type=NEWLINE, content='\n', line=3, col=13) + Token(type=WORD, content='if', line=4, col=0) + Token(type=LEFT_PAREN, content='(', line=4, col=2) + Token(type=WORD, content='FOO', line=4, col=3) + Token(type=WHITESPACE, content=' ', line=4, col=6) + Token(type=WORD, content='AND', line=4, col=7) + Token(type=WHITESPACE, content=' ', line=4, col=10) + Token(type=LEFT_PAREN, content='(', line=4, col=11) + Token(type=WORD, content='BAR', line=4, col=12) + Token(type=WHITESPACE, content=' ', line=4, col=15) + Token(type=WORD, content='OR', line=4, col=16) + Token(type=WHITESPACE, content=' ', line=4, col=18) + Token(type=WORD, content='BAZ', line=4, col=19) + Token(type=RIGHT_PAREN, content=')', line=4, col=22) + Token(type=RIGHT_PAREN, content=')', line=4, col=23) + Token(type=NEWLINE, content='\n', line=4, col=24) + Token(type=WHITESPACE, content=' ', line=5, col=0) + Token(type=WORD, content='add_library', line=5, col=2) + Token(type=LEFT_PAREN, content='(', line=5, col=13) + Token(type=WORD, content='hello', line=5, col=14) + Token(type=WHITESPACE, content=' ', line=5, col=19) + Token(type=UNQUOTED_LITERAL, content='hello.cc', line=5, col=20) + Token(type=RIGHT_PAREN, content=')', line=5, col=28) + Token(type=NEWLINE, content='\n', line=5, col=29) + Token(type=WORD, content='endif', line=6, col=0) + Token(type=LEFT_PAREN, content='(', line=6, col=5) + Token(type=RIGHT_PAREN, content=')', line=6, col=6) + Token(type=NEWLINE, content='\n', line=6, col=7) + +.. dynamic: dump-example-lex-end + +------------------- +Parser: Syntax Tree +------------------- + +``cmake-format`` parses the token stream in a single pass. +The state machine of the parser is maintained by the program stack +(i.e. the parse functions are called recursively) and each node type in the +tree has it's own parse function. + +There are fourteen types of nodes in the parse tree. They are described below +along with the list of possible child node types. + + +Node Types +========== + ++--------------+---------------------------------------------+----------------+ +| Node Type | Description | Allowed | +| | | Children | ++--------------+---------------------------------------------+----------------+ +| BODY | A generic section of a cmake document. This | COMMENT | +| | node type is found at the root of the parse | STATEMENT | +| | tree and within conditional/flow control | WHITESPACE | +| | statements | | +| | | | ++--------------+---------------------------------------------+----------------+ +| WHITESPACE | A consecutive sequence of whitespace tokens | (none) | +| | between any two other types of nodes. | | ++--------------+---------------------------------------------+----------------+ +| COMMENT | A sequence of one or more comment lines. | (token) | +| | The node consistes of all consecutive | | +| | comment lines unbroken by additional | | +| | newlines or a single BRACKET_COMMENT token. | | ++--------------+---------------------------------------------+----------------+ +| STATEMENT | A cmake statement (i.e. function call) | ARGGROUP | +| | | COMMENT | +| | | FUNNAME | ++--------------+---------------------------------------------+----------------+ +| FLOW_CONTROL | Two or more cmake statements and their | STATEMENT | +| | nested bodies representing a flow control | BODY | +| | construct (i.e. ``if`` or ``foreach``). | | ++--------------+---------------------------------------------+----------------+ +| ARGGROUP | A top-level collection of one or more | PARGGROUP | +| | positional, kwarg, or flag groups | KWARGGROUP | +| | | PARENGROUP | +| | | FLAGGROUP | +| | | COMMENT | ++--------------+---------------------------------------------+----------------+ +| PARGGROUP | A grouping of one or more positional | ARGUMENT | +| | arguments. | COMMENT | ++--------------+---------------------------------------------+----------------+ +| FLAGGROUP | A grouping of one or more positional | FLAG | +| | arguments, each of which is a flag | COMMENT | ++--------------+---------------------------------------------+----------------+ +| KWARGGROUP | A KEYWORD group, starting with the keyword | KEYWORD | +| | and ending with the last argument associated| ARGGROUP | +| | with that keyword | | ++--------------+---------------------------------------------+----------------+ +| PARENGROUP | A parenthetical group, starting with a left | ARGGROUP | +| | parenthesis and ending with the matching | | +| | right parenthesis | | ++--------------+---------------------------------------------+----------------+ +| FUNNAME | Consists of a single token containing the | (token) | +| | name of the function/command in a statement | | +| | with that keyword | | ++--------------+---------------------------------------------+----------------+ +| ARGUMENT | Consists of a single token, containing the | (token) | +| | literal argument of a statement, and | COMMENT | +| | optionally a comment associated with it | | ++--------------+---------------------------------------------+----------------+ +| KEYWORD | Consists of a single token, containing the | (token) | +| | literal keyword of a keyword group, and | COMMENT | +| | optionally a comment associated with it | | ++--------------+---------------------------------------------+----------------+ +| FLAG | Consists of a single token, containing the | (token) | +| | literal keyword of a statment flag, and | COMMENT | +| | optionally a comment associated with it | | ++--------------+---------------------------------------------+----------------+ +| ONOFFSWITCH | Consists of a single token, containing the | (token) | +| | sentinal comment line ``# cmake-format: on``| | +| | or ``# cmake-format: off``. | | ++--------------+---------------------------------------------+----------------+ + +You can inspect the parse tree of a listfile by ``cmake-format`` with +``--dump parse``. For example: + +.. dynamic: dump-example-parse-begin + +.. code:: text + + └─ BODY: 1:0 + ├─ WHITESPACE: 1:0 + │ └─ Token(type=NEWLINE, content='\n', line=1, col=0) + ├─ STATEMENT: 2:0 + │ ├─ FUNNAME: 2:0 + │ │ └─ Token(type=WORD, content='cmake_minimum_required', line=2, col=0) + │ ├─ LPAREN: 2:22 + │ │ └─ Token(type=LEFT_PAREN, content='(', line=2, col=22) + │ ├─ ARGGROUP: 2:23 + │ │ └─ KWARGGROUP: 2:23 + │ │ ├─ KEYWORD: 2:23 + │ │ │ └─ Token(type=WORD, content='VERSION', line=2, col=23) + │ │ ├─ Token(type=WHITESPACE, content=' ', line=2, col=30) + │ │ └─ ARGGROUP: 2:31 + │ │ └─ PARGGROUP: 2:31 + │ │ └─ ARGUMENT: 2:31 + │ │ └─ Token(type=UNQUOTED_LITERAL, content='3.5', line=2, col=31) + │ └─ RPAREN: 2:34 + │ └─ Token(type=RIGHT_PAREN, content=')', line=2, col=34) + ├─ WHITESPACE: 2:35 + │ └─ Token(type=NEWLINE, content='\n', line=2, col=35) + ├─ STATEMENT: 3:0 + │ ├─ FUNNAME: 3:0 + │ │ └─ Token(type=WORD, content='project', line=3, col=0) + │ ├─ LPAREN: 3:7 + │ │ └─ Token(type=LEFT_PAREN, content='(', line=3, col=7) + │ ├─ ARGGROUP: 3:8 + │ │ └─ PARGGROUP: 3:8 + │ │ └─ ARGUMENT: 3:8 + │ │ └─ Token(type=WORD, content='demo', line=3, col=8) + │ └─ RPAREN: 3:12 + │ └─ Token(type=RIGHT_PAREN, content=')', line=3, col=12) + ├─ WHITESPACE: 3:13 + │ └─ Token(type=NEWLINE, content='\n', line=3, col=13) + ├─ FLOW_CONTROL: 4:0 + │ ├─ STATEMENT: 4:0 + │ │ ├─ FUNNAME: 4:0 + │ │ │ └─ Token(type=WORD, content='if', line=4, col=0) + │ │ ├─ LPAREN: 4:2 + │ │ │ └─ Token(type=LEFT_PAREN, content='(', line=4, col=2) + │ │ ├─ ARGGROUP: 4:3 + │ │ │ ├─ PARGGROUP: 4:3 + │ │ │ │ ├─ ARGUMENT: 4:3 + │ │ │ │ │ └─ Token(type=WORD, content='FOO', line=4, col=3) + │ │ │ │ └─ Token(type=WHITESPACE, content=' ', line=4, col=6) + │ │ │ └─ KWARGGROUP: 4:7 + │ │ │ ├─ KEYWORD: 4:7 + │ │ │ │ └─ Token(type=WORD, content='AND', line=4, col=7) + │ │ │ ├─ Token(type=WHITESPACE, content=' ', line=4, col=10) + │ │ │ └─ ARGGROUP: 4:11 + │ │ │ └─ PARENGROUP: 4:11 + │ │ │ ├─ LPAREN: 4:11 + │ │ │ │ └─ Token(type=LEFT_PAREN, content='(', line=4, col=11) + │ │ │ ├─ ARGGROUP: 4:12 + │ │ │ │ ├─ PARGGROUP: 4:12 + │ │ │ │ │ ├─ ARGUMENT: 4:12 + │ │ │ │ │ │ └─ Token(type=WORD, content='BAR', line=4, col=12) + │ │ │ │ │ └─ Token(type=WHITESPACE, content=' ', line=4, col=15) + │ │ │ │ └─ KWARGGROUP: 4:16 + │ │ │ │ ├─ KEYWORD: 4:16 + │ │ │ │ │ └─ Token(type=WORD, content='OR', line=4, col=16) + │ │ │ │ ├─ Token(type=WHITESPACE, content=' ', line=4, col=18) + │ │ │ │ └─ ARGGROUP: 4:19 + │ │ │ │ └─ PARGGROUP: 4:19 + │ │ │ │ └─ ARGUMENT: 4:19 + │ │ │ │ └─ Token(type=WORD, content='BAZ', line=4, col=19) + │ │ │ └─ RPAREN: 4:22 + │ │ │ └─ Token(type=RIGHT_PAREN, content=')', line=4, col=22) + │ │ └─ RPAREN: 4:23 + │ │ └─ Token(type=RIGHT_PAREN, content=')', line=4, col=23) + │ ├─ BODY: 4:24 + │ │ ├─ WHITESPACE: 4:24 + │ │ │ ├─ Token(type=NEWLINE, content='\n', line=4, col=24) + │ │ │ └─ Token(type=WHITESPACE, content=' ', line=5, col=0) + │ │ ├─ STATEMENT: 5:2 + │ │ │ ├─ FUNNAME: 5:2 + │ │ │ │ └─ Token(type=WORD, content='add_library', line=5, col=2) + │ │ │ ├─ LPAREN: 5:13 + │ │ │ │ └─ Token(type=LEFT_PAREN, content='(', line=5, col=13) + │ │ │ ├─ ARGGROUP: 5:14 + │ │ │ │ ├─ PARGGROUP: 5:14 + │ │ │ │ │ ├─ ARGUMENT: 5:14 + │ │ │ │ │ │ └─ Token(type=WORD, content='hello', line=5, col=14) + │ │ │ │ │ └─ Token(type=WHITESPACE, content=' ', line=5, col=19) + │ │ │ │ └─ PARGGROUP: 5:20 + │ │ │ │ └─ ARGUMENT: 5:20 + │ │ │ │ └─ Token(type=UNQUOTED_LITERAL, content='hello.cc', line=5, col=20) + │ │ │ └─ RPAREN: 5:28 + │ │ │ └─ Token(type=RIGHT_PAREN, content=')', line=5, col=28) + │ │ └─ WHITESPACE: 5:29 + │ │ └─ Token(type=NEWLINE, content='\n', line=5, col=29) + │ └─ STATEMENT: 6:0 + │ ├─ FUNNAME: 6:0 + │ │ └─ Token(type=WORD, content='endif', line=6, col=0) + │ ├─ LPAREN: 6:5 + │ │ └─ Token(type=LEFT_PAREN, content='(', line=6, col=5) + │ ├─ ARGGROUP: 0:0 + │ └─ RPAREN: 6:6 + │ └─ Token(type=RIGHT_PAREN, content=')', line=6, col=6) + └─ WHITESPACE: 6:7 + └─ Token(type=NEWLINE, content='\n', line=6, col=7) + +.. dynamic: dump-example-parse-end + +---------------------- +Formatter: Layout Tree +---------------------- + +As of version ``0.4.0``, ``cmake-format`` will create a tree structure parallel +to the parse tree and called the "layout tree". Each node in the layout tree +points to at most one node in the parse tree. The structure of the layout tree +is essentially the same as the parse tree with the following exceptions: + +1. The primary argument group of a statement is expanded, so that the possible + children of a ``STATEMENT`` layout node are: ``ARGGROUP``, ``ARGUMENT``, + ``COMMENT``, ``FLAG``, ``FUNNAME``, ``KWARGROUP``. +2. ``WHITESPACE`` nodes containing less than two newlines are dropped, and not + represented in the layout tree. + +You can inspect the layout tree of a listfile by ``cmake-format`` with +``--dump layout``. For example: + +.. dynamic: dump-example-layout-begin + +.. code:: text + + └─ BODY,(passno=0,wrap=F,ok=T) pos:(0,0) colextent:35 + ├─ STATEMENT,(passno=0,wrap=F,ok=T) pos:(0,0) colextent:35 + │ ├─ FUNNAME,(passno=0,wrap=F,ok=T) pos:(0,0) colextent:22 + │ ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(0,22) colextent:23 + │ ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(0,23) colextent:34 + │ │ └─ KWARGGROUP,(passno=0,wrap=F,ok=T) pos:(0,23) colextent:34 + │ │ ├─ KEYWORD,(passno=0,wrap=F,ok=T) pos:(0,23) colextent:30 + │ │ └─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(0,31) colextent:34 + │ │ └─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(0,31) colextent:34 + │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(0,31) colextent:34 + │ └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(0,34) colextent:35 + ├─ STATEMENT,(passno=0,wrap=F,ok=T) pos:(1,0) colextent:13 + │ ├─ FUNNAME,(passno=0,wrap=F,ok=T) pos:(1,0) colextent:7 + │ ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(1,7) colextent:8 + │ ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(1,8) colextent:12 + │ │ └─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(1,8) colextent:12 + │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(1,8) colextent:12 + │ └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(1,12) colextent:13 + └─ FLOW_CONTROL,(passno=0,wrap=F,ok=T) pos:(2,0) colextent:29 + ├─ STATEMENT,(passno=0,wrap=F,ok=T) pos:(2,0) colextent:24 + │ ├─ FUNNAME,(passno=0,wrap=F,ok=T) pos:(2,0) colextent:2 + │ ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(2,2) colextent:3 + │ ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,3) colextent:23 + │ │ ├─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,3) colextent:6 + │ │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(2,3) colextent:6 + │ │ └─ KWARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,7) colextent:23 + │ │ ├─ KEYWORD,(passno=0,wrap=F,ok=T) pos:(2,7) colextent:10 + │ │ └─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,11) colextent:23 + │ │ └─ PARENGROUP,(passno=0,wrap=F,ok=T) pos:(2,11) colextent:23 + │ │ ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(2,11) colextent:12 + │ │ ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,12) colextent:22 + │ │ │ ├─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,12) colextent:15 + │ │ │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(2,12) colextent:15 + │ │ │ └─ KWARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,16) colextent:22 + │ │ │ ├─ KEYWORD,(passno=0,wrap=F,ok=T) pos:(2,16) colextent:18 + │ │ │ └─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,19) colextent:22 + │ │ │ └─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(2,19) colextent:22 + │ │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(2,19) colextent:22 + │ │ └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(2,22) colextent:23 + │ └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(2,23) colextent:24 + ├─ BODY,(passno=0,wrap=F,ok=T) pos:(3,2) colextent:29 + │ └─ STATEMENT,(passno=0,wrap=F,ok=T) pos:(3,2) colextent:29 + │ ├─ FUNNAME,(passno=0,wrap=F,ok=T) pos:(3,2) colextent:13 + │ ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(3,13) colextent:14 + │ ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(3,14) colextent:28 + │ │ ├─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(3,14) colextent:19 + │ │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(3,14) colextent:19 + │ │ └─ PARGGROUP,(passno=0,wrap=F,ok=T) pos:(3,20) colextent:28 + │ │ └─ ARGUMENT,(passno=0,wrap=F,ok=T) pos:(3,20) colextent:28 + │ └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(3,28) colextent:29 + └─ STATEMENT,(passno=0,wrap=F,ok=T) pos:(4,0) colextent:7 + ├─ FUNNAME,(passno=0,wrap=F,ok=T) pos:(4,0) colextent:5 + ├─ LPAREN,(passno=0,wrap=F,ok=T) pos:(4,5) colextent:6 + ├─ ARGGROUP,(passno=0,wrap=F,ok=T) pos:(4,6) colextent:6 + └─ RPAREN,(passno=0,wrap=F,ok=T) pos:(4,6) colextent:7 + +.. dynamic: dump-example-layout-end + +------------ +Example file +------------ + +The example file used to create the tree dumps above is::: + + cmake_minimum_required(VERSION 3.5) + project(demo) + if(FOO AND (BAR OR BAZ)) + add_library(hello hello.cc) + endif() diff --git a/cmake_format/doc/release_notes.rst b/cmake_format/doc/release_notes.rst index 319d0c5..afce38d 100644 --- a/cmake_format/doc/release_notes.rst +++ b/cmake_format/doc/release_notes.rst @@ -8,6 +8,26 @@ some high level notes and highlights from each release. v0.6 series =========== +------ +v0.6.8 +------ + +There is now an embeded a database of known variables and properties. +``cmake-lint`` uses this database to implement checks on assignment/use of +variables that are likely to be typos of a builtin variable name. There are +also two new configuration options ``vartags`` and ``proptags`` that can +be used to affect how the parser and formatter treat certain variables and +properties. + +Line comments within a statement are now consumed the same as line comments at +block-scope. This means that your multiline mid-statement comments will be +reflowed whereas they would previously have been left alone. + +The CI Build has gotten a little more complicated. Generated documentation +sources are no longer committed to the repository. Instead, they are pushed to +a separate staging repository from which the read-the-docs pages are built. + + ------ v0.6.7 ------ diff --git a/cmake_format/doc/requirements.txt b/cmake_format/doc/requirements.txt new file mode 100644 index 0000000..c620156 --- /dev/null +++ b/cmake_format/doc/requirements.txt @@ -0,0 +1,2 @@ +buntstrap==0.1.5 +uchroot==0.1.4 diff --git a/cmake_format/doc/usage.rst b/cmake_format/doc/usage.rst index e0f9acf..44badc2 100644 --- a/cmake_format/doc/usage.rst +++ b/cmake_format/doc/usage.rst @@ -2,7 +2,7 @@ Usage ===== -.. dynamic: usage-begin +.. dynamic: format-usage-begin .. code:: text @@ -50,6 +50,12 @@ Usage Various configuration options/parameters for formatting: + Options affecting listfile parsing: + --vartags [VARTAGS [VARTAGS ...]] + Specify variable tags. + --proptags [PROPTAGS [PROPTAGS ...]] + Specify property tags. + Options effecting formatting.: --line-width LINE_WIDTH How wide to allow formatted cmake files @@ -113,6 +119,15 @@ Usage the last, most agressive attempt that it made. If this flag is True, however, cmake-format will print error, exit with non-zero status code, and write-out nothing + --allow-superfluous-newlines ALLOW_SUPERFLUOUS_NEWLINES + Allow up to this many superfluous newlines between + elements at block level. In other words, don't + collapse whitespace up to (allow_superfluous_newlines + + 1) lines long. + --numlines-pre-statement-comment NUMLINES_PRE_STATEMENT_COMMENT + Output additional newlines to increase visual + separation before a statement which is immediately + preceeded by a block comment Options affecting comment reflow and formatting.: --bullet-char BULLET_CHAR @@ -204,4 +219,4 @@ Usage utf-8. Note that cmake only claims to support utf-8 so be careful when using anything else -.. dynamic: usage-end +.. dynamic: format-usage-end diff --git a/cmake_format/formatter.py b/cmake_format/formatter.py index d54a82b..8015325 100644 --- a/cmake_format/formatter.py +++ b/cmake_format/formatter.py @@ -70,10 +70,11 @@ def get_comment_lines(config, node): return inlines -def format_comment_lines(node, config, line_width): +def format_comment_lines(node, stack_context, line_width): """ Reflow comment lines into the given line width, parsing markup as necessary. """ + config = stack_context.config inlines = get_comment_lines(config, node) if (isinstance(node, simple_nodes.CommentNode) and node.is_explicit_trailing): @@ -89,9 +90,9 @@ def format_comment_lines(node, config, line_width): if literal_comment_regex.match('\n'.join(inlines)): return [prefix + line.rstrip() for line in inlines] - if node.children[0] is config.first_token and ( + if node.children[0] is stack_context.first_token and ( config.markup.first_comment_is_literal - or config.first_token.spelling.startswith("#!")): + or stack_context.first_token.spelling.startswith("#!")): return [prefix + line.rstrip() for line in inlines] items = markup.parse(inlines, config) @@ -209,9 +210,10 @@ class StackContext(object): through all of the nested :code:`reflow()` function calls. """ - def __init__(self, config): + def __init__(self, config, first_token=None): self.config = config self.node_path = [] + self.first_token = first_token @contextlib.contextmanager def push_node(self, node): @@ -262,6 +264,7 @@ def __init__(self, pnode): # the current depth self.statement_terminal = False + self._parent = None self._children = [] self._passno = -1 @@ -269,6 +272,7 @@ def __init__(self, pnode): # different than just "position" + "size" in the case that the node # contains multiline string or bracket arguments self._colextent = 0 + self._rowextent = 0 # Depth of this node in the subtree starting at the statement node. # If this node is not in a statement subtree the value is zero. @@ -293,6 +297,20 @@ def __init__(self, pnode): assert isinstance(pnode, TreeNode) + def _index_in_parent(self): + for idx, child in enumerate(self._parent.children): + if child is self: + return idx + return -1 + + def next_sibling(self): + if self._parent is None: + return None + next_idx = self._index_in_parent() + 1 + if next_idx >= len(self._parent.children): + return None + return self._parent.children[next_idx] + @property def name(self): """ @@ -349,13 +367,17 @@ def children(self): """ return self._children + @property + def rowextent(self): + return self._rowextent + def __repr__(self): boolmap = {True: "T", False: "F"} - return "{},(passno={},wrap={},ok={}) pos:({},{}) colextent:{}".format( - self.node_type.name, + return "{}({}),(passno={},wrap={},ok={}) pos:({},{}) ext:({},{})".format( + self.__class__.__name__, self.node_type.name, self._passno, boolmap[self._wrap], boolmap[self._reflow_valid], self.position[0], self.position[1], - self.colextent) + self.rowextent, self.colextent) def has_terminal_comment(self): """ @@ -390,6 +412,10 @@ def lock(self, config, stmt_depth=0): self._children = tuple(self._children) self._locked = True + for child in self._children: + # pylint: disable=protected-access + child._parent = self + if self.node_type == NodeType.STATEMENT: nextdepth = 1 elif stmt_depth > 0: @@ -443,6 +469,7 @@ def reflow(self, stack_context, cursor, parent_passno=0): (re-)compute the layout of this node under the assumption that it should be placed at the given `cursor` on the current `parent_passno`. """ + assert self._locked assert isinstance(self.pnode, TreeNode) @@ -1096,7 +1123,8 @@ def _reflow(self, stack_context, cursor, passno): if self.statement_terminal and not children: child.statement_terminal = True - cursor = child.reflow(stack_context, cursor, passno) + input_cursor = cursor + cursor = child.reflow(stack_context, input_cursor, passno) if (not cursor_is_at_column) and (not self._wrap): # If we are in horizontal wrapping mode, then we need to look at a @@ -1129,9 +1157,8 @@ def _reflow(self, stack_context, cursor, passno): if needs_wrap: rowcount += 1 - column_cursor[0] = cursor[0] + 1 - cursor = Cursor(*column_cursor) - cursor = child.reflow(stack_context, cursor, passno) + column_cursor[0] = input_cursor[0] + 1 + cursor = child.reflow(stack_context, column_cursor, passno) # NOTE(josh): we must keep updating the extent after each child because # the child might be an argument with a multiline string or a bracket @@ -1139,6 +1166,7 @@ def _reflow(self, stack_context, cursor, passno): self._reflow_valid &= child.reflow_valid self._colextent = max(self._colextent, child.colextent) + self._rowextent = rowcount # NOTE(josh): there is a subtle distinction between invalidating a reflow # and forcing mode=vertical. The difference is whether or not a parent # node has to advance it's decision state. If we force to vertical at @@ -1163,7 +1191,7 @@ def count_subgroups(children): if child.node_type in (NodeType.KWARGGROUP, NodeType.PARGGROUP, NodeType.PARENGROUP): numgroups += 1 - elif child.node_type is NodeType.COMMENT: + elif child.node_type in (NodeType.COMMENT, NodeType.ONOFFSWITCH): continue else: raise ValueError( @@ -1475,6 +1503,10 @@ class CommentNode(LayoutNode): acts like an argument comment. """ + def __init__(self, pnode): + super(CommentNode, self).__init__(pnode) + self._lines = [] + def is_tag(self): return comment_is_tag(self.pnode.children[0]) @@ -1504,7 +1536,9 @@ def _reflow(self, stack_context, cursor, passno): return cursor allocation = config.format.linewidth - cursor[1] - lines = list(format_comment_lines(self.pnode, config, allocation)) + with stack_context.push_node(self): + self._lines = lines = list( + format_comment_lines(self.pnode, stack_context, allocation)) self._colextent = cursor[1] + max(len(line) for line in lines) increment = (len(lines) - 1, len(lines[-1])) @@ -1520,9 +1554,7 @@ def write(self, config, ctx): content = normalize_line_endings(self.pnode.children[0].spelling) ctx.outfile.write_at(self.position, content) else: - allocation = config.format.linewidth - self.position[1] - lines = list(format_comment_lines(self.pnode, config, allocation)) - for idx, line in enumerate(lines): + for idx, line in enumerate(self._lines): ctx.outfile.write_at(self.position + (idx, 0), line) @@ -1535,8 +1567,7 @@ def _reflow(self, stack_context, cursor, passno): """ Compute the size of a whitespace block """ - self._colextent = 0 - return cursor + return cursor.clone() def write(self, config, ctx): return @@ -1574,7 +1605,7 @@ def create_box_tree(pnode): return layout_root -def layout_tree(parsetree_root, config, linewidth=None): +def layout_tree(parsetree_root, config, linewidth=None, first_token=None): """ Top-level function to construct a layout tree from a parse tree, and then iterate through layout passes until the entire tree is satisfactory. Returns @@ -1586,7 +1617,7 @@ def layout_tree(parsetree_root, config, linewidth=None): root_box = create_box_tree(parsetree_root) root_box.lock(config) - stack_context = StackContext(config) + stack_context = StackContext(config, first_token) root_box.reflow(stack_context, Cursor(0, 0)) return root_box diff --git a/cmake_format/genparsers.py b/cmake_format/genparsers.py index adf1ccd..0e5abcf 100644 --- a/cmake_format/genparsers.py +++ b/cmake_format/genparsers.py @@ -179,7 +179,6 @@ def process_file(config, infile_content): config.set_line_ending(detected) tokens = lexer.tokenize(infile_content) - config.first_token = lexer.get_first_non_whitespace_token(tokens) parse_db = parse_funs.get_parse_db() parse_db.update(parse_funs.get_legacy_parse(config.parse.fn_spec).kwargs) ctx = parse.ParseContext(parse_db, config=config) diff --git a/cmake_format/layout_tests.py b/cmake_format/layout_tests.py index 723eca3..58e786f 100644 --- a/cmake_format/layout_tests.py +++ b/cmake_format/layout_tests.py @@ -781,8 +781,6 @@ def test_keyword_comment(self): (NodeType.KEYWORD, 0, 2, 2, 12, []), (NodeType.ARGGROUP, 4, 2, 13, 53, [ (NodeType.COMMENT, 0, 2, 13, 53, []), - (NodeType.COMMENT, 0, 3, 13, 52, []), - (NodeType.COMMENT, 0, 4, 13, 53, []), (NodeType.PARGGROUP, 0, 5, 13, 43, [ (NodeType.ARGUMENT, 0, 5, 13, 43, []), ]), diff --git a/cmake_format/lexer.py b/cmake_format/lexer.py index 0fc0b44..0a65ff6 100644 --- a/cmake_format/lexer.py +++ b/cmake_format/lexer.py @@ -190,11 +190,6 @@ def tokenize(contents): contents = contents[1:] tokens, remainder = scanner.scan(contents) - assert not remainder, "Unparsed tokens: {}".format(remainder) - if remainder: - raise common.UserError( - "Lexer Error: failed to tokenize input starting at: \n {}" - .format(remainder)) # Now add line, column, and serial number to token objects. We get lineno # by maintaining a running count of newline characters encountered among @@ -224,6 +219,11 @@ def tokenize(contents): begin=begin, end=SourceLocation((lineno, col, offset)))) + if remainder: + raise common.UserError( + "Lexer Error: failed to tokenize input starting at: {}:{} with:\n {}" + .format(lineno, col, remainder[:-20])) + return tokens_return diff --git a/cmake_format/parse/additional_nodes.py b/cmake_format/parse/additional_nodes.py index bfd86c3..d513d34 100644 --- a/cmake_format/parse/additional_nodes.py +++ b/cmake_format/parse/additional_nodes.py @@ -11,9 +11,9 @@ from cmake_format.parse.common import ( NodeType, TreeNode, ) -from cmake_format.parse.simple_nodes import CommentNode +from cmake_format.parse.simple_nodes import CommentNode, OnOffNode from cmake_format.parse.argument_nodes import ( - PositionalGroupNode, PositionalParser, StandardArgTree + PositionalGroupNode, PositionalParser, PositionalSpec, StandardArgTree ) logger = logging.getLogger(__name__) @@ -71,8 +71,15 @@ def parse(cls, ctx, tokens, npargs, ntup, flags, breakstack): child.children.append(token) continue + # If it's a sentinel comment, then add it at the current depth + if tokens[0].type in (lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + tree.children.append(OnOffNode.consume(ctx, tokens)) + continue + if subtree is None: subtree = PositionalGroupNode() + subtree.spec = PositionalSpec(2, False, [], flags) tree.children.append(subtree) ntup_consumed = 0 @@ -163,8 +170,8 @@ def parse(cls, ctx, tokens, flags, breakstack): Parse a continuous sequence of flags """ - # TODO(josh): use a bespoke FLAGGROUP? tree = cls() + tree.spec = PositionalSpec('+', False, [], flags) while tokens: # Break if the next token belongs to a parent parser, i.e. if it # matches a keyword argument of something higher in the stack, or if diff --git a/cmake_format/parse/argument_nodes.py b/cmake_format/parse/argument_nodes.py index 6e41f1d..0a23dd5 100644 --- a/cmake_format/parse/argument_nodes.py +++ b/cmake_format/parse/argument_nodes.py @@ -13,7 +13,7 @@ from cmake_format.parse.common import ( NodeType, ParenBreaker, KwargBreaker, TreeNode ) -from cmake_format.parse.simple_nodes import CommentNode +from cmake_format.parse.simple_nodes import CommentNode, OnOffNode logger = logging.getLogger(__name__) @@ -116,9 +116,13 @@ def parse2(cls, ctx, tokens, pargspecs, kwargs, breakstack): # If it's a comment, then add it at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): - child = TreeNode(NodeType.COMMENT) - tree.children.append(child) - child.children.append(tokens.pop(0)) + tree.children.append(CommentNode.consume(ctx, tokens)) + continue + + # If it's a sentinel comment, then add it at the current depth + if tokens[0].type in (lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + tree.children.append(OnOffNode.consume(ctx, tokens)) continue ntokens = len(tokens) @@ -351,13 +355,16 @@ def parse2(cls, ctx, tokens, spec, breakstack): # directly into the parse tree at the current depth if tokens[0].type in (lexer.TokenType.COMMENT, lexer.TokenType.BRACKET_COMMENT): - before = len(tokens) child = CommentNode.consume(ctx, tokens) - assert len(tokens) < before, \ - "consume_comment didn't consume any tokens" tree.children.append(child) continue + # If it's a sentinel comment, then add it at the current depth + if tokens[0].type in (lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + tree.children.append(OnOffNode.consume(ctx, tokens)) + continue + # Otherwise is it is a positional argument, so add it to the tree as such if get_normalized_kwarg(tokens[0]) in spec.flags: child = TreeNode(NodeType.FLAG) @@ -518,6 +525,12 @@ def parse(cls, ctx, tokens, breakstack): child.children.append(tokens.pop(0)) continue + # If it's a sentinel comment, then add it at the current depth + if tokens[0].type in (lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + tree.children.append(OnOffNode.consume(ctx, tokens)) + continue + # If this is the start of a parenthetical group, then parse the group if tokens[0].type == lexer.TokenType.LEFT_PAREN: subtree = ParenGroupNode.parse(ctx, tokens, breakstack) diff --git a/cmake_format/parse/properties.py b/cmake_format/parse/properties.py new file mode 100644 index 0000000..2b8ca04 --- /dev/null +++ b/cmake_format/parse/properties.py @@ -0,0 +1,391 @@ +# pylint: skip-file +""" +Database of known property names +NOTE: this file is automatically generated by the script +python -Bm cmake_format.tools.parse_cmake_help properties +""" +import re + + +PATTERNS = [ + "(?P.*)_OUTPUT_NAME", + "(?P.*)_POSTFIX", + "(?P.*)_CLANG_TIDY", + "(?P.*)_COMPILER_LAUNCHER", + "(?P.*)_CPPCHECK", + "(?P.*)_CPPLINT", + "(?P.*)_INCLUDE_WHAT_YOU_USE", + "(?P.*)_VISIBILITY_PRESET", + "ABSTRACT", + "ADDITIONAL_MAKE_CLEAN_FILES", + "ADVANCED", + "ALIASED_TARGET", + "ALLOW_DUPLICATE_CUSTOM_TARGETS", + "ANDROID_ANT_ADDITIONAL_OPTIONS", + "ANDROID_API", + "ANDROID_API_MIN", + "ANDROID_ARCH", + "ANDROID_ASSETS_DIRECTORIES", + "ANDROID_GUI", + "ANDROID_JAR_DEPENDENCIES", + "ANDROID_JAR_DIRECTORIES", + "ANDROID_JAVA_SOURCE_DIR", + "ANDROID_NATIVE_LIB_DEPENDENCIES", + "ANDROID_NATIVE_LIB_DIRECTORIES", + "ANDROID_PROCESS_MAX", + "ANDROID_PROGUARD", + "ANDROID_PROGUARD_CONFIG_PATH", + "ANDROID_SECURE_PROPS_PATH", + "ANDROID_SKIP_ANT_STEP", + "ANDROID_STL_TYPE", + "ARCHIVE_OUTPUT_DIRECTORY", + "ARCHIVE_OUTPUT_DIRECTORY_(?P.*)", + "ARCHIVE_OUTPUT_NAME", + "ARCHIVE_OUTPUT_NAME_(?P.*)", + "ATTACHED_FILES", + "ATTACHED_FILES_ON_FAIL", + "AUTOGEN_BUILD_DIR", + "AUTOGEN_SOURCE_GROUP", + "AUTOGEN_TARGETS_FOLDER", + "AUTOGEN_TARGET_DEPENDS", + "AUTOMOC", + "AUTOMOC_COMPILER_PREDEFINES", + "AUTOMOC_DEPEND_FILTERS", + "AUTOMOC_MACRO_NAMES", + "AUTOMOC_MOC_OPTIONS", + "AUTOMOC_SOURCE_GROUP", + "AUTOMOC_TARGETS_FOLDER", + "AUTORCC", + "AUTORCC_OPTIONS", + "AUTORCC_OPTIONS", + "AUTORCC_SOURCE_GROUP", + "AUTOUIC", + "AUTOUIC_OPTIONS", + "AUTOUIC_OPTIONS", + "AUTOUIC_SEARCH_PATHS", + "BINARY_DIR", + "BINARY_DIR", + "BUILDSYSTEM_TARGETS", + "BUILD_RPATH", + "BUILD_WITH_INSTALL_NAME_DIR", + "BUILD_WITH_INSTALL_RPATH", + "BUNDLE", + "BUNDLE_EXTENSION", + "CACHE_VARIABLES", + "CLEAN_NO_CUSTOM", + "CMAKE_CONFIGURE_DEPENDS", + "CMAKE_CXX_KNOWN_FEATURES", + "CMAKE_C_KNOWN_FEATURES", + "COMPATIBLE_INTERFACE_BOOL", + "COMPATIBLE_INTERFACE_NUMBER_MAX", + "COMPATIBLE_INTERFACE_NUMBER_MIN", + "COMPATIBLE_INTERFACE_STRING", + "COMPILE_DEFINITIONS", + "COMPILE_DEFINITIONS", + "COMPILE_DEFINITIONS", + "COMPILE_DEFINITIONS_(?P.*)", + "COMPILE_DEFINITIONS_(?P.*)", + "COMPILE_DEFINITIONS_(?P.*)", + "COMPILE_FEATURES", + "COMPILE_FLAGS", + "COMPILE_FLAGS", + "COMPILE_OPTIONS", + "COMPILE_OPTIONS", + "COMPILE_PDB_NAME", + "COMPILE_PDB_NAME_(?P.*)", + "COMPILE_PDB_OUTPUT_DIRECTORY", + "COMPILE_PDB_OUTPUT_DIRECTORY_(?P.*)", + "COST", + "CPACK_DESKTOP_SHORTCUTS", + "CPACK_NEVER_OVERWRITE", + "CPACK_PERMANENT", + "CPACK_STARTUP_SHORTCUTS", + "CPACK_START_MENU_SHORTCUTS", + "CPACK_WIX_ACL", + "CROSSCOMPILING_EMULATOR", + "CUDA_EXTENSIONS", + "CUDA_PTX_COMPILATION", + "CUDA_RESOLVE_DEVICE_SYMBOLS", + "CUDA_SEPARABLE_COMPILATION", + "CUDA_STANDARD", + "CUDA_STANDARD_REQUIRED", + "CXX_EXTENSIONS", + "CXX_STANDARD", + "CXX_STANDARD_REQUIRED", + "C_EXTENSIONS", + "C_STANDARD", + "C_STANDARD_REQUIRED", + "DEBUG_CONFIGURATIONS", + "DEBUG_POSTFIX", + "DEFINE_SYMBOL", + "DEFINITIONS", + "DEPENDS", + "DEPLOYMENT_REMOTE_DIRECTORY", + "DISABLED", + "DISABLED_FEATURES", + "ECLIPSE_EXTRA_NATURES", + "ENABLED_FEATURES", + "ENABLED_LANGUAGES", + "ENABLE_EXPORTS", + "ENVIRONMENT", + "EXCLUDE_FROM_ALL", + "EXCLUDE_FROM_ALL", + "EXCLUDE_FROM_DEFAULT_BUILD", + "EXCLUDE_FROM_DEFAULT_BUILD_(?P.*)", + "EXPORT_NAME", + "EXTERNAL_OBJECT", + "EchoString", + "FAIL_REGULAR_EXPRESSION", + "FIND_LIBRARY_USE_LIB32_PATHS", + "FIND_LIBRARY_USE_LIB64_PATHS", + "FIND_LIBRARY_USE_LIBX32_PATHS", + "FIND_LIBRARY_USE_OPENBSD_VERSIONING", + "FIXTURES_CLEANUP", + "FIXTURES_REQUIRED", + "FIXTURES_SETUP", + "FOLDER", + "FRAMEWORK", + "FRAMEWORK_VERSION", + "Fortran_FORMAT", + "Fortran_FORMAT", + "Fortran_MODULE_DIRECTORY", + "GENERATED", + "GENERATOR_FILE_NAME", + "GENERATOR_IS_MULTI_CONFIG", + "GLOBAL_DEPENDS_DEBUG_MODE", + "GLOBAL_DEPENDS_NO_CYCLES", + "GNUtoMS", + "HAS_CXX", + "HEADER_FILE_ONLY", + "HELPSTRING", + "IMPLICIT_DEPENDS_INCLUDE_TRANSFORM", + "IMPLICIT_DEPENDS_INCLUDE_TRANSFORM", + "IMPORTED", + "IMPORTED_CONFIGURATIONS", + "IMPORTED_IMPLIB", + "IMPORTED_IMPLIB_(?P.*)", + "IMPORTED_LIBNAME", + "IMPORTED_LIBNAME_(?P.*)", + "IMPORTED_LINK_DEPENDENT_LIBRARIES", + "IMPORTED_LINK_DEPENDENT_LIBRARIES_(?P.*)", + "IMPORTED_LINK_INTERFACE_LANGUAGES", + "IMPORTED_LINK_INTERFACE_LANGUAGES_(?P.*)", + "IMPORTED_LINK_INTERFACE_LIBRARIES", + "IMPORTED_LINK_INTERFACE_LIBRARIES_(?P.*)", + "IMPORTED_LINK_INTERFACE_MULTIPLICITY", + "IMPORTED_LINK_INTERFACE_MULTIPLICITY_(?P.*)", + "IMPORTED_LOCATION", + "IMPORTED_LOCATION_(?P.*)", + "IMPORTED_NO_SONAME", + "IMPORTED_NO_SONAME_(?P.*)", + "IMPORTED_OBJECTS", + "IMPORTED_OBJECTS_(?P.*)", + "IMPORTED_SONAME", + "IMPORTED_SONAME_(?P.*)", + "IMPORT_PREFIX", + "IMPORT_SUFFIX", + "INCLUDE_DIRECTORIES", + "INCLUDE_DIRECTORIES", + "INCLUDE_REGULAR_EXPRESSION", + "INSTALL_NAME_DIR", + "INSTALL_RPATH", + "INSTALL_RPATH_USE_LINK_PATH", + "INTERFACE_AUTOUIC_OPTIONS", + "INTERFACE_COMPILE_DEFINITIONS", + "INTERFACE_COMPILE_FEATURES", + "INTERFACE_COMPILE_OPTIONS", + "INTERFACE_INCLUDE_DIRECTORIES", + "INTERFACE_LINK_LIBRARIES", + "INTERFACE_POSITION_INDEPENDENT_CODE", + "INTERFACE_SOURCES", + "INTERFACE_SYSTEM_INCLUDE_DIRECTORIES", + "INTERPROCEDURAL_OPTIMIZATION", + "INTERPROCEDURAL_OPTIMIZATION", + "INTERPROCEDURAL_OPTIMIZATION_(?P.*)", + "INTERPROCEDURAL_OPTIMIZATION_(?P.*)", + "IN_TRY_COMPILE", + "IOS_INSTALL_COMBINED", + "JOB_POOLS", + "JOB_POOL_COMPILE", + "JOB_POOL_LINK", + "KEEP_EXTENSION", + "LABELS", + "LABELS", + "LABELS", + "LABELS", + "LANGUAGE", + "LIBRARY_OUTPUT_DIRECTORY", + "LIBRARY_OUTPUT_DIRECTORY_(?P.*)", + "LIBRARY_OUTPUT_NAME", + "LIBRARY_OUTPUT_NAME_(?P.*)", + "LINKER_LANGUAGE", + "LINK_DEPENDS", + "LINK_DEPENDS_NO_SHARED", + "LINK_DIRECTORIES", + "LINK_FLAGS", + "LINK_FLAGS_(?P.*)", + "LINK_INTERFACE_LIBRARIES", + "LINK_INTERFACE_LIBRARIES_(?P.*)", + "LINK_INTERFACE_MULTIPLICITY", + "LINK_INTERFACE_MULTIPLICITY_(?P.*)", + "LINK_LIBRARIES", + "LINK_SEARCH_END_STATIC", + "LINK_SEARCH_START_STATIC", + "LINK_WHAT_YOU_USE", + "LISTFILE_STACK", + "LOCATION", + "LOCATION", + "LOCATION_(?P.*)", + "MACOSX_BUNDLE", + "MACOSX_BUNDLE_INFO_PLIST", + "MACOSX_FRAMEWORK_INFO_PLIST", + "MACOSX_PACKAGE_LOCATION", + "MACOSX_RPATH", + "MACROS", + "MANUALLY_ADDED_DEPENDENCIES", + "MAP_IMPORTED_CONFIG_(?P.*)", + "MEASUREMENT", + "MODIFIED", + "NAME", + "NO_SONAME", + "NO_SYSTEM_FROM_IMPORTED", + "OBJECT_DEPENDS", + "OBJECT_OUTPUTS", + "OSX_ARCHITECTURES", + "OSX_ARCHITECTURES_(?P.*)", + "OUTPUT_NAME", + "OUTPUT_NAME_(?P.*)", + "PACKAGES_FOUND", + "PACKAGES_NOT_FOUND", + "PARENT_DIRECTORY", + "PASS_REGULAR_EXPRESSION", + "PDB_NAME", + "PDB_NAME_(?P.*)", + "PDB_OUTPUT_DIRECTORY", + "PDB_OUTPUT_DIRECTORY_(?P.*)", + "POSITION_INDEPENDENT_CODE", + "POST_INSTALL_SCRIPT", + "PREDEFINED_TARGETS_FOLDER", + "PREFIX", + "PRE_INSTALL_SCRIPT", + "PRIVATE_HEADER", + "PROCESSORS", + "PROJECT_LABEL", + "PUBLIC_HEADER", + "REPORT_UNDEFINED_PROPERTIES", + "REQUIRED_FILES", + "RESOURCE", + "RESOURCE_LOCK", + "RULE_LAUNCH_COMPILE", + "RULE_LAUNCH_COMPILE", + "RULE_LAUNCH_COMPILE", + "RULE_LAUNCH_CUSTOM", + "RULE_LAUNCH_CUSTOM", + "RULE_LAUNCH_CUSTOM", + "RULE_LAUNCH_LINK", + "RULE_LAUNCH_LINK", + "RULE_LAUNCH_LINK", + "RULE_MESSAGES", + "RUNTIME_OUTPUT_DIRECTORY", + "RUNTIME_OUTPUT_DIRECTORY_(?P.*)", + "RUNTIME_OUTPUT_NAME", + "RUNTIME_OUTPUT_NAME_(?P.*)", + "RUN_SERIAL", + "SKIP_AUTOGEN", + "SKIP_AUTOMOC", + "SKIP_AUTORCC", + "SKIP_AUTOUIC", + "SKIP_BUILD_RPATH", + "SKIP_RETURN_CODE", + "SOURCES", + "SOURCE_DIR", + "SOURCE_DIR", + "SOVERSION", + "STATIC_LIBRARY_FLAGS", + "STATIC_LIBRARY_FLAGS_(?P.*)", + "STRINGS", + "SUBDIRECTORIES", + "SUFFIX", + "SYMBOLIC", + "TARGET_ARCHIVES_MAY_BE_SHARED_LIBS", + "TARGET_MESSAGES", + "TARGET_SUPPORTS_SHARED_LIBS", + "TEST_INCLUDE_FILE", + "TEST_INCLUDE_FILES", + "TIMEOUT", + "TIMEOUT_AFTER_MATCH", + "TYPE", + "TYPE", + "USE_FOLDERS", + "VALUE", + "VARIABLES", + "VERSION", + "VISIBILITY_INLINES_HIDDEN", + "VS_CONFIGURATION_TYPE", + "VS_COPY_TO_OUT_DIR", + "VS_CSHARP_(?P.*)", + "VS_DEBUGGER_WORKING_DIRECTORY", + "VS_DEPLOYMENT_CONTENT", + "VS_DEPLOYMENT_LOCATION", + "VS_DESKTOP_EXTENSIONS_VERSION", + "VS_DOTNET_REFERENCEPROP_(?P.*)_TAG_(?P.*)", + "VS_DOTNET_REFERENCES", + "VS_DOTNET_REFERENCES_COPY_LOCAL", + "VS_DOTNET_REFERENCE_(?P.*)", + "VS_DOTNET_TARGET_FRAMEWORK_VERSION", + "VS_GLOBAL_(?P.*)", + "VS_GLOBAL_KEYWORD", + "VS_GLOBAL_PROJECT_TYPES", + "VS_GLOBAL_ROOTNAMESPACE", + "VS_GLOBAL_SECTION_POST_(?P
.*)", + "VS_GLOBAL_SECTION_PRE_(?P
.*)", + "VS_INCLUDE_IN_VSIX", + "VS_IOT_EXTENSIONS_VERSION", + "VS_IOT_STARTUP_TASK", + "VS_KEYWORD", + "VS_MOBILE_EXTENSIONS_VERSION", + "VS_RESOURCE_GENERATOR", + "VS_SCC_AUXPATH", + "VS_SCC_LOCALPATH", + "VS_SCC_PROJECTNAME", + "VS_SCC_PROVIDER", + "VS_SDK_REFERENCES", + "VS_SHADER_ENTRYPOINT", + "VS_SHADER_FLAGS", + "VS_SHADER_MODEL", + "VS_SHADER_OUTPUT_HEADER_FILE", + "VS_SHADER_TYPE", + "VS_SHADER_VARIABLE_NAME", + "VS_STARTUP_PROJECT", + "VS_TOOL_OVERRIDE", + "VS_USER_PROPS", + "VS_WINDOWS_TARGET_PLATFORM_MIN_VERSION", + "VS_WINRT_COMPONENT", + "VS_WINRT_EXTENSIONS", + "VS_WINRT_REFERENCES", + "VS_XAML_TYPE", + "WILL_FAIL", + "WIN32_EXECUTABLE", + "WINDOWS_EXPORT_ALL_SYMBOLS", + "WORKING_DIRECTORY", + "WRAP_EXCLUDE", + "XCODE_ATTRIBUTE_(?P.*)", + "XCODE_EMIT_EFFECTIVE_PLATFORM_NAME", + "XCODE_EXPLICIT_FILE_TYPE", + "XCODE_EXPLICIT_FILE_TYPE", + "XCODE_FILE_ATTRIBUTES", + "XCODE_LAST_KNOWN_FILE_TYPE", + "XCODE_PRODUCT_TYPE", + "XCTEST", +] + + +def stripped_patterns(): + regex = re.compile(r"\?P<[\w_]+>") + return [regex.sub("", pattern) for pattern in PATTERNS] + + +CASE_SENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns())) +CASE_INSENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns()), re.IGNORECASE) diff --git a/cmake_format/parse/simple_nodes.py b/cmake_format/parse/simple_nodes.py index 85efadf..736da79 100644 --- a/cmake_format/parse/simple_nodes.py +++ b/cmake_format/parse/simple_nodes.py @@ -59,10 +59,14 @@ def consume(cls, ctx, tokens): Consume sequential comment lines, removing tokens from the input list and returning a comment Block """ + len_before = len(tokens) node = cls() - if tokens[0].type == lexer.TokenType.BRACKET_COMMENT: - # Bracket comments get their own node because they are caapable of + if tokens[0].type in ( + lexer.TokenType.BRACKET_COMMENT, + lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + # Bracket comments get their own node because they are capable of # globbing up their newlines. Thus they are their own semantic comment, # and are not part of any larger semantic structures. node.children.append(tokens.pop(0)) @@ -98,6 +102,8 @@ def consume(cls, ctx, tokens): node.children.append(tokens.pop(0)) node.children.append(tokens.pop(0)) + assert len(tokens) < len_before, \ + "CommentNode.consume didn't consume any tokens" return node @classmethod @@ -213,4 +219,10 @@ def consume_whitespace_and_comments(ctx, tokens, tree): child = CommentNode.consume(ctx, tokens) tree.children.append(child) continue + + # If it's a sentinel comment, then add it at the current depth + if tokens[0].type in (lexer.TokenType.FORMAT_OFF, + lexer.TokenType.FORMAT_ON): + tree.children.append(OnOffNode.consume(ctx, tokens)) + continue break diff --git a/cmake_format/parse/util.py b/cmake_format/parse/util.py index 6495648..60d2264 100644 --- a/cmake_format/parse/util.py +++ b/cmake_format/parse/util.py @@ -26,6 +26,7 @@ # wont get reflowed within a comment block so there is no reason to parse them # as such. COMMENT_TOKENS = (lexer.TokenType.COMMENT,) + ONOFF_TOKENS = (lexer.TokenType.FORMAT_ON, lexer.TokenType.FORMAT_OFF) @@ -213,9 +214,18 @@ def get_min_npargs(npargs): return 1 if npargs.endswith("+"): - return int(npargs[:-1]) + try: + return int(npargs[:-1]) + except ValueError: + pass + + try: + return int(npargs) + except ValueError: + pass - raise ValueError("Unexpected npargs {}".format(npargs)) + raise ValueError( + "Unexpected npargs {}({})".format(npargs, type(npargs).__name__)) LINE_TAG = re.compile(r"#\s*(cmake-format|cmf): ([^\n]*)") diff --git a/cmake_format/parse/variables.py b/cmake_format/parse/variables.py new file mode 100644 index 0000000..5ed6788 --- /dev/null +++ b/cmake_format/parse/variables.py @@ -0,0 +1,500 @@ +# pylint: skip-file +""" +Database of known variable names +NOTE: this file is automatically generated by the script +python -Bm cmake_format.tools.parse_cmake_help variables +""" +import re + + +PATTERNS = [ + "(?P.*)_BINARY_DIR", + "(?P.*)_SOURCE_DIR", + "(?P.*)_VERSION", + "(?P.*)_VERSION_MAJOR", + "(?P.*)_VERSION_MINOR", + "(?P.*)_VERSION_PATCH", + "(?P.*)_VERSION_TWEAK", + "ANDROID", + "APPLE", + "BORLAND", + "BUILD_SHARED_LIBS", + "CMAKE_(?P.*)_POSTFIX", + "CMAKE_(?P.*)_ANDROID_TOOLCHAIN_MACHINE", + "CMAKE_(?P.*)_ANDROID_TOOLCHAIN_PREFIX", + "CMAKE_(?P.*)_ANDROID_TOOLCHAIN_SUFFIX", + "CMAKE_(?P.*)_ARCHIVE_APPEND", + "CMAKE_(?P.*)_ARCHIVE_CREATE", + "CMAKE_(?P.*)_ARCHIVE_FINISH", + "CMAKE_(?P.*)_CLANG_TIDY", + "CMAKE_(?P.*)_COMPILER", + "CMAKE_(?P.*)_COMPILER_ABI", + "CMAKE_(?P.*)_COMPILER_AR", + "CMAKE_(?P.*)_COMPILER_ARCHITECTURE_ID", + "CMAKE_(?P.*)_COMPILER_EXTERNAL_TOOLCHAIN", + "CMAKE_(?P.*)_COMPILER_ID", + "CMAKE_(?P.*)_COMPILER_LAUNCHER", + "CMAKE_(?P.*)_COMPILER_LOADED", + "CMAKE_(?P.*)_COMPILER_PREDEFINES_COMMAND", + "CMAKE_(?P.*)_COMPILER_RANLIB", + "CMAKE_(?P.*)_COMPILER_TARGET", + "CMAKE_(?P.*)_COMPILER_VERSION", + "CMAKE_(?P.*)_COMPILER_VERSION_INTERNAL", + "CMAKE_(?P.*)_COMPILE_OBJECT", + "CMAKE_(?P.*)_CPPCHECK", + "CMAKE_(?P.*)_CPPLINT", + "CMAKE_(?P.*)_CREATE_SHARED_LIBRARY", + "CMAKE_(?P.*)_CREATE_SHARED_MODULE", + "CMAKE_(?P.*)_CREATE_STATIC_LIBRARY", + "CMAKE_(?P.*)_FLAGS", + "CMAKE_(?P.*)_FLAGS_DEBUG", + "CMAKE_(?P.*)_FLAGS_DEBUG_INIT", + "CMAKE_(?P.*)_FLAGS_INIT", + "CMAKE_(?P.*)_FLAGS_MINSIZEREL", + "CMAKE_(?P.*)_FLAGS_MINSIZEREL_INIT", + "CMAKE_(?P.*)_FLAGS_RELEASE", + "CMAKE_(?P.*)_FLAGS_RELEASE_INIT", + "CMAKE_(?P.*)_FLAGS_RELWITHDEBINFO", + "CMAKE_(?P.*)_FLAGS_RELWITHDEBINFO_INIT", + "CMAKE_(?P.*)_GHS_KERNEL_FLAGS_DEBUG", + "CMAKE_(?P.*)_GHS_KERNEL_FLAGS_MINSIZEREL", + "CMAKE_(?P.*)_GHS_KERNEL_FLAGS_RELEASE", + "CMAKE_(?P.*)_GHS_KERNEL_FLAGS_RELWITHDEBINFO", + "CMAKE_(?P.*)_IGNORE_EXTENSIONS", + "CMAKE_(?P.*)_IMPLICIT_INCLUDE_DIRECTORIES", + "CMAKE_(?P.*)_IMPLICIT_LINK_DIRECTORIES", + "CMAKE_(?P.*)_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES", + "CMAKE_(?P.*)_IMPLICIT_LINK_LIBRARIES", + "CMAKE_(?P.*)_INCLUDE_WHAT_YOU_USE", + "CMAKE_(?P.*)_LIBRARY_ARCHITECTURE", + "CMAKE_(?P.*)_LINKER_PREFERENCE", + "CMAKE_(?P.*)_LINKER_PREFERENCE_PROPAGATES", + "CMAKE_(?P.*)_LINK_EXECUTABLE", + "CMAKE_(?P.*)_OUTPUT_EXTENSION", + "CMAKE_(?P.*)_PLATFORM_ID", + "CMAKE_(?P.*)_SIMULATE_ID", + "CMAKE_(?P.*)_SIMULATE_VERSION", + "CMAKE_(?P.*)_SIZEOF_DATA_PTR", + "CMAKE_(?P.*)_SOURCE_FILE_EXTENSIONS", + "CMAKE_(?P.*)_STANDARD_INCLUDE_DIRECTORIES", + "CMAKE_(?P.*)_STANDARD_LIBRARIES", + "CMAKE_(?P.*)_VISIBILITY_PRESET", + "CMAKE_ABSOLUTE_DESTINATION_FILES", + "CMAKE_ANDROID_ANT_ADDITIONAL_OPTIONS", + "CMAKE_ANDROID_API", + "CMAKE_ANDROID_API_MIN", + "CMAKE_ANDROID_ARCH", + "CMAKE_ANDROID_ARCH_ABI", + "CMAKE_ANDROID_ARM_MODE", + "CMAKE_ANDROID_ARM_NEON", + "CMAKE_ANDROID_ASSETS_DIRECTORIES", + "CMAKE_ANDROID_GUI", + "CMAKE_ANDROID_JAR_DEPENDENCIES", + "CMAKE_ANDROID_JAR_DIRECTORIES", + "CMAKE_ANDROID_JAVA_SOURCE_DIR", + "CMAKE_ANDROID_NATIVE_LIB_DEPENDENCIES", + "CMAKE_ANDROID_NATIVE_LIB_DIRECTORIES", + "CMAKE_ANDROID_NDK", + "CMAKE_ANDROID_NDK_DEPRECATED_HEADERS", + "CMAKE_ANDROID_NDK_TOOLCHAIN_HOST_TAG", + "CMAKE_ANDROID_NDK_TOOLCHAIN_VERSION", + "CMAKE_ANDROID_PROCESS_MAX", + "CMAKE_ANDROID_PROGUARD", + "CMAKE_ANDROID_PROGUARD_CONFIG_PATH", + "CMAKE_ANDROID_SECURE_PROPS_PATH", + "CMAKE_ANDROID_SKIP_ANT_STEP", + "CMAKE_ANDROID_STANDALONE_TOOLCHAIN", + "CMAKE_ANDROID_STL_TYPE", + "CMAKE_APPBUNDLE_PATH", + "CMAKE_AR", + "CMAKE_ARCHIVE_OUTPUT_DIRECTORY", + "CMAKE_ARCHIVE_OUTPUT_DIRECTORY_(?P.*)", + "CMAKE_ARGC", + "CMAKE_ARGV0", + "CMAKE_AUTOMOC", + "CMAKE_AUTOMOC_COMPILER_PREDEFINES", + "CMAKE_AUTOMOC_DEPEND_FILTERS", + "CMAKE_AUTOMOC_MACRO_NAMES", + "CMAKE_AUTOMOC_MOC_OPTIONS", + "CMAKE_AUTOMOC_RELAXED_MODE", + "CMAKE_AUTORCC", + "CMAKE_AUTORCC_OPTIONS", + "CMAKE_AUTOUIC", + "CMAKE_AUTOUIC_OPTIONS", + "CMAKE_AUTOUIC_SEARCH_PATHS", + "CMAKE_BACKWARDS_COMPATIBILITY", + "CMAKE_BINARY_DIR", + "CMAKE_BUILD_RPATH", + "CMAKE_BUILD_TOOL", + "CMAKE_BUILD_TYPE", + "CMAKE_BUILD_WITH_INSTALL_NAME_DIR", + "CMAKE_BUILD_WITH_INSTALL_RPATH", + "CMAKE_CACHEFILE_DIR", + "CMAKE_CACHE_MAJOR_VERSION", + "CMAKE_CACHE_MINOR_VERSION", + "CMAKE_CACHE_PATCH_VERSION", + "CMAKE_CFG_INTDIR", + "CMAKE_CL_64", + "CMAKE_CODEBLOCKS_EXCLUDE_EXTERNAL_FILES", + "CMAKE_CODELITE_USE_TARGETS", + "CMAKE_COLOR_MAKEFILE", + "CMAKE_COMMAND", + "CMAKE_COMPILER_2005", + "CMAKE_COMPILER_IS_GNUCC", + "CMAKE_COMPILER_IS_GNUCXX", + "CMAKE_COMPILER_IS_GNUG77", + "CMAKE_COMPILE_PDB_OUTPUT_DIRECTORY", + "CMAKE_COMPILE_PDB_OUTPUT_DIRECTORY_(?P.*)", + "CMAKE_CONFIGURATION_TYPES", + "CMAKE_CROSSCOMPILING", + "CMAKE_CROSSCOMPILING_EMULATOR", + "CMAKE_CTEST_COMMAND", + "CMAKE_CUDA_EXTENSIONS", + "CMAKE_CUDA_HOST_COMPILER", + "CMAKE_CUDA_STANDARD", + "CMAKE_CUDA_STANDARD_REQUIRED", + "CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES", + "CMAKE_CURRENT_BINARY_DIR", + "CMAKE_CURRENT_LIST_DIR", + "CMAKE_CURRENT_LIST_FILE", + "CMAKE_CURRENT_LIST_LINE", + "CMAKE_CURRENT_SOURCE_DIR", + "CMAKE_CXX_COMPILE_FEATURES", + "CMAKE_CXX_EXTENSIONS", + "CMAKE_CXX_STANDARD", + "CMAKE_CXX_STANDARD_REQUIRED", + "CMAKE_C_COMPILE_FEATURES", + "CMAKE_C_EXTENSIONS", + "CMAKE_C_STANDARD", + "CMAKE_C_STANDARD_REQUIRED", + "CMAKE_DEBUG_POSTFIX", + "CMAKE_DEBUG_TARGET_PROPERTIES", + "CMAKE_DEPENDS_IN_PROJECT_ONLY", + "CMAKE_DIRECTORY_LABELS", + "CMAKE_DISABLE_FIND_PACKAGE_(?P.*)", + "CMAKE_DL_LIBS", + "CMAKE_ECLIPSE_GENERATE_LINKED_RESOURCES", + "CMAKE_ECLIPSE_GENERATE_SOURCE_PROJECT", + "CMAKE_ECLIPSE_MAKE_ARGUMENTS", + "CMAKE_ECLIPSE_VERSION", + "CMAKE_EDIT_COMMAND", + "CMAKE_ENABLE_EXPORTS", + "CMAKE_ERROR_DEPRECATED", + "CMAKE_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION", + "CMAKE_EXECUTABLE_SUFFIX", + "CMAKE_EXE_LINKER_FLAGS", + "CMAKE_EXE_LINKER_FLAGS_(?P.*)", + "CMAKE_EXE_LINKER_FLAGS_(?P.*)_INIT", + "CMAKE_EXE_LINKER_FLAGS_INIT", + "CMAKE_EXPORT_COMPILE_COMMANDS", + "CMAKE_EXPORT_NO_PACKAGE_REGISTRY", + "CMAKE_EXTRA_GENERATOR", + "CMAKE_EXTRA_SHARED_LIBRARY_SUFFIXES", + "CMAKE_FIND_APPBUNDLE", + "CMAKE_FIND_FRAMEWORK", + "CMAKE_FIND_LIBRARY_CUSTOM_LIB_SUFFIX", + "CMAKE_FIND_LIBRARY_PREFIXES", + "CMAKE_FIND_LIBRARY_SUFFIXES", + "CMAKE_FIND_NO_INSTALL_PREFIX", + "CMAKE_FIND_PACKAGE_NAME", + "CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY", + "CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY", + "CMAKE_FIND_PACKAGE_SORT_DIRECTION", + "CMAKE_FIND_PACKAGE_SORT_ORDER", + "CMAKE_FIND_PACKAGE_WARN_NO_MODULE", + "CMAKE_FIND_ROOT_PATH", + "CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", + "CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", + "CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", + "CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", + "CMAKE_FRAMEWORK_PATH", + "CMAKE_Fortran_FORMAT", + "CMAKE_Fortran_MODDIR_DEFAULT", + "CMAKE_Fortran_MODDIR_FLAG", + "CMAKE_Fortran_MODOUT_FLAG", + "CMAKE_Fortran_MODULE_DIRECTORY", + "CMAKE_GENERATOR", + "CMAKE_GENERATOR_PLATFORM", + "CMAKE_GENERATOR_TOOLSET", + "CMAKE_GNUtoMS", + "CMAKE_HOME_DIRECTORY", + "CMAKE_HOST_APPLE", + "CMAKE_HOST_SOLARIS", + "CMAKE_HOST_SYSTEM", + "CMAKE_HOST_SYSTEM_NAME", + "CMAKE_HOST_SYSTEM_PROCESSOR", + "CMAKE_HOST_SYSTEM_VERSION", + "CMAKE_HOST_UNIX", + "CMAKE_HOST_WIN32", + "CMAKE_IGNORE_PATH", + "CMAKE_IMPORT_LIBRARY_PREFIX", + "CMAKE_IMPORT_LIBRARY_SUFFIX", + "CMAKE_INCLUDE_CURRENT_DIR", + "CMAKE_INCLUDE_CURRENT_DIR_IN_INTERFACE", + "CMAKE_INCLUDE_DIRECTORIES_BEFORE", + "CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE", + "CMAKE_INCLUDE_PATH", + "CMAKE_INSTALL_DEFAULT_COMPONENT_NAME", + "CMAKE_INSTALL_MESSAGE", + "CMAKE_INSTALL_NAME_DIR", + "CMAKE_INSTALL_PREFIX", + "CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT", + "CMAKE_INSTALL_RPATH", + "CMAKE_INSTALL_RPATH_USE_LINK_PATH", + "CMAKE_INTERNAL_PLATFORM_ABI", + "CMAKE_INTERPROCEDURAL_OPTIMIZATION", + "CMAKE_INTERPROCEDURAL_OPTIMIZATION_(?P.*)", + "CMAKE_IOS_INSTALL_COMBINED", + "CMAKE_JOB_POOL_COMPILE", + "CMAKE_JOB_POOL_LINK", + "CMAKE_LIBRARY_ARCHITECTURE", + "CMAKE_LIBRARY_ARCHITECTURE_REGEX", + "CMAKE_LIBRARY_OUTPUT_DIRECTORY", + "CMAKE_LIBRARY_OUTPUT_DIRECTORY_(?P.*)", + "CMAKE_LIBRARY_PATH", + "CMAKE_LIBRARY_PATH_FLAG", + "CMAKE_LINK_DEF_FILE_FLAG", + "CMAKE_LINK_DEPENDS_NO_SHARED", + "CMAKE_LINK_INTERFACE_LIBRARIES", + "CMAKE_LINK_LIBRARY_FILE_FLAG", + "CMAKE_LINK_LIBRARY_FLAG", + "CMAKE_LINK_LIBRARY_SUFFIX", + "CMAKE_LINK_SEARCH_END_STATIC", + "CMAKE_LINK_SEARCH_START_STATIC", + "CMAKE_LINK_WHAT_YOU_USE", + "CMAKE_MACOSX_BUNDLE", + "CMAKE_MACOSX_RPATH", + "CMAKE_MAJOR_VERSION", + "CMAKE_MAKE_PROGRAM", + "CMAKE_MAP_IMPORTED_CONFIG_(?P.*)", + "CMAKE_MATCH_(?P.*)", + "CMAKE_MATCH_COUNT", + "CMAKE_MFC_FLAG", + "CMAKE_MINIMUM_REQUIRED_VERSION", + "CMAKE_MINOR_VERSION", + "CMAKE_MODULE_LINKER_FLAGS", + "CMAKE_MODULE_LINKER_FLAGS_(?P.*)", + "CMAKE_MODULE_LINKER_FLAGS_(?P.*)_INIT", + "CMAKE_MODULE_LINKER_FLAGS_INIT", + "CMAKE_MODULE_PATH", + "CMAKE_MSVCIDE_RUN_PATH", + "CMAKE_NINJA_OUTPUT_PATH_PREFIX", + "CMAKE_NOT_USING_CONFIG_FLAGS", + "CMAKE_NO_BUILTIN_CHRPATH", + "CMAKE_NO_SYSTEM_FROM_IMPORTED", + "CMAKE_OBJECT_PATH_MAX", + "CMAKE_OSX_ARCHITECTURES", + "CMAKE_OSX_DEPLOYMENT_TARGET", + "CMAKE_OSX_SYSROOT", + "CMAKE_PARENT_LIST_FILE", + "CMAKE_PATCH_VERSION", + "CMAKE_PDB_OUTPUT_DIRECTORY", + "CMAKE_PDB_OUTPUT_DIRECTORY_(?P.*)", + "CMAKE_POLICY_DEFAULT_CMP(?P.*)", + "CMAKE_POLICY_WARNING_CMP(?P.*)", + "CMAKE_POSITION_INDEPENDENT_CODE", + "CMAKE_PREFIX_PATH", + "CMAKE_PROGRAM_PATH", + "CMAKE_PROJECT_(?P.*)_INCLUDE", + "CMAKE_PROJECT_DESCRIPTION", + "CMAKE_PROJECT_NAME", + "CMAKE_RANLIB", + "CMAKE_ROOT", + "CMAKE_RUNTIME_OUTPUT_DIRECTORY", + "CMAKE_RUNTIME_OUTPUT_DIRECTORY_(?P.*)", + "CMAKE_SCRIPT_MODE_FILE", + "CMAKE_SHARED_LIBRARY_PREFIX", + "CMAKE_SHARED_LIBRARY_SUFFIX", + "CMAKE_SHARED_LINKER_FLAGS", + "CMAKE_SHARED_LINKER_FLAGS_(?P.*)", + "CMAKE_SHARED_LINKER_FLAGS_(?P.*)_INIT", + "CMAKE_SHARED_LINKER_FLAGS_INIT", + "CMAKE_SHARED_MODULE_PREFIX", + "CMAKE_SHARED_MODULE_SUFFIX", + "CMAKE_SIZEOF_VOID_P", + "CMAKE_SKIP_BUILD_RPATH", + "CMAKE_SKIP_INSTALL_ALL_DEPENDENCY", + "CMAKE_SKIP_INSTALL_RPATH", + "CMAKE_SKIP_INSTALL_RULES", + "CMAKE_SKIP_RPATH", + "CMAKE_SOURCE_DIR", + "CMAKE_STAGING_PREFIX", + "CMAKE_STATIC_LIBRARY_PREFIX", + "CMAKE_STATIC_LIBRARY_SUFFIX", + "CMAKE_STATIC_LINKER_FLAGS", + "CMAKE_STATIC_LINKER_FLAGS_(?P.*)", + "CMAKE_STATIC_LINKER_FLAGS_(?P.*)_INIT", + "CMAKE_STATIC_LINKER_FLAGS_INIT", + "CMAKE_SUBLIME_TEXT_2_ENV_SETTINGS", + "CMAKE_SUBLIME_TEXT_2_EXCLUDE_BUILD_TREE", + "CMAKE_SYSROOT", + "CMAKE_SYSROOT_COMPILE", + "CMAKE_SYSROOT_LINK", + "CMAKE_SYSTEM", + "CMAKE_SYSTEM_APPBUNDLE_PATH", + "CMAKE_SYSTEM_FRAMEWORK_PATH", + "CMAKE_SYSTEM_IGNORE_PATH", + "CMAKE_SYSTEM_INCLUDE_PATH", + "CMAKE_SYSTEM_LIBRARY_PATH", + "CMAKE_SYSTEM_NAME", + "CMAKE_SYSTEM_PREFIX_PATH", + "CMAKE_SYSTEM_PROCESSOR", + "CMAKE_SYSTEM_PROGRAM_PATH", + "CMAKE_SYSTEM_VERSION", + "CMAKE_Swift_LANGUAGE_VERSION", + "CMAKE_TOOLCHAIN_FILE", + "CMAKE_TRY_COMPILE_CONFIGURATION", + "CMAKE_TRY_COMPILE_PLATFORM_VARIABLES", + "CMAKE_TRY_COMPILE_TARGET_TYPE", + "CMAKE_TWEAK_VERSION", + "CMAKE_USER_MAKE_RULES_OVERRIDE", + "CMAKE_USER_MAKE_RULES_OVERRIDE_(?P.*)", + "CMAKE_USE_RELATIVE_PATHS", + "CMAKE_VERBOSE_MAKEFILE", + "CMAKE_VERSION", + "CMAKE_VISIBILITY_INLINES_HIDDEN", + "CMAKE_VS_DEVENV_COMMAND", + "CMAKE_VS_INCLUDE_INSTALL_TO_DEFAULT_BUILD", + "CMAKE_VS_INCLUDE_PACKAGE_TO_DEFAULT_BUILD", + "CMAKE_VS_INTEL_Fortran_PROJECT_VERSION", + "CMAKE_VS_MSBUILD_COMMAND", + "CMAKE_VS_NsightTegra_VERSION", + "CMAKE_VS_PLATFORM_NAME", + "CMAKE_VS_PLATFORM_TOOLSET", + "CMAKE_VS_PLATFORM_TOOLSET_CUDA", + "CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE", + "CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION", + "CMAKE_WARN_DEPRECATED", + "CMAKE_WARN_ON_ABSOLUTE_INSTALL_DESTINATION", + "CMAKE_WIN32_EXECUTABLE", + "CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS", + "CMAKE_XCODE_ATTRIBUTE_(?P.*)", + "CMAKE_XCODE_GENERATE_SCHEME", + "CMAKE_XCODE_PLATFORM_TOOLSET", + "CPACK_ABSOLUTE_DESTINATION_FILES", + "CPACK_COMPONENT_INCLUDE_TOPLEVEL_DIRECTORY", + "CPACK_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION", + "CPACK_INCLUDE_TOPLEVEL_DIRECTORY", + "CPACK_INSTALL_SCRIPT", + "CPACK_PACKAGING_INSTALL_PREFIX", + "CPACK_SET_DESTDIR", + "CPACK_WARN_ON_ABSOLUTE_INSTALL_DESTINATION", + "CTEST_BINARY_DIRECTORY", + "CTEST_BUILD_COMMAND", + "CTEST_BUILD_NAME", + "CTEST_BZR_COMMAND", + "CTEST_BZR_UPDATE_OPTIONS", + "CTEST_CHANGE_ID", + "CTEST_CHECKOUT_COMMAND", + "CTEST_CONFIGURATION_TYPE", + "CTEST_CONFIGURE_COMMAND", + "CTEST_COVERAGE_COMMAND", + "CTEST_COVERAGE_EXTRA_FLAGS", + "CTEST_CURL_OPTIONS", + "CTEST_CUSTOM_COVERAGE_EXCLUDE", + "CTEST_CUSTOM_ERROR_EXCEPTION", + "CTEST_CUSTOM_ERROR_MATCH", + "CTEST_CUSTOM_ERROR_POST_CONTEXT", + "CTEST_CUSTOM_ERROR_PRE_CONTEXT", + "CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE", + "CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS", + "CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS", + "CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE", + "CTEST_CUSTOM_MEMCHECK_IGNORE", + "CTEST_CUSTOM_POST_MEMCHECK", + "CTEST_CUSTOM_POST_TEST", + "CTEST_CUSTOM_PRE_MEMCHECK", + "CTEST_CUSTOM_PRE_TEST", + "CTEST_CUSTOM_TEST_IGNORE", + "CTEST_CUSTOM_WARNING_EXCEPTION", + "CTEST_CUSTOM_WARNING_MATCH", + "CTEST_CVS_CHECKOUT", + "CTEST_CVS_COMMAND", + "CTEST_CVS_UPDATE_OPTIONS", + "CTEST_DROP_LOCATION", + "CTEST_DROP_METHOD", + "CTEST_DROP_SITE", + "CTEST_DROP_SITE_CDASH", + "CTEST_DROP_SITE_PASSWORD", + "CTEST_DROP_SITE_USER", + "CTEST_EXTRA_COVERAGE_GLOB", + "CTEST_GIT_COMMAND", + "CTEST_GIT_INIT_SUBMODULES", + "CTEST_GIT_UPDATE_CUSTOM", + "CTEST_GIT_UPDATE_OPTIONS", + "CTEST_HG_COMMAND", + "CTEST_HG_UPDATE_OPTIONS", + "CTEST_LABELS_FOR_SUBPROJECTS", + "CTEST_MEMORYCHECK_COMMAND", + "CTEST_MEMORYCHECK_COMMAND_OPTIONS", + "CTEST_MEMORYCHECK_SANITIZER_OPTIONS", + "CTEST_MEMORYCHECK_SUPPRESSIONS_FILE", + "CTEST_MEMORYCHECK_TYPE", + "CTEST_NIGHTLY_START_TIME", + "CTEST_P4_CLIENT", + "CTEST_P4_COMMAND", + "CTEST_P4_OPTIONS", + "CTEST_P4_UPDATE_OPTIONS", + "CTEST_SCP_COMMAND", + "CTEST_SITE", + "CTEST_SOURCE_DIRECTORY", + "CTEST_SVN_COMMAND", + "CTEST_SVN_OPTIONS", + "CTEST_SVN_UPDATE_OPTIONS", + "CTEST_TEST_LOAD", + "CTEST_TEST_TIMEOUT", + "CTEST_TRIGGER_SITE", + "CTEST_UPDATE_COMMAND", + "CTEST_UPDATE_OPTIONS", + "CTEST_UPDATE_VERSION_ONLY", + "CTEST_USE_LAUNCHERS", + "CYGWIN", + "ENV", + "EXECUTABLE_OUTPUT_PATH", + "GHS-MULTI", + "LIBRARY_OUTPUT_PATH", + "MINGW", + "MSVC", + "MSVC10", + "MSVC11", + "MSVC12", + "MSVC14", + "MSVC60", + "MSVC70", + "MSVC71", + "MSVC80", + "MSVC90", + "MSVC_IDE", + "MSVC_VERSION", + "PROJECT_BINARY_DIR", + "PROJECT_DESCRIPTION", + "PROJECT_NAME", + "PROJECT_SOURCE_DIR", + "PROJECT_VERSION", + "PROJECT_VERSION_MAJOR", + "PROJECT_VERSION_MINOR", + "PROJECT_VERSION_PATCH", + "PROJECT_VERSION_TWEAK", + "UNIX", + "WIN32", + "WINCE", + "WINDOWS_PHONE", + "WINDOWS_STORE", + "XCODE", + "XCODE_VERSION", +] + + +def stripped_patterns(): + """ + Remove named groups from patterns so we can join them together without + a name collision. + """ + regex = re.compile(r"\?P<[\w_]+>") + return [regex.sub("", pattern) for pattern in PATTERNS] + + +CASE_SENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns())) +CASE_INSENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns()), re.IGNORECASE) diff --git a/cmake_format/parse_funs/fetch_content.py b/cmake_format/parse_funs/fetch_content.py index d4a0b35..0390660 100644 --- a/cmake_format/parse_funs/fetch_content.py +++ b/cmake_format/parse_funs/fetch_content.py @@ -1,6 +1,6 @@ from cmake_format.parse.additional_nodes import ShellCommandNode from cmake_format.parse.argument_nodes import ( - PositionalParser, StandardArgTree) + PositionalParser, StandardArgTree, StandardParser) def parse_fetchcontent_declare(ctx, tokens, breakstack): @@ -159,4 +159,4 @@ def populate_db(parse_db): parse_db["fetchcontent_declare"] = parse_fetchcontent_declare parse_db["fetchcontent_populate"] = parse_fetchcontent_populate parse_db["fetchcontent_getproperties"] = parse_fetchcontent_getproperties - parse_db["fetchcontent_makeavailable"] = PositionalParser('+') + parse_db["fetchcontent_makeavailable"] = StandardParser('+') diff --git a/cmake_format/parse_funs/list.py b/cmake_format/parse_funs/list.py index e9b939f..c6006dd 100644 --- a/cmake_format/parse_funs/list.py +++ b/cmake_format/parse_funs/list.py @@ -112,9 +112,10 @@ def parse_list(ctx, tokens, breakstack): "REMOVE_ITEM": StandardParser("3+", flags=["REMOVE_ITEM"]), "REMOVE_AT": StandardParser("3+", flags=["REMOVE_AT"]), "REMOVE_DUPLICATES": StandardParser(2, flags=["REMOVE_DUPLICATES"]), - "TRANSFORM": StandardParser(2, TRANSFORM_KWARGS), + "TRANSFORM": StandardParser( + 2, flags=["TRANSFORM"], kwargs=TRANSFORM_KWARGS), "REVERSE": StandardParser(2, flags=["REVERSE"]), - "SORT": StandardParser(2, kwargs=SORT_KWARGS) + "SORT": StandardParser(2, flags=["SORT"], kwargs=SORT_KWARGS) } if descriminator not in parsemap: diff --git a/cmake_format/parse_funs/set.py b/cmake_format/parse_funs/set.py index 2ed70a8..5ac1cb6 100644 --- a/cmake_format/parse_funs/set.py +++ b/cmake_format/parse_funs/set.py @@ -80,9 +80,6 @@ def parse_set(ctx, tokens, breakstack): ntokens = len(tokens) - # NOTE(josh): each flag is also stored in kwargs as with a positional parser - # of size zero. This is a legacy thing that should be removed, but for now - # just make sure we check flags first. word = get_normalized_kwarg(tokens[0]) if word == "CACHE": subtree = KeywordGroupNode.parse( @@ -119,6 +116,9 @@ def parse_set(ctx, tokens, breakstack): else: subtree = PositionalGroupNode.parse( ctx, tokens, '+', [], kwarg_breakstack) + for pattern, tags in ctx.config.parse.vartags_: + if pattern.match(tree.varname.spelling): + subtree.tags.extend(tags) tree.value_group = subtree assert len(tokens) < ntokens diff --git a/cmake_format/patches/01_file_command_single_kwargs.patch b/cmake_format/patches/01_file_command_single_kwargs.patch deleted file mode 100644 index 8cb2c9c..0000000 --- a/cmake_format/patches/01_file_command_single_kwargs.patch +++ /dev/null @@ -1,119 +0,0 @@ -diff --git a/cmake_format/commands.py b/cmake_format/commands.py -index 77fd83f..e9b1b49 100644 ---- a/cmake_format/commands.py -+++ b/cmake_format/commands.py -@@ -240,45 +240,50 @@ def get_fn_spec(): - flags=["FOLLOW_SYMLINKS", "GENERATE", "HEX", "NEWLINE_CONSUME", - "NO_HEX_CONVERSION", "SHOW_PROGRESS", "UTC"], - kwargs={ -- "APPEND": ZERO_OR_MORE, -- "DOWNLOAD": ZERO_OR_MORE, -- "EXPECTED_HASH": ZERO_OR_MORE, -- "EXPECTED_MD5": ZERO_OR_MORE, -- "GLOB": ZERO_OR_MORE, -- "GLOB_RECURSE": ZERO_OR_MORE, -- "INACTIVITY_TIMEOUT": ZERO_OR_MORE, -- "LENGTH_MAXIMUM": ZERO_OR_MORE, -- "LENGTH_MINIMUM": ZERO_OR_MORE, -- "LIMIT": ZERO_OR_MORE, -- "LIMIT_COUNT": ZERO_OR_MORE, -- "LIMIT_INPUT": ZERO_OR_MORE, -- "LIMIT_OUTPUT": ZERO_OR_MORE, -- "LOG": ZERO_OR_MORE, -- "MAKE_DIRECTORY": ZERO_OR_MORE, -- "MD5": ZERO_OR_MORE, -- "OFFSET": ZERO_OR_MORE, -- "OUTPUTINPUTCONTENTCONDITION": ZERO_OR_MORE, -- "READ": ZERO_OR_MORE, -- "REGEX": ZERO_OR_MORE, -- "RELATIVE": ZERO_OR_MORE, -- "RELATIVE_PATH": ZERO_OR_MORE, -- "REMOVE": ZERO_OR_MORE, -- "REMOVE_RECURSE": ZERO_OR_MORE, -- "RENAME": ZERO_OR_MORE, -- "SHA1": ZERO_OR_MORE, -- "SHA256": ZERO_OR_MORE, -- "SHA384": ZERO_OR_MORE, -- "SHA512": ZERO_OR_MORE, -- "STATUS": ZERO_OR_MORE, -- "STRINGS": ZERO_OR_MORE, -- "TIMEOUT": ZERO_OR_MORE, -- "TIMESTAMP": ZERO_OR_MORE, -- "TLS_CAINFO": ZERO_OR_MORE, -- "TLS_VERIFY": ZERO_OR_MORE, -- "TO_CMAKE_PATH": ZERO_OR_MORE, -- "TO_NATIVE_PATH": ZERO_OR_MORE, -- "UPLOAD": ZERO_OR_MORE, -- "WRITE": ZERO_OR_MORE -+ # NOTE(josh): semantically these really act more like kwargs -+ # with one or more arguments but since the subargs are all more or -+ # less unstructured the formatting is nicer if we treat them as -+ # taking a single arg. That way it will be packed together if it can -+ # be. -+ "APPEND": 1, -+ "DOWNLOAD": 1, -+ "EXPECTED_HASH": 1, -+ "EXPECTED_MD5": 1, -+ "GLOB": 1, -+ "GLOB_RECURSE": 1, -+ "INACTIVITY_TIMEOUT": 1, -+ "LENGTH_MAXIMUM": 1, -+ "LENGTH_MINIMUM": 1, -+ "LIMIT": 1, -+ "LIMIT_COUNT": 1, -+ "LIMIT_INPUT": 1, -+ "LIMIT_OUTPUT": 1, -+ "LOG": 1, -+ "MAKE_DIRECTORY": 1, -+ "MD5": 1, -+ "OFFSET": 1, -+ "OUTPUTINPUTCONTENTCONDITION": 1, -+ "READ": 1, -+ "REGEX": 1, -+ "RELATIVE": 1, -+ "RELATIVE_PATH": 1, -+ "REMOVE": 1, -+ "REMOVE_RECURSE": 1, -+ "RENAME": 1, -+ "SHA1": 1, -+ "SHA256": 1, -+ "SHA384": 1, -+ "SHA512": 1, -+ "STATUS": 1, -+ "STRINGS": 1, -+ "TIMEOUT": 1, -+ "TIMESTAMP": 1, -+ "TLS_CAINFO": 1, -+ "TLS_VERIFY": 1, -+ "TO_CMAKE_PATH": 1, -+ "TO_NATIVE_PATH": 1, -+ "UPLOAD": 1, -+ "WRITE": 1 - }) - - fn_spec.add( -diff --git a/cmake_format/format_tests.py b/cmake_format/format_tests.py -index d6557e2..80cd95d 100644 ---- a/cmake_format/format_tests.py -+++ b/cmake_format/format_tests.py -@@ -1019,6 +1019,20 @@ class TestCanonicalFormatting(unittest.TestCase): - # for these lines - """) - -+ def test_file_command(self): -+ # NOTE(josh): because the test uses \n in the strings we need to use a -+ # raw string here. However then we need to pop off the initial line break -+ # so we do that with the `[1:]`.` -+ self.do_format_test(r""" -+ file(WRITE ${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}FooBar.txt -+ "this is a long line that should defitely be wrapped below\n" -+ "this is another long line of text content that takes up space\n") -+ """[1:], r""" -+ file(WRITE ${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}FooBar.txt -+ "this is a long line that should defitely be wrapped below\n" -+ "this is another long line of text content that takes up space\n") -+ """[1:]) -+ - def test_example_file(self): - thisdir = os.path.dirname(__file__) - infile_path = os.path.join(thisdir, 'test', 'test_in.cmake') diff --git a/cmake_format/patches/02-additional-whitespace-features.patch b/cmake_format/patches/02-additional-whitespace-features.patch new file mode 100644 index 0000000..9067359 --- /dev/null +++ b/cmake_format/patches/02-additional-whitespace-features.patch @@ -0,0 +1,182 @@ +diff --git a/cmake_format/command_tests/__init__.py b/cmake_format/command_tests/__init__.py +index 8456d03..5923fa4 100644 +--- a/cmake_format/command_tests/__init__.py ++++ b/cmake_format/command_tests/__init__.py +@@ -547,3 +547,10 @@ class TestSet(TestBase): + Test various examples of the set() function + """ + kExpectNumSidecarTests = 8 ++ ++ ++class TestWhitespace(TestBase): ++ """ ++ Test various specific cases involving whitespace formatting ++ """ ++ kExpectNumSidecarTests = 2 +diff --git a/cmake_format/command_tests/__main__.py b/cmake_format/command_tests/__main__.py +index f851f39..bec3262 100644 +--- a/cmake_format/command_tests/__main__.py ++++ b/cmake_format/command_tests/__main__.py +@@ -12,7 +12,8 @@ from cmake_format.command_tests import ( + TestForeach, + TestInstall, + TestSetTargetProperties, +- TestSet) ++ TestSet, ++ TestWhitespace) + + from cmake_format.command_tests.add_executable_tests \ + import TestAddExecutableCommand +diff --git a/cmake_format/command_tests/whitespace_tests.cmake b/cmake_format/command_tests/whitespace_tests.cmake +new file mode 100644 +index 0000000..7006945 +--- /dev/null ++++ b/cmake_format/command_tests/whitespace_tests.cmake +@@ -0,0 +1,24 @@ ++# test: allow_superfluous_newlines ++#[=[ ++allow_superfluous_newlines = 1 ++]=] ++# This comment is just here to start the file ++ ++ ++# This comment has an extra newline before it ++statement(foo bar baz) ++ ++ ++statement(foo bar baz) ++ ++# test: numlines_pre_statement_comment ++#[=[ ++numlines_pre_statement_comment = 1 ++]=] ++# This comment is just here to start the file ++ ++ ++# This comment has an extra newline before it ++statement(foo bar baz) ++ ++statement(foo bar baz) +diff --git a/cmake_format/configuration.py b/cmake_format/configuration.py +index c6334d2..dcf384c 100644 +--- a/cmake_format/configuration.py ++++ b/cmake_format/configuration.py +@@ -315,6 +315,19 @@ class FormattingConfig(ConfigObject): + " the documentation for more information." + ) + ++ allow_superfluous_newlines = FieldDescriptor( ++ 0, ++ "Allow up to this many superfluous newlines between elements at block" ++ " level. In other words, don't collapse whitespace up to " ++ " (allow_superfluous_newlines + 1) lines long." ++ ) ++ ++ numlines_pre_statement_comment = FieldDescriptor( ++ 0, ++ "Output additional newlines to increase visual separation before a" ++ " statement which is immediately preceeded by a block comment" ++ ) ++ + def __init__(self, **kwargs): + super(FormattingConfig, self).__init__(**kwargs) + self.endl = None +diff --git a/cmake_format/doc/README.rst b/cmake_format/doc/README.rst +index 47dcd19..f42fa8e 100644 +--- a/cmake_format/doc/README.rst ++++ b/cmake_format/doc/README.rst +@@ -156,6 +156,15 @@ Usage + the last, most agressive attempt that it made. If this + flag is True, however, cmake-format will print error, + exit with non-zero status code, and write-out nothing ++ --allow-superfluous-newlines ALLOW_SUPERFLUOUS_NEWLINES ++ Allow up to this many superfluous newlines between ++ elements at block level. In other words, don't ++ collapse whitespace up to (allow_superfluous_newlines ++ + 1) lines long. ++ --numlines-pre-statement-comment NUMLINES_PRE_STATEMENT_COMMENT ++ Output additional newlines to increase visual ++ separation before a statement which is immediately ++ preceeded by a block comment + + Options affecting comment reflow and formatting.: + --bullet-char BULLET_CHAR +@@ -358,6 +367,15 @@ pleasant way. + # documentation for more information. + layout_passes = {} + ++ # Allow up to this many superfluous newlines between elements at block level. ++ # In other words, don't collapse whitespace up to (allow_superfluous_newlines ++ # + 1) lines long. ++ allow_superfluous_newlines = 0 ++ ++ # Output additional newlines to increase visual separation before a statement ++ # which is immediately preceeded by a block comment ++ numlines_pre_statement_comment = 0 ++ + # ------------------------------------------------ + # Options affecting comment reflow and formatting. + # ------------------------------------------------ +diff --git a/cmake_format/doc/changelog.rst b/cmake_format/doc/changelog.rst +index 68ba1ce..a651535 100644 +--- a/cmake_format/doc/changelog.rst ++++ b/cmake_format/doc/changelog.rst +@@ -13,6 +13,8 @@ v0.6.8 + * Add build rules to generate variable and property pattern lists + * Implement lint checks on assignment/use of variables that are "close" to + builtins except for case. ++* Add configuration options to allow increased whitespace in general or ++ before statement comments in particular + * Move first_token from configuration object into format context + * Add line, col info to lex error message + * Fix wrong root parser for FetchContent_MakeAvailable +diff --git a/cmake_format/formatter.py b/cmake_format/formatter.py +index 8015325..8b97cfc 100644 +--- a/cmake_format/formatter.py ++++ b/cmake_format/formatter.py +@@ -1567,7 +1567,30 @@ class WhitespaceNode(LayoutNode): + """ + Compute the size of a whitespace block + """ +- return cursor.clone() ++ fmtconf = stack_context.config.format ++ self._colextent = 0 ++ ++ first_sibling = self.next_sibling() ++ if first_sibling is None: ++ second_sibling = None ++ else: ++ second_sibling = first_sibling.next_sibling() ++ ++ additional_rows = 0 ++ if (isinstance(first_sibling, CommentNode) and ++ isinstance(second_sibling, StatementNode)): ++ additional_rows = fmtconf.numlines_pre_statement_comment ++ ++ if fmtconf.allow_superfluous_newlines: ++ # NOTE: whitespace nodes from the parse tree only emit layout ++ # nodes if they contain 2+ newlines, so all layout nodes contain ++ # at least two ++ clamped_input_rows = clamp( ++ self.pnode.count_newlines() - 2, 0, ++ fmtconf.allow_superfluous_newlines) ++ additional_rows = max(additional_rows, clamped_input_rows) ++ ++ return cursor + (additional_rows, 0) + + def write(self, config, ctx): + return +diff --git a/cmake_format/tests.py b/cmake_format/tests.py +index 64e179a..4b29482 100644 +--- a/cmake_format/tests.py ++++ b/cmake_format/tests.py +@@ -21,7 +21,8 @@ from cmake_format.command_tests import ( + TestFile, + TestInstall, + TestSetTargetProperties, +- TestSet) ++ TestSet, ++ TestWhitespace) + + from cmake_format.command_tests.add_executable_tests \ + import TestAddExecutableCommand diff --git a/cmake_format/pypi/setup.py b/cmake_format/pypi/setup.py index 92eb232..6855c91 100644 --- a/cmake_format/pypi/setup.py +++ b/cmake_format/pypi/setup.py @@ -50,8 +50,8 @@ ], }, extras_require={ - 'YAML': ["pyyaml==5.3"], + 'YAML': ["pyyaml>=5.3"], 'html-gen': ["jinja2==2.10.3"] }, - install_requires=["six==1.14.0"] + install_requires=["six>=1.13.0"] ) diff --git a/cmake_format/tests.py b/cmake_format/tests.py index d48a201..64e179a 100644 --- a/cmake_format/tests.py +++ b/cmake_format/tests.py @@ -12,6 +12,7 @@ from cmake_format.command_tests import ( TestAddCustomCommand, + TestComment, TestConditional, TestCustomCommand, TestExport, @@ -20,7 +21,7 @@ TestFile, TestInstall, TestSetTargetProperties, - TestSet,) + TestSet) from cmake_format.command_tests.add_executable_tests \ import TestAddExecutableCommand @@ -32,8 +33,6 @@ import TestContributorAgreements from cmake_format.contrib.validate_pullrequest \ import TestContribution -from cmake_format.doc.docsources_test \ - import TestDocSources from cmake_format.test.version_number_test \ import TestVersionNumber from cmake_format.test.command_db_test \ diff --git a/cmake_format/tools/bump_version.py b/cmake_format/tools/bump_version.py index d1cd849..9d44d13 100644 --- a/cmake_format/tools/bump_version.py +++ b/cmake_format/tools/bump_version.py @@ -120,6 +120,8 @@ def main(): field_idx = fields.index(args.field) new_version = list(current_version) new_version[field_idx] += 1 + for idx in range(field_idx + 1, len(new_version)): + new_version[idx] = 0 process_init(init_path, new_version) process_installation_rst( diff --git a/cmake_format/tools/create_pseudorelease_tag.py b/cmake_format/tools/create_pseudorelease_tag.py new file mode 100644 index 0000000..90c7aaf --- /dev/null +++ b/cmake_format/tools/create_pseudorelease_tag.py @@ -0,0 +1,58 @@ +""" +Create a tag pseudo- pointing to the current head of on +github. +""" + + +import argparse +import logging +import os + +import github + +logger = logging.getLogger(__name__) + +PSEUDO_MESSAGE = """ +This is a pseudo-release used only to stage artifacts for the release pipeline. +Please do not rely on the artifacts contained in this release as they change +frequently and are very likely to be broken. +""".replace("\n", "") + + +def create_tag(branch): + # TODO(josh): get slug out of the script so that it can be reusable + reposlug = "cheshirekow/cmake_format" + + access_token = os.environ.get("GITHUB_ACCESS_TOKEN") + if access_token is None: + raise RuntimeError("GITHUB_ACCESS_TOKEN missing from environment") + + # TODO(josh): get title out of the script so that it can be reusable + hub = github.Github(access_token) + repo = hub.get_repo(reposlug) + branchobj = repo.get_branch(branch) + logger.info("Creating tag pseudo-%s -> %s", branch, branchobj.commit.sha) + refname = "tags/pseudo-" + branch + try: + existing_ref = repo.get_git_ref(refname) + logger.info("Updating existing ref for %s", refname) + existing_ref.edit(branchobj.commit.sha, force=True) + return + except github.UnknownObjectException: + pass + + logger.info("Creating ref %s", refname) + refname = "refs/" + refname + repo.create_git_ref(refname, branchobj.commit.sha) + + +def main(): + logging.basicConfig(level=logging.INFO) + argparser = argparse.ArgumentParser(description=__doc__) + argparser.add_argument("branch") + args = argparser.parse_args() + create_tag(args.branch) + + +if __name__ == "__main__": + main() diff --git a/cmake_format/tools/gen_rtd_requirements.py b/cmake_format/tools/gen_rtd_requirements.py new file mode 100644 index 0000000..51462d3 --- /dev/null +++ b/cmake_format/tools/gen_rtd_requirements.py @@ -0,0 +1,33 @@ +""" +Generate the rtd-requirements.txt file given a release tag and version number +""" + +import argparse +import io +import os +import sys + +TEMPLATE = "\n".join([ + ("https://github.com/cheshirekow/cmake_format/releases/download/" + "{_tag}/cmake_format-{_version}-py3-none-any.whl"), + "PyYAML==5.3" +]) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tag") + parser.add_argument("version") + parser.add_argument("-o", "--outfile-path", default="-") + + args = parser.parse_args() + if args.outfile_path == "-": + args.outfile_path = os.dup(sys.stdout.fileno()) + + outfile = io.open(args.outfile_path, "w", encoding="utf-8") + content = TEMPLATE.format(_tag=args.tag, _version=args.version) + outfile.write(content) + + +if __name__ == "__main__": + main() diff --git a/cmake_format/tools/get_release_notes.py b/cmake_format/tools/get_release_notes.py new file mode 100644 index 0000000..737334d --- /dev/null +++ b/cmake_format/tools/get_release_notes.py @@ -0,0 +1,84 @@ +""" +Extract release notes for the given tag +""" + +import argparse +import io +import logging +import os +import re +import sys + +logger = logging.getLogger(__name__) + +PSEUDO_MESSAGE = """ +This is a pseudo-release used only to stage artifacts for the release pipeline. +Please do not rely on the artifacts contained in this release as they change +frequently and are very likely to be broken. +""".replace("\n", "") + +GENERIC_MESSAGE = """ +This release was automatically generated by the release pipeline. +""".strip() + + +def iterate_until_version(infile, version): + history = [] + ruler = re.compile("^-+$") + + for line in infile: + line = line.rstrip() + history.append(line) + + if len(history) < 3: + continue + + if (ruler.match(history[-3]) and + version.match(history[-2]) and + ruler.match(history[-1])): + return + + if len(history) > 3: + for buffered_line in history[:-3]: + yield buffered_line + history = history[-3:] + + for buffered_line in history: + yield buffered_line + + +def get_note_text(infile_path, tag): + if tag.startswith("pseudo-"): + return PSEUDO_MESSAGE + + version_regex = re.compile(r"^(v\d+\.\d+\.\d+).*") + match = version_regex.match(tag) + if not match: + return GENERIC_MESSAGE + + version_str = match.group(1) + with io.open(infile_path, "r", encoding="utf-8") as infile: + for _ in iterate_until_version(infile, re.compile(version_str)): + pass + + content = "\n".join(iterate_until_version(infile, version_regex)) + return re.sub(r"(\S)\n(\S)", r"\1\2", content).strip() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("infile_path") + parser.add_argument("tag") + parser.add_argument("-o", "--outfile-path", default="-") + + args = parser.parse_args() + + if args.outfile_path == "-": + args.outfile_path = os.dup(sys.stdout.fileno()) + with io.open(args.outfile_path, "w", encoding="utf-8") as outfile: + outfile.write(get_note_text(args.infile_path, args.tag)) + outfile.write("\n") + + +if __name__ == "__main__": + main() diff --git a/cmake_format/tools/parse_cmake_help.py b/cmake_format/tools/parse_cmake_help.py index 06b93c9..f3d237a 100644 --- a/cmake_format/tools/parse_cmake_help.py +++ b/cmake_format/tools/parse_cmake_help.py @@ -8,6 +8,8 @@ that we can generate parsers for. """ +from __future__ import print_function, unicode_literals + import argparse import io import logging @@ -25,13 +27,27 @@ def get_abspath(relpath): return os.path.abspath(os.path.join(os.path.dirname(__file__), relpath)) +def sub_callback(match): + # NOTE(josh): it would be handly to use named groups in regular expressions, + # but since we use | to join them together into a big regex... the names + # will collide so we cannot + return "(?P<{}>.*)".format(re.sub(r"\W", "_", match.group(1))) + + +GENERIC_LABEL = re.compile(r"<([^>]+)>") + + def make_pattern(namestr): """ Look for any generic labels within a cmake property or variable name (e.g. `` in `_CPPLINT`) and convert the pattern string to a regular expression pattern (e.g. `.*_CPPLINT`) """ - return re.sub("<[^>]+>", ".*", namestr) + return GENERIC_LABEL.sub(sub_callback, namestr) + + +def strip_named_groups(pattern): + return re.sub(r"\(?P<[\w_]>", "(", pattern) def get_properties(args, jenv): @@ -125,18 +141,20 @@ def get_usages(helpstr): return usage -def cmd_get_usages(args, jenv): +def cmd_get_usages(args, jenv): # pylint: disable=W0613 for command_name in get_command_list(args): helpstr = get_command_help(args, command_name) for usage in get_usages(helpstr): print(usage) -def cmd_print_deprecated(args, jenv): + +def cmd_print_deprecated(args, jenv): # pylint: disable=W0613 for command_name in get_command_list(args): helpstr = get_command_help(args, command_name) if "Deprecated" in helpstr: print(command_name) + def setup_argparse(parser): """Setup argument parser""" parser.add_argument( diff --git a/cmake_format/tools/properties.jinja.py b/cmake_format/tools/properties.jinja.py index 3f5eacd..c138bf5 100644 --- a/cmake_format/tools/properties.jinja.py +++ b/cmake_format/tools/properties.jinja.py @@ -1,14 +1,25 @@ # pylint: skip-file """ Database of known property names +NOTE: this file is automatically generated by the script +python -Bm cmake_format.tools.parse_cmake_help properties """ +import re -def get_regex(): - return re.compile("|".join(PATTERNS)) PATTERNS = [ - {%for pattern in patterns%} + {%-for pattern in patterns%} "{{pattern}}", {%-endfor%} ] + +def stripped_patterns(): + regex = re.compile(r"\?P<[\w_]+>") + return [regex.sub("", pattern) for pattern in PATTERNS] + + +CASE_SENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns())) +CASE_INSENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns()), re.IGNORECASE) diff --git a/cmake_format/tools/push_github_release.py b/cmake_format/tools/push_github_release.py new file mode 100644 index 0000000..f13e8af --- /dev/null +++ b/cmake_format/tools/push_github_release.py @@ -0,0 +1,82 @@ +""" +Create or modify the release for the currently built tag on travis. +""" + +import argparse +import io +import logging +import os + +import github +import magic + +logger = logging.getLogger(__name__) + +PSEUDO_MESSAGE = """ +This is a pseudo-release used only to stage artifacts for the release pipeline. +Please do not rely on the artifacts contained in this release as they change +frequently and are very likely to be broken. +""".replace("\n", "") + + +def push_release(tag, message, filepaths): + # TODO(josh): get slug out of the script so that it can be reusable + reposlug = "cheshirekow/cmake_format" + + access_token = os.environ.get("GITHUB_ACCESS_TOKEN") + if access_token is None: + raise RuntimeError("GITHUB_ACCESS_TOKEN missing from environment") + + # TODO(josh): get title out of the script so that it can be reusable + if tag.startswith("pseudo-"): + prerelease = True + title = "pseudo-release artifacts for " + tag[len("pseudo-"):] + else: + prerelease = False + title = "cmake-format " + tag + + filenames = set(filepath.rsplit(os.sep, 1)[-1] for filepath in filepaths) + hub = github.Github(access_token) + repo = hub.get_repo(reposlug) + try: + release = repo.get_release(tag) + logger.info("Found release for tag %s", tag) + if release.title != title or release.body != message: + logger.info("Updating release message") + release.update_release(title, message, prerelease=prerelease) + except github.UnknownObjectException: + logger.info("Creating release for tag %s", tag) + release = repo.create_git_release( + tag, title, message, prerelease=prerelease) + + for asset in release.get_assets(): + if asset.name in filenames: + logger.info("Deleting asset %s", asset.name) + asset.delete_asset() + + for filepath in filepaths: + filename = filepath.rsplit(os.sep, 1)[-1] + mime = magic.Magic(mime=True) + release.upload_asset( + filepath, content_type=mime.from_file(filepath), name=filename) + + +def main(): + argparser = argparse.ArgumentParser(description=__doc__) + argparser.add_argument( + "-m", "--message", + help="path to a file containing release notes message") + argparser.add_argument("tag", help="the tag we are deploying") + argparser.add_argument("files", nargs="*", help="files to upload") + args = argparser.parse_args() + + if args.message: + with io.open(args.message, encoding="utf-8") as infile: + message = infile.read() + else: + message = "" + push_release(args.tag, message, args.files) + + +if __name__ == "__main__": + main() diff --git a/cmake_format/tools/variables.jinja.py b/cmake_format/tools/variables.jinja.py index 43cc978..e52ff23 100644 --- a/cmake_format/tools/variables.jinja.py +++ b/cmake_format/tools/variables.jinja.py @@ -1,14 +1,29 @@ # pylint: skip-file """ Database of known variable names +NOTE: this file is automatically generated by the script +python -Bm cmake_format.tools.parse_cmake_help variables """ +import re -def get_regex(): - return re.compile("|".join(PATTERNS)) PATTERNS = [ - {%for pattern in patterns%} + {%-for pattern in patterns%} "{{pattern}}", {%-endfor%} ] + +def stripped_patterns(): + """ + Remove named groups from patterns so we can join them together without + a name collision. + """ + regex = re.compile(r"\?P<[\w_]+>") + return [regex.sub("", pattern) for pattern in PATTERNS] + + +CASE_SENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns())) +CASE_INSENSITIVE_REGEX = re.compile( + "|".join(stripped_patterns()), re.IGNORECASE) diff --git a/cmake_format/vscode_extension/package-lock.json b/cmake_format/vscode_extension/package-lock.json index 150858a..083b861 100644 --- a/cmake_format/vscode_extension/package-lock.json +++ b/cmake_format/vscode_extension/package-lock.json @@ -1,6 +1,6 @@ { "name": "cmake-format", - "version": "0.6.7", + "version": "0.6.8-dev0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cmake_format/vscode_extension/package.json b/cmake_format/vscode_extension/package.json index 0b32fee..35411ad 100644 --- a/cmake_format/vscode_extension/package.json +++ b/cmake_format/vscode_extension/package.json @@ -2,7 +2,7 @@ "name": "cmake-format", "displayName": "cmake-format", "description": "Format listfiles so they don't look like crap", - "version": "0.6.7", + "version": "0.6.8-dev0", "publisher": "cheshirekow", "repository": "https://github.com/cheshirekow/cmake_format", "icon": "images/cmake-format-logo.png", diff --git a/cmake_lint/CMakeLists.txt b/cmake_lint/CMakeLists.txt index 1ef71d6..a416c16 100644 --- a/cmake_lint/CMakeLists.txt +++ b/cmake_lint/CMakeLists.txt @@ -24,3 +24,12 @@ add_test( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} # --exclude "test/.*" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +set(_genfiles ${CMAKE_CURRENT_SOURCE_DIR}/test/expect_lint.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/test/lint_tests.cmake) +add_custom_command( + OUTPUT ${_genfiles} + COMMAND python -Bm cmake_lint.test.genfiles + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +add_custom_target(cmake_lint-test-genfiles DEPENDS ${_genfiles}) +add_dependencies(gen cmake_lint-test-genfiles) diff --git a/cmake_lint/__main__.py b/cmake_lint/__main__.py index 991ee4e..bdd6131 100644 --- a/cmake_lint/__main__.py +++ b/cmake_lint/__main__.py @@ -35,13 +35,12 @@ def process_file(config, local_ctx, infile_content): basic_checker.check_basics(config, local_ctx, infile_content) tokens = lexer.tokenize(infile_content) - config.first_token = lexer.get_first_non_whitespace_token(tokens) parse_db = parse_funs.get_parse_db() parse_db.update(parse_funs.get_legacy_parse(config.parse.fn_spec).kwargs) ctx = parse.ParseContext(parse_db, local_ctx, config) parse_tree = parse.parse(tokens, ctx) parse_tree.build_ancestry() - basic_checker.check_tree(config, local_ctx, parse_tree) + basic_checker.check_parse_tree(config, local_ctx, parse_tree) def setup_argparse(argparser): diff --git a/cmake_lint/basic_checker.py b/cmake_lint/basic_checker.py index 1cef636..f5a16eb 100644 --- a/cmake_lint/basic_checker.py +++ b/cmake_lint/basic_checker.py @@ -11,6 +11,7 @@ from cmake_format.parse.common import NodeType, TreeNode from cmake_format.parse.statement_node import StatementNode from cmake_format.parse.util import get_min_npargs +from cmake_format.parse import variables def find_statements_in_subtree(subtree, funnames): @@ -376,7 +377,7 @@ def check_arggroup(cfg, local_ctx, node): def check_positional_group(cfg, local_ctx, node): - """Perform checks on a positinal group node.""" + """Perform checks on a positional group node.""" min_npargs = get_min_npargs(node.spec.npargs) semantic_tokens = node.get_semantic_tokens() if len(semantic_tokens) < min_npargs: @@ -506,6 +507,40 @@ def check_statement(cfg, local_ctx, node): check_tree(cfg, local_ctx, child) +def check_varname(cfg, local_ctx, varname, token): + """ + Record lint if the varname is invalid + """ + if (variables.CASE_INSENSITIVE_REGEX.match(varname) and + not variables.CASE_SENSITIVE_REGEX.match(varname)): + local_ctx.record_lint( + "W0105", "Assignment to", varname, + location=token.get_location()) + + +def check_variable_assignments(cfg, local_ctx, tree): + for stmt in find_statements_in_subtree(tree, ["set", "list"]): + if stmt.get_funname() == "set": + token = stmt.argtree.varname + elif stmt.get_funname() == "list": + token = stmt.argtree.parg_groups[0].get_tokens(kind="semantic")[1] + else: + continue + check_varname(cfg, local_ctx, token.spelling, token) + + +def check_variable_references(cfg, local_ctx, tree): + # TODO(josh): replace with a stateful parser that builds up + # global/directory/local namespaces and can check for usage before assignment, + # shadowing, etc + for token in tree.get_tokens(kind="semantic"): + if token.type not in ( + TokenType.QUOTED_LITERAL, TokenType.DEREF): + continue + for varname in re.findall(r"\$\{([\w_]+)\}", token.spelling): + check_varname(cfg, local_ctx, varname, token) + + def check_tree(cfg, local_ctx, node): if not isinstance(node, TreeNode): return @@ -522,3 +557,9 @@ def check_tree(cfg, local_ctx, node): else: for child in node.children: check_tree(cfg, local_ctx, child) + + +def check_parse_tree(cfg, local_ctx, node): + check_variable_assignments(cfg, local_ctx, node) + check_variable_references(cfg, local_ctx, node) + check_tree(cfg, local_ctx, node) diff --git a/cmake_lint/gendocs.py b/cmake_lint/gendocs.py index c520214..91b576b 100644 --- a/cmake_lint/gendocs.py +++ b/cmake_lint/gendocs.py @@ -3,6 +3,8 @@ Generate linter documentation page """ +from __future__ import unicode_literals + import argparse import io import logging @@ -11,7 +13,7 @@ import textwrap import cmake_format -from cmake_format.doc.gendoc_sources import format_directive +from cmake_format.doc.gendoc import format_directive from cmake_lint import lintdb HEADER = """ diff --git a/cmake_lint/lint_util.py b/cmake_lint/lint_util.py index 13fa554..376e218 100644 --- a/cmake_lint/lint_util.py +++ b/cmake_lint/lint_util.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import collections import logging diff --git a/cmake_lint/lintdb.py b/cmake_lint/lintdb.py index 41c54ef..8e1acb6 100644 --- a/cmake_lint/lintdb.py +++ b/cmake_lint/lintdb.py @@ -5,11 +5,12 @@ * (C) convention, for programming standard violation * (R) refactor, for bad code smell -* (W) warning, for python specific problems -* (E) error, for much probably bugs in the code +* (W) warning, for cmake specific problems +* (E) error, for most probably bugs in the code * (F) fatal, if an error occurred which prevented cmake-lint from doing further processing. """ +from __future__ import unicode_literals class Lint(object): @@ -241,5 +242,19 @@ def get_database(): "W0101", "Unreachable code", { }), ( "W0104", "Use of deprecated command {:s}", { -}), +}), ( +"W0105", +"{:s} variable '{:s}' which matches a built-in except for case", { +"description": """ +This warning means that you are using a variable such as, +for example, `cmake_cxx_standard` which matches a builtin variable +(`CMAKE_CXX_STANDARD`) except for the case. If this was intentional, then it's +bad practice as it causes confusion (there are two variables in the namespace +with identical name except for case), though it was probably not intentional +and you probably aren't assigning to the correct variable. + +This warning may be emitted for assignment (e.g. `set()` or `list()`) as +well as for variable expansion in an argument (e.g. `"${CMAKE_Cxx_STANDARD}"`). +""" +}) ] diff --git a/cmake_lint/test/expect_lint.cmake b/cmake_lint/test/expect_lint.cmake index 7bd1962..e35ccd0 100644 --- a/cmake_lint/test/expect_lint.cmake +++ b/cmake_lint/test/expect_lint.cmake @@ -143,4 +143,10 @@ function(foo) set(INVALID_LOCAL_NAME "foo") endfunction() +set(CMAKE_Cxx_STANDARD "11") + +list(APPEND CMAKE_Cxx_STANDARD "11") + +message("Using C++ standard ${CMAKE_Cxx_STANDARD}") + # This file is missing a final newline \ No newline at end of file diff --git a/cmake_lint/test/lint_tests.cmake b/cmake_lint/test/lint_tests.cmake index 3514c34..a5c1728 100644 --- a/cmake_lint/test/lint_tests.cmake +++ b/cmake_lint/test/lint_tests.cmake @@ -197,6 +197,18 @@ function(foo) set(INVALID_LOCAL_NAME "foo") endfunction() +# test: set-wrong-case +# expect: W0105 +set(CMAKE_Cxx_STANDARD "11") + +# test: list-wrong-case +# expect: W0105 +list(APPEND CMAKE_Cxx_STANDARD "11") + +# test: uses-var-wrong-case +# expect: W0105 +message("Using C++ standard ${CMAKE_Cxx_STANDARD}") + # test: missing-final-newline # expect: C0304 # This file is missing a final newline \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py index 3830b8e..7773684 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,15 +1,22 @@ # -*- coding: utf-8 -*- +from __future__ import print_function + import os import sphinx_rtd_theme +import subprocess +import tempfile + from recommonmark.parser import CommonMarkParser from recommonmark.transform import AutoStructify -filepath = os.path.realpath(__file__) -if os.path.islink(filepath): - dirname = os.path.dirname(filepath) - _, project = os.path.split(dirname) -else: - project = 'tangentsky' +project = 'tangentsky' +thisfile = os.path.realpath(__file__) +rootdir = globals().get( + "rootdir", os.sep.join(thisfile.split(os.sep)[:-3])) + +if os.environ.get("READTHEDOCS") == "True": + # Do any RTD-specific setup here + pass # General information about the project. docname = project + u'doc' @@ -28,9 +35,10 @@ 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx.ext.autosectionlabel', + # "readthedocs_ext.readthedocs", ] -autosection_label_prefix_document = True +autosectionlabel_prefix_document = True mathjax_path = ( "https://cdnjs.cloudflare.com/ajax/libs/mathjax/" @@ -117,6 +125,8 @@ intersphinx_mapping = {'https://docs.python.org/': None} # Advanced markdown + + def setup(app): app.add_config_value('recommonmark_config', { 'auto_code_block': True, diff --git a/requirements.txt b/requirements.txt index 89fc99a..ef97bc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ -astroid==2.2.5; python_version >= '3.0' +astroid==2.3.3; python_version >= '3.0' autopep8==1.3.5; python_version < '3.0' autopep8==1.4.3; python_version >= '3.0' flake8==3.6.0; python_version < '3.0' -flake8==3.7.7; python_version >= '3.0' -jinja2==2.10.1 +flake8==3.7.9; python_version >= '3.0' +jinja2==2.10.3 pylint==1.9.1; python_version < '3.0' -pylint==2.2.2; python_version >= '3.0' -pyyaml==4.2b1 +pylint==2.4.4; python_version >= '3.0' +PyYAML==5.3 PGPy==0.5.2; python_version >= '3.0' - +PyGithub==1.45 +python-magic==0.4.15 diff --git a/tangent/CMakeLists.txt b/tangent/CMakeLists.txt new file mode 100644 index 0000000..297005a --- /dev/null +++ b/tangent/CMakeLists.txt @@ -0,0 +1,3 @@ +if(EXISTS gtkutil/CMakeLists.txt) +add_subdirectory(gtkutil) +endif() diff --git a/tangent/__init__.py b/tangent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tangent/tooling/__init__.py b/tangent/tooling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tangent/tooling/clean_stage.py b/tangent/tooling/clean_stage.py new file mode 100644 index 0000000..74d1c94 --- /dev/null +++ b/tangent/tooling/clean_stage.py @@ -0,0 +1,65 @@ +""" +Given a staging directory and a file containing a list of what should be in +that directory, delete any files that shouldn't be there. +""" + +import argparse +import io +import logging +import os +import sys + +logger = logging.getLogger(__name__) + + +def setup_argparser(argparser): + argparser.add_argument( + "manifest_path", help="path to the manifest file") + argparser.add_argument( + "stage_path", help="path to the staging directory") + + +def get_argdict(args): + out = {} + for key, value in vars(args).items(): + if key.startswith("_"): + continue + out[key] = value + return out + + +def inner_main(manifest_path, stage_path): + with io.open(manifest_path, "r", encoding="utf-8") as infile: + manifest = set(line.strip() for line in infile) + + for directory, _dirnames, filenames in os.walk(stage_path): + relpath_dir = os.path.relpath(directory, stage_path) + + for filename in filenames: + if relpath_dir == ".": + relpath_file = filename + else: + relpath_file = os.path.join(relpath_dir, filename) + fullpath_file = os.path.join(directory, filename) + + if relpath_file not in manifest and fullpath_file not in manifest: + logger.info("Deleting %s", relpath_file) + os.unlink(fullpath_file) + + +def main(): + logging.basicConfig(level=logging.INFO) + argparser = argparse.ArgumentParser(description=__doc__) + setup_argparser(argparser) + try: + import argcomplete + argcomplete.autocomplete(argparser) + except ImportError: + pass + args = argparser.parse_args() + inner_main(**get_argdict(args)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tangent/tooling/deploy_keys/cmake-format.enc b/tangent/tooling/deploy_keys/cmake-format.enc new file mode 100644 index 0000000..e46a99c Binary files /dev/null and b/tangent/tooling/deploy_keys/cmake-format.enc differ diff --git a/tangent/tooling/deploy_keys/cmake-tools-rtd.enc b/tangent/tooling/deploy_keys/cmake-tools-rtd.enc new file mode 100644 index 0000000..af90151 Binary files /dev/null and b/tangent/tooling/deploy_keys/cmake-tools-rtd.enc differ diff --git a/tangent/tooling/export_sparse_subrepo.py b/tangent/tooling/export_sparse_subrepo.py new file mode 100644 index 0000000..27754f3 --- /dev/null +++ b/tangent/tooling/export_sparse_subrepo.py @@ -0,0 +1,118 @@ +""" +Synchronize or validate a sparse subrepository export. The sparse export +configuration is stored in the `.sparse-export` file in the root of the +repository. If the file does not exist, then there is nothing to update +or verify. +""" + +import argparse +import hashlib +import io +import logging +import os +import shutil +import subprocess +import sys + +logger = logging.getLogger(__name__) + + +def setup_argparser(argparser): + argparser.add_argument( + "command", choices=["update", "verify"], + help="whether to update or verify the export") + + +def get_argdict(args): + out = {} + for key, value in vars(args).items(): + if key.startswith("_"): + continue + out[key] = value + return out + + +def copy_file(sourcepath, destpath): + destdir = os.path.dirname(destpath) + if not os.path.exists(destdir): + os.makedirs(destdir) + shutil.copy2(sourcepath, destpath) + return 0 + + +def read_chunks(infile, chunksize=4096): + while True: + chunk = infile.read(chunksize) + if chunk == b'': + break + yield chunk + + +def hash_file(filepath, hashname="sha1", chunksize=4096): + hashobj = hashlib.new(hashname) + with open(filepath, "rb") as infile: + for chunk in read_chunks(infile): + hashobj.update(chunk) + return hashobj + + +def verify_file(sourcepath, destpath): + if not os.path.exists(destpath): + logger.error("Missing %s", destpath) + + if hash_file(sourcepath).digest() == hash_file(destpath).digest(): + return 0 + logger.error("File %s differs from source", destpath) + return 1 + + +def inner_main(command): + if command == "update": + operation = copy_file + elif command == "verify": + operation = verify_file + else: + raise ValueError("Unknown command {}".format(command)) + + repodir = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"]).strip().decode("utf-8") + configfile = os.path.join(repodir, ".sparse-export") + if not os.path.exists(configfile): + return 0 + + with io.open(configfile, "r", encoding="utf-8") as infile: + relpath_export = infile.read().strip() + export_dir = os.path.join(repodir, relpath_export) + + returncode = 0 + for directory, _dirnames, filenames in os.walk(export_dir): + reldir = os.path.relpath(directory, export_dir) + for filename in filenames: + if reldir == ".": + relpath_file = filename + else: + relpath_file = os.path.join(reldir, filename) + + sourcepath = os.path.join(directory, filename) + if relpath_file == "sparse-checkout": + relpath_file = ".gitu/info/sparse-checkout" + destpath = os.path.join(repodir, relpath_file) + returncode |= operation(sourcepath, destpath) + return returncode + + +def main(): + logging.basicConfig(level=logging.INFO) + argparser = argparse.ArgumentParser(description=__doc__) + setup_argparser(argparser) + try: + import argcomplete + argcomplete.autocomplete(argparser) + except ImportError: + pass + args = argparser.parse_args() + return inner_main(**get_argdict(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tangent/tooling/github.py b/tangent/tooling/github.py new file mode 100644 index 0000000..fc20f19 --- /dev/null +++ b/tangent/tooling/github.py @@ -0,0 +1,312 @@ +# PYTHON_ARGCOMPLETE_OK +""" +Commandline client for github API +""" + +import argparse +import collections +import inspect +import io +import json +import logging +import os +import re +import sys + +import requests + +logger = logging.getLogger(__name__) + + +def parse_bool(string): + if string.lower() in ('y', 'yes', 't', 'true', '1', 'yup', 'yeah', 'yada'): + return True + if string.lower() in ('n', 'no', 'f', 'false', '0', 'nope', 'nah', 'nada'): + return False + + logger.warning("Ambiguous truthiness of string '%s' evalutes to 'FALSE'", + string) + return False + + +if sys.version_info >= (3, 0, 0): + VALUE_TYPES = (str, int, float) +else: + VALUE_TYPES = (str, unicode, int, float) + + +class Descriptor(object): + # TODO(josh): consider storing a global construction order among + # descriptors. This would allow us to iterate over config object descriptors + # in deterministic (construction) order even in pythons where + # __dict__.items() does not return objects in declaration order. + pass + + +class FieldDescriptor(Descriptor): + """Implements the descriptor interface to store metadata (default value, + docstring, and choices) for each configuration variable. + """ + + def __init__( + self, default_value=None, helptext=None, choices=None, required=False, + positional=False): + super(FieldDescriptor, self).__init__() + self.default_value = default_value + self.helptext = helptext + self.choices = choices + self.required = required + self.positional = positional + self.name = "" + + def __get__(self, obj, objtype): + return getattr(obj, "_" + self.name, self.default_value) + + def __set__(self, obj, value): + setattr(obj, "_" + self.name, value) + + def __set_name__(self, owner, name): + # pylint: disable=protected-access + owner._field_registry.append(self) + self.name = name + + def consume_value(self, obj, value): + """Convenience method to consume values directly from the descriptor + interface.""" + self.__set__(obj, value) + + def has_override(self, obj): + """Return true if `obj` has an override for this configuration variable.""" + return hasattr(obj, "_" + self.name) + + def add_to_argparse(self, optgroup): + """Add the config variable as an argument to the command line parser.""" + if self.name == 'additional_commands': + return + + argname = self.name + kwargs = { + "help": self.helptext + } + if not self.positional: + argname = "--" + self.name.replace("_", "-") + kwargs["required"] = self.required + + if isinstance(self.default_value, bool): + # NOTE(josh): argparse store_true isn't what we want here because we + # want to distinguish between "not specified" = "default" and + # "specified" + optgroup.add_argument( + argname, nargs='?', + default=None, const=(not self.default_value), + type=parse_bool, **kwargs) + elif isinstance(self.default_value, VALUE_TYPES): + optgroup.add_argument( + argname, + type=type(self.default_value), + choices=self.choices, **kwargs) + elif self.default_value is None: + # If the value is None then we can't really tell what it's supposed to + # be. I guess let's assume string in this case. + optgroup.add_argument( + argname, + choices=self.choices, **kwargs) + # NOTE(josh): argparse behavior is that if the flag is not specified on + # the command line the value will be None, whereas if it's specified with + # no arguments then the value will be an empty list. This exactly what we + # want since we can ignore `None` values. + elif isinstance(self.default_value, (list, tuple)): + typearg = None + if self.default_value: + typearg = type(self.default_value[0]) + optgroup.add_argument( + argname, + nargs='*', type=typearg, **kwargs) + + +def class_to_cmd(name): + intermediate = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1-\2', intermediate).lower() + + +class Command(object): + """ + Base class making it a little easier to set up a complex argparse tree by + specifying features of a command as members of a class. + """ + _field_registry = [] + + @classmethod + def cmdname(cls): + """ + Return a string command name formulated by de-camael-casing the class + name. + """ + return class_to_cmd(cls.__name__) + + @classmethod + def add_parser(cls, subparsers): + """ + Add a subparser to the list of subparsers, and then call the classmethod + to configure that subparser. + """ + subparser = subparsers.add_parser(cls.cmdname(), help=cls.__doc__) + for descr in cls._field_registry: + descr.add_to_argparse(subparser) + cls.setup_subparser(subparser) + + @classmethod + def setup_subparser(cls, subparser): + """ + Configure subparser for this command. Override in subclasses. + """ + + def __init__(self, _config, **kwargs): + for _, descr in self._field_registry: + if descr.name in kwargs: + descr.consume_value(self, kwargs.pop(descr.name)) + + def __call__(self, *args, **kwargs): + raise NotImplementedError('__call__ unimplemented for object of type {}' + .format(type(self).__name__)) + + +class ReleaseBase(Command): + _field_registry = [] + + target_commitish = FieldDescriptor( + "", + helptext=( + "Specifies the commitish value that determines where the Git tag is" + " created from. Can be any branch or commit SHA. Unused if the Git" + " tag already exists. Default: the repository's default branch" + " (usually master).")) + + name = FieldDescriptor("", helptext="The name of the release.") + body = FieldDescriptor("", helptext="Text describing the contents of the tag") + draft = FieldDescriptor( + False, + helptext=( + "true to create a draft (unpublished) release, false to create a" + " published one. Default: false")) + + prerelease = FieldDescriptor( + False, + helptext=( + "true to identify the release as a prerelease. false to identify the" + " release as a full release. Default: false")) + + def get_data(self): + if self.body.startswith("file://"): + with io.open(self.body[len("file://"):], "r", encoding="utf-8") as infile: + body = infile.read() + else: + body = self.body + + return { + "tag_name": self.tag_name, + "target_commitish": self.target_commitish, + "name": self.name, + "body": body, + "draft": self.draft, + "prerelease": self.prerelease + } + + +class CreateRelease(ReleaseBase): + """ + :see: https://developer.github.com/v3/repos/releases/#create-a-release + """ + + _field_registry = list(ReleaseBase._field_registry) + + tag_name = FieldDescriptor( + "", positional=True, helptext="Required. The name of the tag.") + + def __call__(self, http): + data = self.get_data() + return http.post("releases") + + +class EditRelease(ReleaseBase): + """ + :see: https://developer.github.com/v3/repos/releases/#edit-a-release + """ + + _field_registry = list(ReleaseBase._field_registry) + + tag_name = FieldDescriptor( + "", helptext="The name of the tag.") + + release_id = FieldDescriptor( + "", positional=True, helptext="id of the release to edit") + + def __call__(self, http): + data = self.get_data() + return http.patch("releases/:release_id") + + +class Repo(Command): + + _field_registry = [] + + repo_slug = FieldDescriptor( + "", positional=True, helptext="Required. /") + + _subcmds = { + cls.cmdname(): cls for cls in [ + CreateRelease, EditRelease + ]} + + @classmethod + def setup_subparser(cls, subparser): + subsubparser = subparser.add_subparsers(dest="repo_cmd") + + for _, cmd in cls._subcmds.items(): + cmd.add_parser(subsubparser) + + def __call__(self, http, repo_cmd, **kwargs): + cmdobj = self._subcmds[repo_cmd](None, **kwargs) + cmdobj(None, **kwargs) + + +def get_argdict(namespace): + out = {} + for key, value in vars(namespace).items(): + if key.startswith("_"): + continue + out[key] = value + return out + + +def setup_argparser(argparser, cmds): + subparsers = argparser.add_subparsers( + dest="command", help="sub-command. See --help") + for _, cmd in cmds.items(): + cmd.add_parser(subparsers) + + +def main(): + logging.basicConfig(level=logging.INFO) + argparser = argparse.ArgumentParser(description=__doc__) + cmds = { + cls.cmdname(): cls for cls in [ + Repo + ]} + setup_argparser(argparser, cmds) + + try: + import argcomplete + argcomplete.autocomplete(argparser) + except ImportError: + pass + + args = argparser.parse_args() + argdict = get_argdict(args) + + cmdobj = cmds[args.command](None, **argdict) + cmdobj(None, **argdict) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tangent/tooling/sync_doc_artifacts.py b/tangent/tooling/sync_doc_artifacts.py new file mode 100644 index 0000000..09254d8 --- /dev/null +++ b/tangent/tooling/sync_doc_artifacts.py @@ -0,0 +1,258 @@ +# PYTHON_ARGCOMPLETE_OK +""" +Given a commit within an export repository, create a new commit in the +corresponding documentation artifacts repository. +""" + +from __future__ import unicode_literals + +import argparse +import io +import logging +import os +import pipes +import random +import shutil +import subprocess +import sys +import tempfile + +logger = logging.getLogger(__name__) + +INITDIRS = [ + "branches", + "hooks", + "info", + "objects/info", + "objects/pack", + "refs/heads", + "refs/tags", +] + +CONFIGTPL = """ +[core] + repositoryformatversion = 0 + filemode = true + bare = true +[remote "origin"] + url = {} + fetch = +refs/heads/*:refs/heads/* +""" + +MSGTPL = """Sync {commit_name} + +This commit was automatically generated by the upstream build. + +Upstream-Commit: {upstream_commit} +""" + +CHARS = "abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXYZ012345679" + + +def make_randstr(strlen=10): + """Generate a random string.""" + return ''.join(random.choice(CHARS) for i in range(strlen)) + + +def init_repository(repodir, repo_url): + """ + Ensure that the specified directory is configured as a git repository. + We create any missing directories and overwrite the config and HEAD files, + putting the repository directory into a known state without blowing away + the object cache. + """ + + # Ensure that the standard directories exit + for dirname in INITDIRS: + dirpath = os.path.join(repodir, dirname) + if os.path.lexists(dirpath): + if os.path.islink(dirpath) or not os.path.isdir(dirpath): + os.unlink(dirpath) + else: + continue + os.makedirs(dirpath) + + # Write out the config and the HEAD files + outpath = os.path.join(repodir, "config") + with io.open(outpath, "w", encoding="utf-8") as outfile: + outfile.write(CONFIGTPL.format(repo_url)) + outpath = os.path.join(repodir, "HEAD") + with io.open(outpath, "w", encoding="utf-8") as outfile: + outfile.write("ref: refs/heads/master") + + +def canonicalize_trailer_key(keystr): + """Return canonical capitalization for a trailer key""" + return "-".join(part.capitalize() for part in keystr.split("-")) + + +def get_trailers(repodir, commit): + """Return a dictionary of git commit trailers.""" + proc = subprocess.Popen([ + "git", "show", "--no-patch", '--format="%(trailers:unfold)', + commit], cwd=repodir, stdout=subprocess.PIPE) + + out = {} + with proc.stdout as stdout: + for line in stdout: + line = line.strip().decode("utf-8") + if ":" not in line: + continue + key, value = line.split(":", 1) + if ", " in value: + value = value.split(", ") + out[canonicalize_trailer_key(key)] = value + proc.wait() + if proc.returncode != 0: + cmdstr = " ".join(pipes.quote(arg) for arg in proc.args) + raise subprocess.CalledProcessError(proc.returncode, cmdstr) + return out + + +def ref_exists(repodir, refname): + return subprocess.call( + ["git", "show-ref", "--verify", "--quiet", "refs/" + refname], + cwd=repodir) == 0 + + +def inner_main( + docrepo_url, docrepo_dir, scratch_dir, stage_dir, deploy_key_path, + branch=None, tag=None): + + current_commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"]).strip().decode("utf-8") + + env = os.environ.copy() + if deploy_key_path: + env["GIT_SSH_COMMAND"] = "ssh -i {} -F /dev/null".format(deploy_key_path) + + # Initialize and fetch the docrepo clone + init_repository(docrepo_dir, docrepo_url) + subprocess.check_call([ + "git", "fetch", "--prune", "--tags"], cwd=docrepo_dir, env=env) + + # Create a temporary branchname for us to do our work in + workbranch = "work-" + make_randstr(10) + + # Clear out stratch tree + if os.path.exists(scratch_dir): + shutil.rmtree(scratch_dir) + os.makedirs(scratch_dir) + + if branch: + commit_name = branch + if ref_exists(docrepo_dir, "heads/" + branch): + # If the branch already exists on the remote, then check it out at + # it's current state + subprocess.check_call( + ["git", "--git-dir=" + docrepo_dir, "--work-tree=" + scratch_dir, + "checkout", "-b", workbranch, branch]) + else: + # Otherwise create a new orphan branch for it + subprocess.check_call( + ["git", "--git-dir=" + docrepo_dir, "--work-tree=" + scratch_dir, + "checkout", "--orphan", workbranch]) + elif tag: + commit_name = tag + if ref_exists(docrepo_dir, "tags/" + tag): + # If the tag already exists on the remote then checkout it's parent + subprocess.check_call( + ["git", "--git-dir=" + docrepo_dir, "--work-tree=" + scratch_dir, + "checkout", "-b", workbranch, "{}^".format(tag)]) + else: + # Otherwise checkout the current HEAD of master and just use that. + subprocess.check_call( + ["git", "--git-dir=" + docrepo_dir, "--work-tree=" + scratch_dir, + "checkout", "-b", workbranch, "master"]) + else: + raise ValueError("Both branch and tag are empty") + + subprocess.check_call( + ["git", "--git-dir=" + docrepo_dir, "--work-tree=" + stage_dir, + "add", "-A"]) + + tfile = tempfile.NamedTemporaryFile( + delete=False, prefix="gitmsg-", mode="w", encoding="utf-8") + with tfile as outfile: + msgpath = outfile.name + outfile.write(MSGTPL.format( + commit_name=commit_name, upstream_commit=current_commit)) + subprocess.check_call([ + "git", "--git-dir=" + docrepo_dir, "--work-tree=" + stage_dir, + "commit", "--file", msgpath]) + os.unlink(msgpath) + + if branch: + subprocess.check_call( + ["git", "push", "-f", "origin", "{}:{}".format(workbranch, branch)], + cwd=docrepo_dir, env=env) + elif tag: + subprocess.check_call( + ["git", "tag", "-f", tag], + cwd=docrepo_dir) + subprocess.check_call( + ["git", "push", "-f", "origin", "{0}:{0}".format(tag)], + cwd=docrepo_dir, env=env) + + +def setup_argparser(argparser): + argparser.add_argument( + "--doc-repo", dest="docrepo_url", + help="URL of the documentation repository") + argparser.add_argument( + "--repo-dir", dest="docrepo_dir", + help="path to where we should checkout the doc artifacts repo") + argparser.add_argument( + "--scratch-tree", dest="scratch_dir", + help="directory to use as git work-tree for intermediate steps") + argparser.add_argument( + "--stage", dest="stage_dir", + help="directory that contains the new content for the repository") + argparser.add_argument( + "--deploy-key", dest="deploy_key_path", + help="Path to the deploy key for the documentation repository") + + mgroup = argparser.add_mutually_exclusive_group(required=True) + mgroup.add_argument( + "--branch", + help="Name of the target branch to push to" + ) + mgroup.add_argument( + "--tag", + help="Name of the target tag to push to" + ) + + +def get_argdict(namespace): + blacklist = tuple() + out = {} + for key, value in vars(namespace).items(): + if key.startswith("_"): + continue + if key in blacklist: + continue + out[key] = value + return out + + +def main(): + logging.basicConfig(level=logging.INFO) + argparser = argparse.ArgumentParser(description=__doc__) + setup_argparser(argparser) + try: + import argcomplete + argcomplete.autocomplete(argparser) + except ImportError: + pass + + args = argparser.parse_args() + if args.tag.startswith("pseudo-"): + args.branch = args.tag[len("pseudo-"):] + args.tag = None + + inner_main(**get_argdict(args)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())