From 15cc0b3086ec002499121ab295587e6538e4f999 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Fri, 13 Dec 2024 17:49:54 -0800 Subject: [PATCH] fix(gazelle): Support parsing files that use Python3.12 PEP 695 (Type Parameter Syntax) by using dougthor42's fork of go-tree-sitter (#2496) Replaces #2413. Fixes #2396. This updates the `go-tree-sitter` dependency to use my fork that includes `BUILD.bazel` files. Specifically, the `BUILD.bazel` files in the fork include references to top-level code like `array.h` which the original Gazelle-generated files for `go-tree-sitter` were not able to handle. I also include the test cases that @maffoo created in #2413 and verified that they (a) fail before the fix and (b) pass after the fix. The fork is: https://github.com/dougthor42/go-tree-sitter The branch that includes all changes is: https://github.com/dougthor42/go-tree-sitter/tree/for-rules-python-gazelle-plugin A couple notes: + I have a PR open to get `go-tree-sitter` into BCR [here](https://github.com/bazelbuild/bazel-central-registry/pull/3366). However: 1. I'm having trouble getting tests to pass and to get things running locally to validate it 2. Using BCR would not fix things for people who still use WORKSPACE (right?) + The fork is _mostly_ [autogenerated BUILD.bazel files from gazelle](https://github.com/smacker/go-tree-sitter/commit/cfa9bdf58beb30159807c1846e1c761bce1e6158) but also contains: + [manual updates so that build files reference the toplevel `array.h` and other files](https://github.com/smacker/go-tree-sitter/commit/63f89cd3d471e7e81b51dc7e7205b201fed70fc1) + [replace all `smacker` with `dougthor42` so that `go build` works](https://github.com/smacker/go-tree-sitter/commit/8a73cbdb0e9b5febc314f8b9b4241aa01cc86bd0) + various other more minor things. + I was unable to get `go mod edit -replace` to work, so I've just manually updated `go.mod` and whatnot everywhere. If someone with more go knowledge has a suggestion I'm happy to hear it. --------- Co-authored-by: Matthew Neeley Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 8 +++++-- gazelle/MODULE.bazel | 2 +- gazelle/deps.bzl | 8 +++---- gazelle/go.mod | 2 +- gazelle/go.sum | 10 ++------- gazelle/python/BUILD.bazel | 4 ++-- gazelle/python/file_parser.go | 8 +++---- gazelle/python/testdata/py312_syntax/BUILD.in | 1 + .../python/testdata/py312_syntax/BUILD.out | 16 ++++++++++++++ .../python/testdata/py312_syntax/README.md | 4 ++++ .../python/testdata/py312_syntax/WORKSPACE | 1 + .../python/testdata/py312_syntax/__init__.py | 0 .../testdata/py312_syntax/_other_module.py | 0 .../py312_syntax/pep_695_type_parameter.py | 22 +++++++++++++++++++ .../python/testdata/py312_syntax/test.yaml | 1 + 15 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 gazelle/python/testdata/py312_syntax/BUILD.in create mode 100644 gazelle/python/testdata/py312_syntax/BUILD.out create mode 100644 gazelle/python/testdata/py312_syntax/README.md create mode 100644 gazelle/python/testdata/py312_syntax/WORKSPACE create mode 100644 gazelle/python/testdata/py312_syntax/__init__.py create mode 100644 gazelle/python/testdata/py312_syntax/_other_module.py create mode 100644 gazelle/python/testdata/py312_syntax/pep_695_type_parameter.py create mode 100644 gazelle/python/testdata/py312_syntax/test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8605a4a03d..33719187f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,12 +65,16 @@ Unreleased changes template. available for all of the platforms and the sdist can be built. This fix is for both WORKSPACE and `bzlmod` setups. Fixes [#2450](https://github.com/bazelbuild/rules_python/issues/2450). +* (gazelle) Gazelle will now correctly parse Python3.12 files that use [PEP 695 Type + Parameter Syntax][pep-695]. (#2396) + +[pep-695]: https://peps.python.org/pep-0695/ {#v0-0-0-added} ### Added * (gazelle) Added `include_stub_packages` flag to `modules_mapping`. When set to `True`, this automatically includes corresponding stub packages for third-party libraries - that are present and used (e.g., `boto3` → `boto3-stubs`), improving + that are present and used (e.g., `boto3` → `boto3-stubs`), improving type-checking support. * (pypi) Freethreaded packages are now fully supported in the {obj}`experimental_index_url` usage or the regular `pip.parse` usage. @@ -137,7 +141,7 @@ Other changes: * (repositories): Add libs/python3.lib and pythonXY.dll to the `libpython` target defined by a repository template. This enables stable ABI builds of Python extensions on Windows (by defining Py_LIMITED_API). -* (rules) `py_test` and `py_binary` targets no longer incorrectly remove the +* (rules) `py_test` and `py_binary` targets no longer incorrectly remove the first `sys.path` entry when using {obj}`--bootstrap_impl=script` {#v1-0-0-added} diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index 0a553831c3..6bbc74bc61 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -21,9 +21,9 @@ use_repo( go_deps, "com_github_bazelbuild_buildtools", "com_github_bmatcuk_doublestar_v4", + "com_github_dougthor42_go_tree_sitter", "com_github_emirpasic_gods", "com_github_ghodss_yaml", - "com_github_smacker_go_tree_sitter", "com_github_stretchr_testify", "in_gopkg_yaml_v2", "org_golang_x_sync", diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index 948d61e5ae..1bdf179e98 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -186,10 +186,10 @@ def go_deps(): version = "v0.0.0-20190812154241-14fe0d1b01d4", ) go_repository( - name = "com_github_smacker_go_tree_sitter", - importpath = "github.com/smacker/go-tree-sitter", - sum = "h1:7QZKUmQfnxncZIJGyvX8M8YeMfn8kM10j3J/2KwVTN4=", - version = "v0.0.0-20240422154435-0628b34cbf9c", + name = "com_github_dougthor42_go_tree_sitter", + importpath = "github.com/dougthor42/go-tree-sitter", + sum = "h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ=", + version = "v0.0.0-20241210060307-2737e1d0de6b", ) go_repository( name = "com_github_stretchr_objx", diff --git a/gazelle/go.mod b/gazelle/go.mod index 4b65e71d67..29a0b5cb0c 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -7,9 +7,9 @@ require ( github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82 github.com/bazelbuild/rules_go v0.41.0 github.com/bmatcuk/doublestar/v4 v4.6.1 + github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 - github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.2.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/gazelle/go.sum b/gazelle/go.sum index 46e0127e8f..d48da9ece3 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum @@ -13,9 +13,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ= +github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b/go.mod h1:87UkDyPt18bTH/FvinLc/kj587VNYOdRKZT1la4T8Hg= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -44,12 +45,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c h1:7QZKUmQfnxncZIJGyvX8M8YeMfn8kM10j3J/2KwVTN4= -github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= @@ -105,7 +100,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 627a867c68..893c82e8e4 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -39,11 +39,11 @@ go_library( "@bazel_gazelle//rule:go_default_library", "@com_github_bazelbuild_buildtools//build:go_default_library", "@com_github_bmatcuk_doublestar_v4//:doublestar", + "@com_github_dougthor42_go_tree_sitter//:go-tree-sitter", + "@com_github_dougthor42_go_tree_sitter//python", "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_emirpasic_gods//sets/treeset", "@com_github_emirpasic_gods//utils", - "@com_github_smacker_go_tree_sitter//:go-tree-sitter", - "@com_github_smacker_go_tree_sitter//python", "@org_golang_x_sync//errgroup", ], ) diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index a1f47f400c..c147984fc3 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -22,8 +22,8 @@ import ( "path/filepath" "strings" - sitter "github.com/smacker/go-tree-sitter" - "github.com/smacker/go-tree-sitter/python" + sitter "github.com/dougthor42/go-tree-sitter" + "github.com/dougthor42/go-tree-sitter/python" ) const ( @@ -115,10 +115,10 @@ func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool { a, b = b, a } if a.Type() == sitterNodeTypeIdentifier && a.Content(p.code) == "__name__" && - // at github.com/smacker/go-tree-sitter@latest (after v0.0.0-20240422154435-0628b34cbf9c we used) + // at github.com/dougthor42/go-tree-sitter@latest (after v0.0.0-20240422154435-0628b34cbf9c we used) // "__main__" is the second child of b. But now, it isn't. // we cannot use the latest go-tree-sitter because of the top level reference in scanner.c. - // https://github.com/smacker/go-tree-sitter/blob/04d6b33fe138a98075210f5b770482ded024dc0f/python/scanner.c#L1 + // https://github.com/dougthor42/go-tree-sitter/blob/04d6b33fe138a98075210f5b770482ded024dc0f/python/scanner.c#L1 b.Type() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" { return true } diff --git a/gazelle/python/testdata/py312_syntax/BUILD.in b/gazelle/python/testdata/py312_syntax/BUILD.in new file mode 100644 index 0000000000..af2c2cea4b --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/py312_syntax/BUILD.out b/gazelle/python/testdata/py312_syntax/BUILD.out new file mode 100644 index 0000000000..7457f335a7 --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/BUILD.out @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +# gazelle:python_generation_mode file + +py_library( + name = "_other_module", + srcs = ["_other_module.py"], + visibility = ["//:__subpackages__"], +) + +py_binary( + name = "pep_695_type_parameter", + srcs = ["pep_695_type_parameter.py"], + visibility = ["//:__subpackages__"], + deps = [":_other_module"], +) diff --git a/gazelle/python/testdata/py312_syntax/README.md b/gazelle/python/testdata/py312_syntax/README.md new file mode 100644 index 0000000000..854a0a3aa6 --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/README.md @@ -0,0 +1,4 @@ +# py312 syntax + +This test case checks that we properly parse certain python 3.12 syntax, such +as pep 695 type parameters, with go-tree-sitter. diff --git a/gazelle/python/testdata/py312_syntax/WORKSPACE b/gazelle/python/testdata/py312_syntax/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/py312_syntax/__init__.py b/gazelle/python/testdata/py312_syntax/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/py312_syntax/_other_module.py b/gazelle/python/testdata/py312_syntax/_other_module.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/py312_syntax/pep_695_type_parameter.py b/gazelle/python/testdata/py312_syntax/pep_695_type_parameter.py new file mode 100644 index 0000000000..eff06de5a7 --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/pep_695_type_parameter.py @@ -0,0 +1,22 @@ +def search_one_more_level[T]( + graph: dict[T, set[T]], seen: set[T], routes: list[list[T]], target: T +) -> list[T] | None: + """This function fails to parse with older versions of go-tree-sitter. + + Args: + graph: The graph to search as input. + seen: The nodes that have been visited as input/output. + routes: The current routes in the breadth-first search as input/output. + target: The target to search in this extra search level. + + Returns: + a route if it ends on the target, or None if no route reaches the + target. + """ + + +import _other_module + + +if __name__ == "__main__": + pass diff --git a/gazelle/python/testdata/py312_syntax/test.yaml b/gazelle/python/testdata/py312_syntax/test.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/gazelle/python/testdata/py312_syntax/test.yaml @@ -0,0 +1 @@ +---