From 8567545183ff48500eb974ed76ac7fa36a5d7303 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 16 Oct 2024 13:35:58 +0500 Subject: [PATCH] Drop Python 3.8 and PyPy 3.9, add more CI tools, fix building docs (#94) --- .bandit.yml | 3 +++ .github/workflows/main.yml | 18 ++++++++--------- .github/workflows/publish.yml | 2 +- .pre-commit-config.yaml | 9 +++++++-- docs/conf.py | 9 --------- itemloaders/processors.py | 1 + pylintrc | 36 ++++++++++++++++++++++++++++++++++ setup.py | 5 ++--- tests/test_base_loader.py | 6 ++++-- tests/test_output_processor.py | 2 +- tests/test_select_jmes.py | 4 ++-- tox.ini | 9 ++++++++- 12 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 .bandit.yml create mode 100644 pylintrc diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 0000000..d85740d --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,3 @@ +skips: +- B101 +exclude_dirs: ['tests'] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b5e928..2f941bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: pre-commit/action@v3.0.1 tests: @@ -21,18 +21,18 @@ jobs: - python-version: '3.12' # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: '3.12' + - python-version: '3.13' + env: + TOXENV: pylint + - python-version: '3.13' env: TOXENV: twinecheck - - python-version: 3.8 + - python-version: '3.13' env: - TOXENV: py + TOXENV: typing - python-version: 3.9 env: TOXENV: py - - python-version: pypy-3.9 - env: - TOXENV: py - python-version: pypy-3.10 env: TOXENV: py @@ -48,13 +48,13 @@ jobs: - python-version: '3.12' env: TOXENV: py - - python-version: '3.13.0-rc.1' + - python-version: '3.13' env: TOXENV: py steps: - uses: actions/checkout@v4 - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.python-version, 'beta') + if: contains(matrix.python-version, 'pypy') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 19ae675..3c5796d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.13 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0f0ff5..52d1de5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,16 @@ default_language_version: - python: python3.12 + python: python3.13 repos: + - hooks: + - id: bandit + args: [-r, -c, .bandit.yml] + repo: https://github.com/PyCQA/bandit + rev: 1.7.10 - hooks: - id: black language_version: python3 repo: https://github.com/ambv/black - rev: 24.8.0 + rev: 24.10.0 - hooks: - id: isort language_version: python3 diff --git a/docs/conf.py b/docs/conf.py index 379df63..933ee8f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,6 @@ import sys from os import path -import sphinx_rtd_theme - # If your extensions are in another directory, add it here. If the directory # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. @@ -108,13 +106,6 @@ # documentation. # html_theme_options = {} -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. diff --git a/itemloaders/processors.py b/itemloaders/processors.py index 10387b5..2376f9b 100644 --- a/itemloaders/processors.py +++ b/itemloaders/processors.py @@ -159,6 +159,7 @@ def __call__(self, values: Any) -> Any: for value in values: if value is not None and value != "": return value + return None class Identity: diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..f303452 --- /dev/null +++ b/pylintrc @@ -0,0 +1,36 @@ +[MASTER] +ignore=typing +persistent=no + +[MESSAGES CONTROL] +disable=broad-exception-caught, + c-extension-no-member, + consider-using-f-string, + disallowed-name, + duplicate-code, + fixme, + import-error, + import-outside-toplevel, + invalid-name, + line-too-long, + missing-class-docstring, + missing-function-docstring, + missing-module-docstring, + no-else-return, + no-member, + not-callable, + parse-error, + protected-access, + raise-missing-from, + redefined-builtin, + redefined-outer-name, + too-few-public-methods, + too-many-arguments, + too-many-lines, + too-many-positional-arguments, + too-many-public-methods, + unidiomatic-typecheck, + unused-argument, + use-a-generator, + wrong-import-order, + wrong-import-position, diff --git a/setup.py b/setup.py index 9707ac8..dae156d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -with open("README.rst") as f: +with open("README.rst", encoding="utf-8") as f: long_description = f.read() setup( @@ -30,7 +30,6 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -39,7 +38,7 @@ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=[ # before updating these versions, be sure they are not higher than # scrapy's requirements diff --git a/tests/test_base_loader.py b/tests/test_base_loader.py index e7919f4..f465f9a 100644 --- a/tests/test_base_loader.py +++ b/tests/test_base_loader.py @@ -41,6 +41,7 @@ def validate_sku(value): # Let's assume a SKU is only digits. if value.isdigit(): return value + return None class MyLoader(ItemLoader): name_out = Compose(lambda vs: vs[0]) # take first which allows empty values @@ -278,10 +279,11 @@ class CustomItemLoader(ItemLoader): il = CustomItemLoader() il.add_value("name", ["$10"]) + expected_exc = None try: float("$10") except Exception as e: - expected_exc_str = str(e) + expected_exc = e exc = None try: @@ -293,7 +295,7 @@ class CustomItemLoader(ItemLoader): assert "name" in s, s assert "$10" in s, s assert "ValueError" in s, s - assert expected_exc_str in s, s + assert str(expected_exc) in s, s def test_output_processor_using_classes(self): il = CustomItemLoader() diff --git a/tests/test_output_processor.py b/tests/test_output_processor.py index 09ef95d..d8c8e0f 100644 --- a/tests/test_output_processor.py +++ b/tests/test_output_processor.py @@ -9,7 +9,7 @@ class TestOutputProcessorDict(unittest.TestCase): def test_output_processor(self): class TempDict(Dict[str, Any]): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) self.setdefault("temp", 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_select_jmes.py b/tests/test_select_jmes.py index 1754863..d9e5f3b 100644 --- a/tests/test_select_jmes.py +++ b/tests/test_select_jmes.py @@ -18,8 +18,8 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for key in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[key] + for key, value in self.test_list_equals.items(): + expr, test_list, expected = value test = SelectJmes(expr)(test_list) self.assertEqual( test, expected, msg=f"test {key!r} got {test} expected {expected}" diff --git a/tox.ini b/tox.ini index 09de7a1..7fc79ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,py312 +envlist = py39,py310,py311,py312,py313 [testenv] deps = @@ -61,3 +61,10 @@ deps = pre-commit commands = pre-commit run {posargs:--all-files} + +[testenv:pylint] +deps = + {[testenv]deps} + pylint==3.3.1 +commands = + pylint docs itemloaders tests setup.py