diff --git a/README.rst b/README.rst index 26bcdbc7..be657342 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,13 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize our python repositories using modern python -packaging and configuration techniques. Using this `blog post`_ as inspiration, this -repository will serve as the base for all new python projects and will be adopted to all -our existing ones as well. +This repo attempts to standardize the structure of the Python-based project's +repositories using modern Python packaging and configuration techniques. +Using this `blog post`_ as inspiration, this repository serves as the base for +all new Python projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ + Usage ===== @@ -36,9 +37,16 @@ This is also the workflow to use when updating the skeleton files in any given r More usage instructions can be found in ``docs/skeleton-usage.rst``. + Release Notes ============= +- 2022-03-04: + - Synchronize configure and configure.bat scripts for sanity + - Update CI operating system support with latest Azure OS images + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies + There are now fewer scripts. See etc/scripts/README.rst for details + - 2021-09-03: - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3fd2c39..089abe9b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -31,6 +31,14 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython + image_name: macos-11 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython diff --git a/configure b/configure index fdfdc855..b2d64c41 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ set -e # Source this script for initial configuration # Use configure --help for details # +# NOTE: please keep in sync with Windows script configure.bat +# # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default ################################ @@ -32,10 +34,8 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai # where we create a virtualenv VIRTUALENV_DIR=venv -# Cleanable files and directories with the --clean option -CLEANABLE=" - build - venv" +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -50,11 +50,14 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + +################################ +# Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ @@ -65,56 +68,50 @@ fi ################################ -# find a proper Python to run -# Use environment variables or a file if available. -# Otherwise the latest Python by default. -if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE - if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then - PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") - else - PYTHON_EXECUTABLE=python3 - fi -fi +# Main command line entry point +main() { + CFG_REQUIREMENTS=$REQUIREMENTS + NO_INDEX="--no-index" + + # We are using getopts to parse option arguments that start with "-" + while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + init ) NO_INDEX="";; + esac;; + esac + done + PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" -################################ -cli_help() { - echo An initial configuration script - echo " usage: ./configure [options]" - echo - echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. - echo - echo The options are: - echo " --clean: clean built and installed files and exit." - echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." - echo " --help: display this help message and exit." - echo - echo By default, the python interpreter version found in the path is used. - echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to - echo configure another Python executable interpreter to use. If this is not - echo set, a file named PYTHON_EXECUTABLE containing a single line with the - echo path of the Python executable to use will be checked last. - set +e - exit + find_python + create_virtualenv "$VIRTUALENV_DIR" + install_packages "$CFG_REQUIREMENTS" + . "$CFG_BIN_DIR/activate" } -clean() { - # Remove cleanable file and directories and files from the root dir. - echo "* Cleaning ..." - for cln in $CLEANABLE; - do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; - done - set +e - exit +################################ +# Find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi + fi } +################################ create_virtualenv() { # create a virtualenv for Python # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -145,6 +142,7 @@ create_virtualenv() { } +################################ install_packages() { # install requirements in virtualenv # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -162,28 +160,43 @@ install_packages() { ################################ -# Main command line entry point -CFG_DEV_MODE=0 -CFG_REQUIREMENTS=$REQUIREMENTS -NO_INDEX="--no-index" - -# We are using getopts to parse option arguments that start with "-" -while getopts :-: optchar; do - case "${optchar}" in - -) - case "${OPTARG}" in - help ) cli_help;; - clean ) clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - init ) NO_INDEX="";; - esac;; - esac -done - -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - -create_virtualenv "$VIRTUALENV_DIR" -install_packages "$CFG_REQUIREMENTS" -. "$CFG_BIN_DIR/activate" +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo Use the --init option if starting a new project and the project + echo dependencies are not available on thirdparty.aboutcode.org/pypi/ + echo and requirements.txt and/or requirements-dev.txt has not been generated. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --init: pull dependencies from PyPI. Used when first setting up a project." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} + + +main set +e diff --git a/configure.bat b/configure.bat index ed061613..2ae4727f 100644 --- a/configure.bat +++ b/configure.bat @@ -14,6 +14,8 @@ @rem # Source this script for initial configuration @rem # Use configure --help for details +@rem # NOTE: please keep in sync with POSIX script configure + @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ @@ -49,10 +51,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -64,7 +67,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point -set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" @@ -74,7 +76,6 @@ if not "%1" == "" ( if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 ) if "%1" EQU "--init" ( set "NO_INDEX= " @@ -87,7 +88,7 @@ set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" @rem ################################ -@rem # find a proper Python to run +@rem # Find a proper Python to run @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @@ -99,6 +100,8 @@ if not defined PYTHON_EXECUTABLE ( ) ) + +@rem ################################ :create_virtualenv @rem # create a virtualenv for Python @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -143,6 +146,7 @@ if %ERRORLEVEL% neq 0 ( ) +@rem ################################ :install_packages @rem # install requirements in virtualenv @rem # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -157,6 +161,9 @@ if %ERRORLEVEL% neq 0 ( %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% + +@rem ################################ +:create_bin_junction @rem # Create junction to bin to have the same directory between linux and windows if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" @@ -171,7 +178,6 @@ exit /b 0 @rem ################################ - :cli_help echo An initial configuration script echo " usage: configure [options]" @@ -195,6 +201,7 @@ exit /b 0 exit /b 0 +@rem ################################ :clean @rem # Remove cleanable file and directories and files from the root dir. echo "* Cleaning ..." @@ -202,4 +209,4 @@ for %%F in (%CLEANABLE%) do ( rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) -exit /b 0 \ No newline at end of file +exit /b 0 diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index d8b00f98..edf82e44 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -15,10 +15,10 @@ Pre-requisites * To generate or update pip requirement files, you need to start with a clean virtualenv as instructed below (This is to avoid injecting requirements - specific to the tools here in the main requirements). + specific to the tools used here in the main requirements). * For other usages, the tools here can run either in their own isolated - virtualenv best or in the the main configured development virtualenv. + virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: pip install --requirement etc/release/requirements.txt @@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files Scripts ~~~~~~~ -* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and - NOTICE files to populate a local a thirdparty directory strictly from our - remote repo and using only pinned packages listed in one or more pip - requirements file(s). Fetch only requirements for specific python versions and - operating systems. Optionally fetch the corresponding source distributions. - -* **publish_files.py** will upload/sync a thirdparty directory of files to our - remote repo. Requires a GitHub personal access token. - -* **build_wheels.py** will build a package binary wheel for multiple OS and - python versions. Optionally wheels that contain native code are built - remotely. Dependent wheels are optionally included. Requires Azure credentials - and tokens if building wheels remotely on multiple operatin systems. - -* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to - add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE - and .LICENSE files. Requires Azure credentials and tokens if requesting the - build of missing wheels remotely on multiple operatin systems. +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. * **check_thirdparty.py** will check a thirdparty directory for errors. -* **bootstrap.py** will bootstrap a thirdparty directory from a requirements - file(s) to add or build missing wheels, sources archives and create .ABOUT, - .NOTICE and .LICENSE files. Requires Azure credentials and tokens if - requesting the build of missing wheels remotely on multiple operatin systems. - - - -Usage -~~~~~ - -See each command line --help option for details. - -* (TODO) **add_package.py** will add or update a Python package including wheels, - sources and ABOUT files and this for multiple Python version and OSes(for use - with upload_packages.py afterwards) You will need an Azure personal access - token for buidling binaries and an optional DejaCode API key to post and fetch - new package versions there. TODO: explain how we use romp - Upgrade virtualenv app ---------------------- diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py deleted file mode 100644 index 31f2f553..00000000 --- a/etc/scripts/bootstrap.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import itertools - -import click - -import utils_thirdparty -from utils_thirdparty import Environment -from utils_thirdparty import PypiPackage - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file(s) to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built and " - "sources, ABOUT and LICENSE files fetched.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version(s) to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS(ses) to use for this build: one of linux, mac or windows.", -) -@click.option( - "-l", - "--latest-version", - is_flag=True, - help="Get the latest version of all packages, ignoring version specifiers.", -) -@click.option( - "--sync-dejacode", - is_flag=True, - help="Synchronize packages with DejaCode.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.help_option("-h", "--help") -def bootstrap( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_deps, - latest_version, - sync_dejacode, - build_remotely=False, -): - """ - Boostrap a thirdparty Python packages directory from pip requirements. - - Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for - the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible - with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` - OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and - .LICENSE files. - - Optionally ignore version specifiers and use the ``--latest-version`` - of everything. - - Sources and wheels are fetched with attempts first from PyPI, then our remote repository. - If missing wheels are built as needed. - """ - # rename variables for clarity since these are lists - requirements_files = requirements_file - python_versions = python_version - operating_systems = operating_system - - # create the environments we need - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - - # collect all packages to process from requirements files - # this will fail with an exception if there are packages we cannot find - - required_name_versions = set() - - for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) - required_name_versions.update(nvs) - if latest_version: - required_name_versions = set((name, None) for name, _ver in required_name_versions) - - print( - f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" - ) - - # fetch all available wheels, keep track of missing - # start with local, then remote, then PyPI - - print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") - # list of all the wheel filenames either pre-existing, fetched or built - # updated as we progress - available_wheel_filenames = [] - - local_packages_by_namever = { - (p.name, p.version): p - for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) - } - - # list of (name, version, environment) not local and to fetch - name_version_envt_to_fetch = [] - - # start with a local check - for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get( - ( - name, - version, - ) - ) - if local_pack: - supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) - if supported_wheels: - available_wheel_filenames.extend(w.filename for w in supported_wheels) - print( - f"====> No fetch or build needed. " - f"Local wheel already available for {name}=={version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - continue - - name_version_envt_to_fetch.append( - ( - name, - version, - envt, - ) - ) - - print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") - - # list of (name, version, environment) not fetch and to build - name_version_envt_to_build = [] - - # then check if the wheel can be fetched without building from remote and Pypi - for name, version, envt in name_version_envt_to_fetch: - - fetched_fwn = utils_thirdparty.fetch_package_wheel( - name=name, - version=version, - environment=envt, - dest_dir=thirdparty_dir, - ) - - if fetched_fwn: - available_wheel_filenames.append(fetched_fwn) - else: - name_version_envt_to_build.append( - ( - name, - version, - envt, - ) - ) - - # At this stage we have all the wheels we could obtain without building - for name, version, envt in name_version_envt_to_build: - print( - f"====> Need to build wheels for {name}=={version} on os: " - f"{envt.operating_system} for Python: {envt.python_version}" - ) - - packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build - ] - - print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") - - package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( - packages_and_envts=packages_and_envts_to_build, - build_remotely=build_remotely, - with_deps=with_deps, - dest_dir=thirdparty_dir, - ) - if wheel_filenames_built: - available_wheel_filenames.extend(available_wheel_filenames) - - for pack, envt in package_envts_not_built: - print( - f"====> FAILED to build any wheel for {pack.name}=={pack.version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - - print(f"==> FETCHING SOURCE DISTRIBUTIONS") - # fetch all sources, keep track of missing - # This is a list of (name, version) - utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - print(f"==> FETCHING ABOUT AND LICENSE FILES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - ############################################################################ - if sync_dejacode: - print(f"==> SYNC WITH DEJACODE") - # try to fetch from DejaCode any missing ABOUT - # create all missing DejaCode packages - pass - - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py deleted file mode 100644 index 8a28176e..00000000 --- a/etc/scripts/build_wheels.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-n", - "--name", - type=str, - metavar="PACKAGE_NAME", - required=True, - help="Python package name to add or build.", -) -@click.option( - "-v", - "--version", - type=str, - default=None, - metavar="VERSION", - help="Python package version to add or build.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def build_wheels( - name, - version, - thirdparty_dir, - python_version, - operating_system, - with_deps, - build_remotely, - remote_build_log_file, - verbose, -): - """ - Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and - optional VERSION. Build wheels compatible with all the `--python-version` - PYVER(s) and `--operating_system` OS(s). - - Build native wheels remotely if needed when `--build-remotely` and include - all dependencies with `--with-deps`. - """ - utils_thirdparty.add_or_upgrade_built_wheels( - name=name, - version=version, - python_versions=python_version, - operating_systems=operating_system, - dest_dir=thirdparty_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - -if __name__ == "__main__": - build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 4fea16c5..0f04b349 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,17 +16,39 @@ @click.command() @click.option( "-d", - "--thirdparty-dir", + "--dest_dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def check_thirdparty_dir( + dest_dir, + wheels, + sdists, +): """ - Check a thirdparty directory for problems. + Check a thirdparty directory for problems and print these on screen. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) if __name__ == "__main__": diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py deleted file mode 100644 index a78861e9..00000000 --- a/etc/scripts/fetch_built_wheels.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "--remote-build-log-file", - type=click.Path(readable=True), - metavar="LOG-FILE", - help="Path to a remote builds log file.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory to save built wheels.", -) -@click.option( - "--no-wait", - is_flag=True, - default=False, - help="Do not wait for build completion.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def fetch_remote_wheels( - remote_build_log_file, - thirdparty_dir, - no_wait, - verbose, -): - """ - Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines - build log file. - """ - utils_thirdparty.fetch_remotely_built_wheels( - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - no_wait=no_wait, - verbose=verbose, - ) - - -if __name__ == "__main__": - fetch_remote_wheels() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py deleted file mode 100644 index 9da9ce96..00000000 --- a/etc/scripts/fetch_requirements.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools - -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="INT", - multiple=True, - default=["36"], - show_default=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - multiple=True, - default=["linux"], - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "-s", - "--with-sources", - is_flag=True, - help="Fetch the corresponding source distributions.", -) -@click.option( - "-a", - "--with-about", - is_flag=True, - help="Fetch the corresponding ABOUT and LICENSE files.", -) -@click.option( - "--allow-unpinned", - is_flag=True, - help="Allow requirements without pinned versions.", -) -@click.option( - "-s", - "--only-sources", - is_flag=True, - help="Fetch only the corresponding source distributions.", -) -@click.option( - "-u", - "--remote-links-url", - type=str, - metavar="URL", - default=utils_thirdparty.REMOTE_LINKS_URL, - show_default=True, - help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", -) -@click.help_option("-h", "--help") -def fetch_requirements( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_sources, - with_about, - allow_unpinned, - only_sources, - remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, -): - """ - Fetch and save to THIRDPARTY_DIR all the required wheels for pinned - dependencies found in the `--requirement` FILE requirements file(s). Only - fetch wheels compatible with the provided `--python-version` and - `--operating-system`. - Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together - with a virtualenv.pyz app. - - Use exclusively wheel not from PyPI but rather found in the PyPI-like link - repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` - as a local directory path to a wheels directory if this is not a a URL. - """ - - # fetch wheels - python_versions = python_version - operating_systems = operating_system - requirements_files = requirements_file - - if not only_sources: - envs = itertools.product(python_versions, operating_systems) - envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) - - for env, reqf in itertools.product(envs, requirements_files): - - for package, error in utils_thirdparty.fetch_wheels( - environment=env, - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch wheel:", package, ":", error) - - # optionally fetch sources - if with_sources or only_sources: - - for reqf in requirements_files: - for package, error in utils_thirdparty.fetch_sources( - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch source:", package, ":", error) - - if with_about: - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - utils_thirdparty.find_problems( - dest_dir=thirdparty_dir, - report_missing_sources=with_sources or only_sources, - report_missing_wheels=not only_sources, - ) - - -if __name__ == "__main__": - fetch_requirements() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 00000000..22147b20 --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import os +import sys + +import click + +import utils_thirdparty +import utils_requirements + +TRACE = True + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEXES, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, +): + """ + Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` PYVER(s) + and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + + Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + """ + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if not os.listdir(dest_dir) and not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + if latest_version: + latest_name_versions = set() + names = set(name for name, _version in sorted(required_name_versions)) + for name in sorted(names): + latests = utils_thirdparty.PypiPackage.sorted( + utils_thirdparty.get_package_versions( + name=name, version=None, index_urls=index_urls + ) + ) + if not latests: + print(f"No distribution found for: {name}") + continue + latest = latests[-1] + latest_name_versions.add((latest.name, latest.version)) + required_name_versions = latest_name_versions + + if TRACE: + print("required_name_versions:", required_name_versions) + + if wheels: + # create the environments matrix we need for wheels + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + wheels_not_found = {} + sdists_not_found = {} + # iterate over requirements, one at a time + for name, version in sorted(required_name_versions): + nv = name, version + existing_package = existing_packages_by_nv.get(nv) + if wheels: + for environment in environments: + if existing_package: + existing_wheels = list( + existing_package.get_supported_wheels(environment=environment) + ) + else: + existing_wheels = None + + if existing_wheels: + if TRACE: + print( + f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" + ) + if all(w.is_pure() for w in existing_wheels): + break + else: + continue + + if TRACE: + print(f"Fetching wheel for: {name}=={version} on: {environment}") + + try: + ( + fetched_wheel_filenames, + existing_wheel_filenames, + ) = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + index_urls=index_urls, + ) + if TRACE: + if existing_wheel_filenames: + print( + f" ====> Wheels already available: {name}=={version} on: {environment}" + ) + for whl in existing_wheel_filenames: + print(f" {whl}") + if fetched_wheel_filenames: + print(f" ====> Wheels fetched: {name}=={version} on: {environment}") + for whl in fetched_wheel_filenames: + print(f" {whl}") + + fwfns = fetched_wheel_filenames + existing_wheel_filenames + + if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): + break + + except utils_thirdparty.DistributionNotFound as e: + wheels_not_found[f"{name}=={version}"] = str(e) + + if sdists: + if existing_package and existing_package.sdist: + if TRACE: + print( + f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" + ) + continue + + if TRACE: + print(f" Fetching sdist for: {name}=={version}") + + try: + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + index_urls=index_urls, + ) + + if TRACE: + if not fetched: + print( + f" ====> Sdist already available: {name}=={version} on: {environment}" + ) + else: + print( + f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + ) + + except utils_thirdparty.DistributionNotFound as e: + sdists_not_found[f"{name}=={version}"] = str(e) + + if wheels and wheels_not_found: + print(f"==> MISSING WHEELS") + for wh in wheels_not_found: + print(f" {wh}") + + if sdists and sdists_not_found: + print(f"==> MISSING SDISTS") + for sd in sdists_not_found: + print(f" {sd}") + + print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py deleted file mode 100644 index d664c9c4..00000000 --- a/etc/scripts/fix_thirdparty.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to fix.", -) -@click.option( - "--build-wheels", - is_flag=True, - help="Build all missing wheels .", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--strip-classifiers", - is_flag=True, - help="Remove danglingf classifiers", -) -@click.help_option("-h", "--help") -def fix_thirdparty_dir( - thirdparty_dir, - build_wheels, - build_remotely, - remote_build_log_file, - strip_classifiers, -): - """ - Fix a thirdparty directory of dependent package wheels and sdist. - - Multiple fixes are applied: - - fetch or build missing binary wheels - - fetch missing source distributions - - derive, fetch or add missing ABOUT files - - fetch missing .LICENSE and .NOTICE files - - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files - - Optionally build missing binary wheels for all supported OS and Python - version combos locally or remotely. - """ - if strip_classifiers: - print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - else: - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" - ) - - print("***FIND PROBLEMS***") - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 53db9b0a..8de2b960 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -5,65 +5,25 @@ # Copyright (c) 2010 David Wolever . All rights reserved. # originally from https://github.com/wolever/pip2pi +import hashlib import os import re import shutil - +from collections import defaultdict from html import escape from pathlib import Path +from typing import NamedTuple """ -name: pip compatibility tags -version: 20.3.1 -download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py -copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) -license_expression: mit -notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py - -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Generate a PyPI simple index froma directory. """ -get_wheel_from_filename = re.compile( - r"""^(?P(?P.+?)-(?P.*?)) - ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl)$""", - re.VERBOSE, -).match - -sdist_exts = ( - ".tar.gz", - ".tar.bz2", - ".zip", - ".tar.xz", -) -wheel_ext = ".whl" -app_ext = ".pyz" -dist_exts = sdist_exts + (wheel_ext, app_ext) class InvalidDistributionFilename(Exception): pass -def get_package_name_from_filename(filename, normalize=True): +def get_package_name_from_filename(filename): """ Return the package name extracted from a package ``filename``. Optionally ``normalize`` the name according to distribution name rules. @@ -132,18 +92,99 @@ def get_package_name_from_filename(filename, normalize=True): if not name: raise InvalidDistributionFilename(filename) - if normalize: - name = name.lower().replace("_", "-") + name = normalize_name(name) return name -def build_pypi_index(directory, write_index=False): +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = f""" + + + Links for all packages + + """ + document.append(header) + + for _name, packages in packages_by_package_name.items(): + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI simple - directory index at ``directory``/simple/ populated with the proper PyPI simple - index directory structure crafted using symlinks. + Using a ``directory`` directory of wheels and sdists, create the a PyPI + simple directory index at ``directory``/simple/ populated with the proper + PyPI simple index directory structure crafted using symlinks. WARNING: The ``directory``/simple/ directory is removed if it exists. + NOTE: in addition to the a PyPI simple index.html there is also a links.html + index file generated which is suitable to use with pip's --find-links """ directory = Path(directory) @@ -153,14 +194,15 @@ def build_pypi_index(directory, write_index=False): shutil.rmtree(str(index_dir), ignore_errors=True) index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) - if write_index: - simple_html_index = [ - "PyPI Simple Index", - "", - ] + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '' '', + ] - package_names = set() for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name @@ -172,23 +214,99 @@ def build_pypi_index(directory, write_index=False): ): continue - pkg_name = get_package_name_from_filename(pkg_filename) + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) pkg_index_dir = index_dir / pkg_name pkg_index_dir.mkdir(parents=True, exist_ok=True) pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename pkg_indexed_file.symlink_to(link_target) - if write_index and pkg_name not in package_names: + if pkg_name not in packages_by_package_name: esc_name = escape(pkg_name) simple_html_index.append(f'{esc_name}
') - package_names.add(pkg_name) - if write_index: - simple_html_index.append("") - index_html = index_dir / "index.html" - index_html.write_text("\n".join(simple_html_index)) + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) if __name__ == "__main__": import sys diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py deleted file mode 100644 index 86693631..00000000 --- a/etc/scripts/publish_files.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import hashlib -import os -import sys - -from pathlib import Path - -import click -import requests -import utils_thirdparty - -from github_release_retry import github_release_retry as grr - -""" -Create GitHub releases and upload files there. -""" - - -def get_files(location): - """ - Return an iterable of (filename, Path, md5) tuples for files in the `location` - directory tree recursively. - """ - for top, _dirs, files in os.walk(location): - for filename in files: - pth = Path(os.path.join(top, filename)) - with open(pth, "rb") as fi: - md5 = hashlib.md5(fi.read()).hexdigest() - yield filename, pth, md5 - - -def get_etag_md5(url): - """ - Return the cleaned etag of URL `url` or None. - """ - headers = utils_thirdparty.get_remote_headers(url) - headers = {k.lower(): v for k, v in headers.items()} - etag = headers.get("etag") - if etag: - etag = etag.strip('"').lower() - return etag - - -def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, -): - """ - Create or update a GitHub release at https://github.com// for - `tag_name` tag using the optional `description` for this release. - Use the provided `token` as a GitHub token for API calls authentication. - Upload all files found in the `directory` tree to that GitHub release. - Retry API calls up to `retry_limit` time to work around instability the - GitHub API. - - Remote files that are not the same as the local files are deleted and re- - uploaded. - """ - release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" - - # scrape release page HTML for links - urls_by_filename = { - os.path.basename(l): l - for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) - } - - # compute what is new, modified or unchanged - print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") - - new_to_upload = [] - unchanged_to_skip = [] - modified_to_delete_and_reupload = [] - for filename, pth, md5 in get_files(directory): - url = urls_by_filename.get(filename) - if not url: - print(f"{filename} content is NEW, will upload") - new_to_upload.append(pth) - continue - - out_of_date = get_etag_md5(url) != md5 - if out_of_date: - print(f"{url} content is CHANGED based on md5 etag, will re-upload") - modified_to_delete_and_reupload.append(pth) - else: - # print(f'{url} content is IDENTICAL, skipping upload based on Etag') - unchanged_to_skip.append(pth) - print(".") - - ghapi = grr.GithubApi( - github_api_url="https://api.github.com", - user=user, - repo=repo, - token=token, - retry_limit=retry_limit, - ) - - # yank modified - print( - f"Unpublishing {len(modified_to_delete_and_reupload)} published but " - f"locally modified files in {release_homepage_url}" - ) - - release = ghapi.get_release_by_tag(tag_name) - - for pth in modified_to_delete_and_reupload: - filename = os.path.basename(pth) - asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print(f" Unpublishing file: {filename}).") - response = ghapi.delete_asset(asset_id) - if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f"failed asset deletion: {response}") - - # finally upload new and modified - to_upload = new_to_upload + modified_to_delete_and_reupload - print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") - release = grr.Release(tag_name=tag_name, body=description) - grr.make_release(ghapi, release, to_upload) - - -TOKEN_HELP = ( - "The Github personal acess token is used to authenticate API calls. " - "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " - "See for details: https://github.com/settings/tokens and " - "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" -) - - -@click.command() -@click.option( - "--user-repo-tag", - help="The GitHub qualified repository user/name/tag in which " - "to create the release such as in nexB/thirdparty/pypi", - type=str, - required=True, -) -@click.option( - "-d", - "--directory", - help="The directory that contains files to upload to the release.", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, -) -@click.option( - "--token", - help=TOKEN_HELP, - default=os.environ.get("GITHUB_TOKEN", None), - type=str, - required=False, -) -@click.option( - "--description", - help="Text description for the release. Ignored if the release exists.", - default=None, - type=str, - required=False, -) -@click.option( - "--retry_limit", - help="Number of retries when making failing GitHub API calls. " - "Retrying helps work around transient failures of the GitHub API.", - type=int, - default=10, -) -@click.help_option("-h", "--help") -def publish_files( - user_repo_tag, - directory, - retry_limit=10, - token=None, - description=None, -): - """ - Publish all the files in DIRECTORY as assets to a GitHub release. - Either create or update/replace remote files' - """ - if not token: - click.secho("--token required option is missing.") - click.secho(TOKEN_HELP) - sys.exit(1) - - user, repo, tag_name = user_repo_tag.split("/") - - create_or_update_release_and_upload_directory( - user=user, - repo=repo, - tag_name=tag_name, - description=description, - retry_limit=retry_limit, - token=token, - directory=directory, - ) - - -if __name__ == "__main__": - publish_files() diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7753ea02..fbc456db 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -14,95 +14,63 @@ """ Utilities to manage requirements files and call pip. NOTE: this should use ONLY the standard library and not import anything else -becasue this is used for boostrapping. +because this is used for boostrapping with no requirements installed. """ -def load_requirements(requirements_file="requirements.txt", force_pinned=True): +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): """ Yield package (name, version) tuples for each requirement in a `requirement` - file. Every requirement versions must be pinned if `force_pinned` is True. - Otherwise un-pinned requirements are returned with a None version + file. Only accept requirements pinned to an exact version. """ with open(requirements_file) as reqs: req_lines = reqs.read().splitlines(False) - return get_required_name_versions(req_lines, force_pinned) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) -def get_required_name_versions( - requirement_lines, - force_pinned=True, -): +def get_required_name_versions(requirement_lines, with_unpinned=False): """ Yield required (name, version) tuples given a`requirement_lines` iterable of - requirement text lines. Every requirement versions must be pinned if - `force_pinned` is True. Otherwise un-pinned requirements are returned with a - None version. - + requirement text lines. Only accept requirements pinned to an exact version. """ + for req_line in requirement_lines: req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if force_pinned: - if "==" not in req_line: - raise Exception(f"Requirement version is not pinned: {req_line}") - name = req_line - version = None - else: - if req_line.startswith("-"): - print(f"Requirement skipped, is not supported: {req_line}") - - if "==" in req_line: - name, _, version = req_line.partition("==") - version = version.lower().strip() - else: - # FIXME: we do not support unpinned requirements yet! - name = strip_reqs(req_line) - version = None - - name = name.lower().strip() - yield name, version - - -def strip_reqs(line): - """ - Return a name given a pip reuirement text ``line` striping version and - requirements. - - For example:: - - >>> s = strip_reqs("foo <=12, >=13,!=12.6") - >>> assert s == "foo" - """ - if "--" in line: - raise Exception(f"Unsupported requirement style: {line}") - - line = line.strip() - - ops = ">>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> try: + ... assert not get_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) """ - requires = [c for c in requires.splitlines(False) if c] - if not requires: - return [] - - requires = ["".join(r.split()) for r in requires if r and r.strip()] - return sorted(requires) + requirement = requirement and "".join(requirement.lower().split()) + assert requirement, f"specifier is required is empty:{requirement!r}" + name, operator, version = split_req(requirement) + assert name, f"Name is required: {requirement}" + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + return name, version def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): @@ -139,8 +107,47 @@ def lock_dev_requirements( def get_installed_reqs(site_packages_dir): """ - Return the installed pip requirements as text found in `site_packages_dir` as a text. + Return the installed pip requirements as text found in `site_packages_dir` + as a text. """ - # Also include these packages in the output with --all: wheel, distribute, setuptools, pip + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + assert req + # do not allow multiple constraints and tags + assert not any(c in req for c in ",;") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index a2fbe4e5..829cf8c0 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -11,12 +11,10 @@ from collections import defaultdict import email import itertools -import operator import os import re import shutil import subprocess -import tarfile import tempfile import time import urllib @@ -26,29 +24,30 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags -import utils_pypi_supported_tags - from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version +from urllib.parse import quote_plus + +import utils_pip_compatibility_tags from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. -- update pip requirement files from installed packages for prod. and dev. -- build and save wheels for all required packages -- also build variants for wheels with native code for all each supported - operating systems (Linux, macOS, Windows) and Python versions (3.x) - combinations using remote Ci jobs -- collect source distributions for all required packages -- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like - repository (using GitHub) -- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development Approach @@ -56,35 +55,65 @@ The processing is organized around these key objects: -- A PyPiPackage represents a PyPI package with its name and version. It tracks - the downloadable Distribution objects for that version: +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: - - one Sdist source Distribution object - - a list of Wheel binary Distribution objects + - one Sdist source Distribution + - a list of Wheel binary Distribution - A Distribution (either a Wheel or Sdist) is identified by and created from its - filename. It also has the metadata used to populate an .ABOUT file and - document origin and license. A Distribution can be fetched from Repository. - Metadata can be loaded from and dumped to ABOUT files and optionally from - DejaCode package data. + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +""" +Wheel downloader -- An Environment is a combination of a Python version and operating system. - A Wheel Distribution also has Python/OS tags is supports and these can be - supported in a given Environment. +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} -- Paths or URLs to "filenames" live in a Repository, either a plain - LinksRepository (an HTML page listing URLs or a local directory) or a - PypiRepository (a PyPI simple index where each package name has an HTML page - listing URLs to all distribution types and versions). - Repositories and Distributions are related through filenames. + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message - The Wheel models code is partially derived from the mit-licensed pip and the - Distribution/Wheel/Sdist design has been heavily inspired by the packaging- - dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung """ -TRACE = False +TRACE = True +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False # Supported environments PYTHON_VERSIONS = "36", "37", "38", "39", "310" @@ -106,16 +135,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m"], - "37": ["cp37", "cp37m"], - "38": ["cp38", "cp38m"], - "39": ["cp39", "cp39m"], - "310": ["cp310", "cp310m"], - "36": ["cp36", "abi3"], - "37": ["cp37", "abi3"], - "38": ["cp38", "abi3"], - "39": ["cp39", "abi3"], - "310": ["cp310", "abi3"], + "36": ["cp36", "cp36m", "abi3"], + "37": ["cp37", "cp37m", "abi3"], + "38": ["cp38", "cp38m", "abi3"], + "39": ["cp39", "cp39m", "abi3"], + "310": ["cp310", "cp310m", "abi3"], } PLATFORMS_BY_OS = { @@ -154,7 +178,13 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" + +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" + +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( @@ -171,169 +201,134 @@ def get_python_dot_version(version): ) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = "https://pypi.org/simple" - LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() -# time to wait build for in seconds, as a string -# 0 measn no wait -DEFAULT_ROMP_BUILD_WAIT = "5" +collect_urls = re.compile('href="([^"]+)"').findall ################################################################################ -# -# Fetch remote wheels and sources locally -# +# Fetch wheels and sources locally ################################################################################ -def fetch_wheels( - environment=None, - requirements_file="requirements.txt", - allow_unpinned=False, +class DistributionNotFound(Exception): + pass + + +def download_wheel( + name, + version, + environment, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the wheel of packages listed in the ``requirements_file`` - requirements file into ``dest_dir`` directory. + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints from the + PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` + directory. - Only get wheels for the ``environment`` Enviromnent constraints. If the - provided ``environment`` is None then the current Python interpreter - environment is used implicitly. + Raise a DistributionNotFound if no wheel is not found. Otherwise, return a + tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + """ + if TRACE_DEEP: + print(f" download_wheel: {name}=={version}: {environment}") - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. + fetched_wheel_filenames = [] + existing_wheel_filenames = [] + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.wheels: + continue - Use exclusively direct downloads from a remote repo at URL - ``remote_links_url``. If ``remote_links_url`` is a path, use this as a - directory of links instead of a URL. + supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) + if not supported_wheels: + continue - Yield tuples of (PypiPackage, error) where is None on success. - """ - missed = [] + for wheel in supported_wheels: + if os.path.exists(os.path.join(dest_dir, wheel.filename)): + # do not refetch + existing_wheel_filenames.append(wheel.filename) + continue - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + if TRACE: + print(f" Fetching wheel from index: {wheel.download_url}") + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.add(fetched_wheel_filename) - try: - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) except Exception as e: - raise Exception( - dict( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) from e - - fetched_filenames = set() - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - version, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_wheels: Missing package in remote repo: {nv}" + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e - else: - fetched_filename = package.fetch_wheel( - environment=environment, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) + if not fetched_wheel_filenames and not existing_wheel_filenames: + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") - if fetched_filename: - fetched_filenames.add(fetched_filename) - error = None - else: - if fetched_filename in fetched_filenames: - error = None - else: - error = f"Failed to fetch" - yield package, error - - if missed: - rr = get_remote_repo() - print() - print(f"===> fetch_wheels: Missed some packages") - for n, v in missed: - nv = f"{n}=={v}" if v else n - print(f"Missed package {nv} in remote repo, has only:") - for pv in rr.get_versions(n): - print(" ", pv) - raise Exception("Missed some packages in remote repo") + return fetched_wheel_filenames, existing_wheel_filenames -def fetch_sources( - requirements_file="requirements.txt", - allow_unpinned=False, +def download_sdist( + name, + version, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the dependent package sources listed in the - ``requirements_file`` requirements file into ``dest_dir`` destination - directory. - - Use direct downloads to achieve this (not pip download). Use exclusively the - packages found from a remote repo at URL ``remote_links_url``. If - ``remote_links_url`` is a path, use this as a directory of links instead of - a URL. + Download the sdist source distribution of package ``name`` and ``version`` + from the PyPI simple repository ``index_urls`` list of URLs into the + ``dest_dir`` directory. - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. - - Yield tuples of (PypiPackage, error message) for each package where error - message will empty on success. + Raise a DistributionNotFound if this was not found. Return the filename if + downloaded and False if not downloaded because it already exists. """ - missed = [] + if TRACE_DEEP: + print(f"download_sdist: {name}=={version}: ") - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.sdist: + continue - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) + if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): + # do not refetch + return False + if TRACE: + print(f" Fetching sources from index: {pypi_package.sdist.download_url}") + fetched = pypi_package.sdist.download(dest_dir=dest_dir) + if fetched: + return pypi_package.sdist.filename - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - name, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_sources: Missing package in remote repo: {nv}" + except Exception as e: + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e - elif not package.sdist: - yield package, f"Missing sdist in links" + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") - else: - fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f"Failed to fetch" if not fetched else None - yield package, error - if missed: - raise Exception(f"Missing source packages in {remote_links_url}", missed) + +def get_package_versions( + name, + version=None, + index_urls=PYPI_INDEXES, +): + """ + Yield PypiPackages with ``name`` and ``version`` from the PyPI simple + repository ``index_urls`` list of URLs. + If ``version`` is not provided, return the latest available versions. + """ + for index_url in index_urls: + try: + repo = get_pypi_repo(index_url) + package = repo.get_package(name, version) + if package: + yield package + except RemoteNotFetchedException as e: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ @@ -387,7 +382,7 @@ def sortable_name_version(self): @classmethod def sorted(cls, namevers): - return sorted(namevers, key=cls.sortable_name_version) + return sorted(namevers or [], key=cls.sortable_name_version) @attr.attributes @@ -411,13 +406,6 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) - path_or_url = attr.ib( - repr=False, - type=str, - default="", - metadata=dict(help="Path or download URL."), - ) - sha256 = attr.ib( repr=False, type=str, @@ -546,21 +534,60 @@ def package_url(self): @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith("https://"): - return self.path_or_url - else: - return self.get_best_download_url() + return self.get_best_download_url() + + def get_best_download_url( + self, + index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), + ): + """ + Return the best download URL for this distribution where best means that + PyPI is better and our selfhosted repo URLs are second. + If none is found, return a synthetic remote URL. + """ + for index_url in index_urls: + pypi_package = get_pypi_package( + name=self.normalized_name, + version=self.version, + index_url=index_url, + ) + if pypi_package: + if isinstance(pypi_package, tuple): + raise Exception("############", repr(pypi_package)) + try: + pypi_url = pypi_package.get_url_for_filename(self.filename) + except Exception as e: + raise Exception(repr(pypi_package)) from e + if pypi_url: + return pypi_url + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + fetch_and_save_path_or_url( + filename=self.filename, + dest_dir=dest_dir, + path_or_url=self.path_or_url, + as_text=False, + ) + return self.filename @property def about_filename(self): return f"{self.filename}.ABOUT" - def has_about_file(self, dest_dir=THIRDPARTY_DIR): - return os.path.exists(os.path.join(dest_dir, self.about_filename)) - @property def about_download_url(self): - return self.build_remote_download_url(self.about_filename) + return f"{ABOUT_BASE_URL}/{self.about_filename}" @property def notice_filename(self): @@ -568,7 +595,7 @@ def notice_filename(self): @property def notice_download_url(self): - return self.build_remote_download_url(self.notice_filename) + return f"{ABOUT_BASE_URL}/{self.notice_filename}" @classmethod def from_path_or_url(cls, path_or_url): @@ -601,81 +628,10 @@ def from_filename(cls, filename): Return a distribution built from the data found in a `filename` string. Raise an exception if this is not a valid filename """ + filename = os.path.basename(filename.strip("/")) clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - @classmethod - def from_data(cls, data, keep_extra=False): - """ - Return a distribution built from a `data` mapping. - """ - filename = data["filename"] - dist = cls.from_filename(filename) - dist.update(data, keep_extra=keep_extra) - return dist - - @classmethod - def from_dist(cls, data, dist): - """ - Return a distribution built from a `data` mapping and update it with data - from another dist Distribution. Return None if it cannot be created - """ - # We can only create from a dist of the same package - has_same_key_fields = all( - data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") - ) - if not has_same_key_fields: - print( - f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) - if not has_key_field_values: - print( - f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - data = dict(data) - # do not overwrite the data with the other dist - # only supplement - data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)}) - return cls.from_data(data) - - @classmethod - def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): - """ - Return a direct download URL for a file in our remote repo - """ - return f"{base_url}/{filename}" - - def get_best_download_url(self): - """ - Return the best download URL for this distribution where best means that - PyPI is better and our own remote repo URLs are second. - If none is found, return a synthetic remote URL. - """ - name = self.normalized_name - version = self.version - filename = self.filename - - pypi_package = get_pypi_package(name=name, version=version) - if pypi_package: - pypi_url = pypi_package.get_url_for_filename(filename) - if pypi_url: - return pypi_url - - remote_package = get_remote_package(name=name, version=version) - if remote_package: - remote_url = remote_package.get_url_for_filename(filename) - if remote_url: - return remote_url - else: - # the package may not have been published yet, so we craft a URL - # using our remote base URL - return self.build_remote_download_url(self.filename) - def purl_identifiers(self, skinny=False): """ Return a mapping of non-empty identifier name/values for the purl @@ -781,9 +737,11 @@ def save_if_modified(location, content): fo.write(content) return True + as_about = self.to_about() + save_if_modified( location=os.path.join(dest_dir, self.about_filename), - content=saneyaml.dump(self.to_about()), + content=saneyaml.dump(as_about), ) notice_text = self.notice_text and self.notice_text.strip() @@ -844,7 +802,10 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url) + about_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.about_download_url, + as_text=True, + ) except RemoteNotFetchedException: return False @@ -855,7 +816,10 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) + notice_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.notice_download_url, + as_text=True, + ) if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: @@ -892,26 +856,23 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): return False return True - def get_pip_hash(self): - """ - Return a pip hash option string as used in requirements for this dist. - """ - assert self.sha256, f"Missinh SHA256 for dist {self}" - return f"--hash=sha256:{self.sha256}" - def get_license_keys(self): try: - keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) except license_expression.ExpressionParseError: return ["unknown"] return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ - Fetch license files is missing in `dest_dir`. + Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - paths_or_urls = get_remote_repo().links + urls = LinksRepository.from_url().links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -924,7 +885,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try remotely first - lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save_path_or_url( filename=filename, @@ -960,9 +921,17 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = "zip" if self.filename.endswith(".whl") else None - dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right @@ -983,7 +952,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f"!!!!PKG-INFO not found in {self.filename}") + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) @@ -1075,6 +1044,20 @@ def update(self, data, overwrite=False, keep_extra=True): return updated +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + class InvalidDistributionFilename(Exception): pass @@ -1243,15 +1226,12 @@ def is_supported_by_tags(self, tags): """ Return True is this wheel is compatible with one of a list of PEP 425 tags. """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) return not self.tags.isdisjoint(tags) - def is_supported_by_environment(self, environment): - """ - Return True if this wheel is compatible with the Environment - `environment`. - """ - return not self.is_supported_by_tags(environment.tags) - def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be @@ -1306,8 +1286,8 @@ class PypiPackage(NameVer): sdist = attr.ib( repr=False, - type=str, - default="", + type=Sdist, + default=None, metadata=dict(help="Sdist source distribution for this package."), ) @@ -1328,22 +1308,14 @@ def specifier(self): else: return self.name - @property - def specifier_with_hashes(self): - """ - Return a requirement specifier for this package with --hash options for - all its distributions - """ - items = [self.specifier] - items += [d.get_pip_hashes() for d in self.get_distributions()] - return " \\\n ".join(items) - - def get_supported_wheels(self, environment): + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the Environment `environment`. """ envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) for wheel in self.wheels: if wheel.is_supported_by_tags(envt_tags): yield wheel @@ -1369,6 +1341,8 @@ def package_from_dists(cls, dists): >>> assert package.wheels == [w1, w2] """ dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") if not dists: return @@ -1379,13 +1353,21 @@ def package_from_dists(cls, dists): package = PypiPackage(name=normalized_name, version=version) for dist in dists: - if dist.normalized_name != normalized_name or dist.version != version: + if dist.normalized_name != normalized_name: if TRACE: print( - f" Skipping inconsistent dist name and version: {dist} " - f'Expected instead package name: {normalized_name} and version: "{version}"' + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" ) continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue if isinstance(dist, Sdist): package.sdist = dist @@ -1396,39 +1378,41 @@ def package_from_dists(cls, dists): else: raise Exception(f"Unknown distribution type: {dist}") + if TRACE_DEEP: + print(f"package_from_dists: {package}") + return package @classmethod - def packages_from_one_path_or_url(cls, path_or_url): + def packages_from_dir(cls, directory): """ - Yield PypiPackages built from files found in at directory path or the - URL to an HTML page (that will be fetched). + Yield PypiPackages built from files found in at directory path. """ - extracted_paths_or_urls = get_paths_or_urls(path_or_url) - return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls) + base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return cls.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. """ - dists = cls.get_dists(paths_or_urls) + dists = cls.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( dists, key=NameVer.sortable_name_version, ): - yield PypiPackage.package_from_dists(dists_of_package) - - @classmethod - def get_versions_from_path_or_url(cls, name, path_or_url): - """ - Return a subset list from a list of PypiPackages version at `path_or_url` - that match PypiPackage `name`. - """ - packages = cls.packages_from_one_path_or_url(path_or_url) - return cls.get_versions(name, packages) + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package @classmethod def get_versions(cls, name, packages): @@ -1451,15 +1435,6 @@ def get_latest_version(cls, name, packages): return return versions[-1] - @classmethod - def get_outdated_versions(cls, name, packages): - """ - Return all versions except the latest version of PypiPackage `name` from a - list of `packages`. - """ - versions = cls.get_versions(name, packages) - return versions[:-1] - @classmethod def get_name_version(cls, name, version, packages): """ @@ -1467,100 +1442,23 @@ def get_name_version(cls, name, version, packages): or None if it is not found. If `version` is None, return the latest version found. """ - if version is None: + if TRACE_ULTRA_DEEP: + print("get_name_version:", name, version, packages) + if not version: return cls.get_latest_version(name, packages) nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return + return name, version if len(nvs) == 1: return nvs[0] raise Exception(f"More than one PypiPackage with {name}=={version}") - def fetch_wheel( - self, - environment=None, - fetched_filenames=None, - dest_dir=THIRDPARTY_DIR, - ): - """ - Download a binary wheel of this package matching the ``environment`` - Enviromnent constraints into ``dest_dir`` directory. - - Return the wheel filename if it was fetched, None otherwise. - - If the provided ``environment`` is None then the current Python - interpreter environment is used implicitly. Do not refetch wheel if - their name is in a provided ``fetched_filenames`` set. - """ - fetched_wheel_filename = None - if fetched_filenames is not None: - fetched_filenames = fetched_filenames - else: - fetched_filenames = set() - - supported_wheels = list(self.get_supported_wheels(environment)) - for wheel in supported_wheels: - - if wheel.filename not in fetched_filenames: - fetch_and_save_path_or_url( - filename=wheel.filename, - path_or_url=wheel.path_or_url, - dest_dir=dest_dir, - as_text=False, - ) - fetched_filenames.add(wheel.filename) - fetched_wheel_filename = wheel.filename - - # TODO: what if there is more than one? - break - - return fetched_wheel_filename - - def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): - """ - Download the source distribution into `dest_dir` directory. Return the - fetched filename if it was fetched, False otherwise. - """ - if self.sdist: - assert self.sdist.filename - if TRACE: - print("Fetching source for package:", self.name, self.version) - fetch_and_save_path_or_url( - filename=self.sdist.filename, - dest_dir=dest_dir, - path_or_url=self.sdist.path_or_url, - as_text=False, - ) - if TRACE: - print(" --> file:", self.sdist.filename) - return self.sdist.filename - else: - print(f"Missing sdist for: {self.name}=={self.version}") - return False - - def delete_files(self, dest_dir=THIRDPARTY_DIR): - """ - Delete all PypiPackage files from `dest_dir` including wheels, sdist and - their ABOUT files. Note that we do not delete licenses since they can be - shared by several packages: therefore this would be done elsewhere in a - function that is aware of all used licenses. - """ - for to_delete in self.wheels + [self.sdist]: - if not to_delete: - continue - tdfn = to_delete.filename - for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: - target = os.path.join(dest_dir, deletable) - if os.path.exists(target): - print(f"Deleting outdated {target}") - fileutils.delete(target) - @classmethod - def get_dists(cls, paths_or_urls): + def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of `paths_or_urls` to wheels or source distributions. @@ -1574,9 +1472,9 @@ def get_dists(cls, paths_or_urls): ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl - ... httsp://example.com/bar/bitarray-0.8.1.tar.gz + ... https://example.com/bar/bitarray-0.8.1.tar.gz ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.get_dists(paths_or_urls)) + >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: ... r.filename = '' ... r.path_or_url = '' @@ -1590,18 +1488,28 @@ def get_dists(cls, paths_or_urls): ... Wheel(name='bitarray', version='0.8.1', build='', ... python_versions=['cp36'], abis=['cp36m'], ... platforms=['win_amd64']), + ... Sdist(name='bitarray', version='0.8.1'), ... Sdist(name='bitarray', version='0.8.1') ... ] >>> assert expected == result """ + dists = [] + if TRACE_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: - yield Distribution.from_path_or_url(path_or_url) + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + ) except InvalidDistributionFilename: - if TRACE: - print(f"Skipping invalid distribution from: {path_or_url}") + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") continue + return dists def get_distributions(self): """ @@ -1626,10 +1534,11 @@ class Environment: """ An Environment describes a target installation environment with its supported Python version, ABI, platform, implementation and related - attributes. We can use these to pass as `pip download` options and force - fetching only the subset of packages that match these Environment - constraints as opposed to the current running Python interpreter - constraints. + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. """ python_version = attr.ib( @@ -1648,18 +1557,21 @@ class Environment: type=str, default="cp", metadata=dict(help="Python implementation supported by this environment."), + repr=False, ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help="List of ABI tags supported by this environment."), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, ) platforms = attr.ib( type=list, default=attr.Factory(list), metadata=dict(help="List of platform tags supported by this environment."), + repr=False, ) @classmethod @@ -1677,18 +1589,20 @@ def from_pyver_and_os(cls, python_version, operating_system): def get_pip_cli_options(self): """ - Return a list of pip command line options for this environment. + Return a list of pip download command line options for this environment. """ options = [ "--python-version", self.python_version, "--implementation", self.implementation, - "--abi", - self.abi, ] + for abi in self.abis: + options.extend(["--abi", abi]) + for platform in self.platforms: options.extend(["--platform", platform]) + return options def tags(self): @@ -1713,11 +1627,18 @@ def tags(self): @attr.attributes -class Repository: +class PypiSimpleRepository: """ - A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc. + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. """ + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), + ) + packages_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), @@ -1730,125 +1651,157 @@ class Repository: metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), ) - def get_links(self, *args, **kwargs): - raise NotImplementedError() - def get_versions(self, name): """ Return a list of all available PypiPackage version for this package name. The list may be empty. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + self._populate_links_and_packages(name) + return self.packages_by_normalized_name.get(name, []) + + def get_latest_version(self, name): + """ + Return the latest PypiPackage version for this package name or None. + """ + versions = self.get_versions(name) + return PypiPackage.get_latest_version(name, versions) def get_package(self, name, version): """ Return the PypiPackage with name and version or None. """ - raise NotImplementedError() + versions = self.get_versions(name) + if TRACE_DEEP: + print("PypiPackage.get_package:versions:", versions) + return PypiPackage.get_name_version(name, version, versions) - def get_latest_version(self, name): + def _fetch_links(self, name, _LINKS={}): """ - Return the latest PypiPackage version for this package name or None. + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + index_url = self.index_url + name = name and NameVer.normalize_name(name) + index_url = index_url.strip("/") + index_url = f"{index_url}/{name}" -@attr.attributes -class LinksRepository(Repository): - """ - Represents a simple links repository which is either a local directory with - Python wheels and sdist or a remote URL to an HTML with links to these. - (e.g. suitable for use with pip --find-links). - """ + if TRACE_DEEP: + print( + f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", + index_url in _LINKS, + ) - path_or_url = attr.ib( - type=str, - default="", - metadata=dict(help="Package directory path or URL"), - ) + if index_url not in _LINKS: + text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - links = attr.ib( - type=list, - default=attr.Factory(list), - metadata=dict(help="List of links available in this repo"), - ) + links = _LINKS[index_url] + if TRACE_DEEP: + print(f" Found links {links!r}") + return links - def __attrs_post_init__(self): - if not self.links: - self.links = get_paths_or_urls(links_url=self.path_or_url) - if not self.packages_by_normalized_name: - for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links): - normalized_name = p.normalized_name - self.packages_by_normalized_name[normalized_name].append(p) - self.packages_by_normalized_name_version[(normalized_name, p.version)] = p + def _populate_links_and_packages(self, name): + name = name and NameVer.normalize_name(name) - def get_links(self, *args, **kwargs): - return self.links or [] + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:name:", name) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - return self.packages_by_normalized_name.get(name, []) + links = self._fetch_links(name) + packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:packages:", packages) - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + self.packages_by_normalized_name[name] = packages + + for p in packages: + name = name and NameVer.normalize_name(p.name) + self.packages_by_normalized_name_version[(name, p.version)] = p @attr.attributes -class PypiRepository(Repository): +class LinksRepository: """ - Represents the public PyPI simple index. - It is populated lazily based on requested packages names + Represents a simple links repository such an HTTP directory listing or a + page with links. """ - simple_url = attr.ib( + url = attr.ib( type=str, - default=PYPI_SIMPLE_URL, - metadata=dict(help="Base PyPI simple URL for this index."), + default="", + metadata=dict(help="Links directory URL"), ) - links_by_normalized_name = attr.ib( - type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [links]} available in this repo"), + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of links available in this repo"), ) - def _fetch_links(self, name): - name = name and NameVer.normalize_name(name) - return find_pypi_links(name=name, simple_url=self.simple_url) + def __attrs_post_init__(self): + if not self.links: + self.links = self.find_links() - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - if name in self.links_by_normalized_name: - return + def find_links(self): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() - links = self._fetch_links(name) - self.links_by_normalized_name[name] = links + if TRACE_DEEP: + print(f"Base URL {base_url}") - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - self.packages_by_normalized_name[name] = packages + text = fetch_content_from_path_or_url_through_cache( + path_or_url=links_url, + as_text=True, + ) - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue - def get_links(self, name, *args, **kwargs): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + plink = urllib.parse.urlsplit(link) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.packages_by_normalized_name.get(name, []) + if plink.scheme: + # full URL kept as-is + url = link - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + else: + # relative link + url = f"{links_url}/{link}" + + if TRACE_DEEP: + print(f"Adding URL: {url}") + + links.append(url) + + if TRACE: + print(f"Found {len(links)} links at {links_url}") + return links + + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url) + return _LINKS_REPO[url] ################################################################################ @@ -1862,51 +1815,27 @@ def get_local_packages(directory=THIRDPARTY_DIR): Return the list of all PypiPackage objects built from a local directory. Return an empty list if the package cannot be found. """ - return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory)) - - -def get_local_repo(directory=THIRDPARTY_DIR): - return LinksRepository(path_or_url=directory) + return list(PypiPackage.packages_from_dir(directory=directory)) -_REMOTE_REPO = None +def get_pypi_repo(index_url, _PYPI_REPO={}): + if index_url not in _PYPI_REPO: + _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) + return _PYPI_REPO[index_url] -def get_remote_repo(remote_links_url=REMOTE_LINKS_URL): - global _REMOTE_REPO - if not _REMOTE_REPO: - _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url) - return _REMOTE_REPO - - -def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): +def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: - return get_remote_repo(remote_links_url).get_package(name, version) - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - -_PYPI_REPO = None - - -def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL): - global _PYPI_REPO - if not _PYPI_REPO: - _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url) - return _PYPI_REPO - + package = get_pypi_repo(index_url).get_package(name, version) + if verbose: + print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + return package -def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): - """ - Return a PypiPackage or None. - """ - try: - return get_pypi_repo(pypi_simple_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ @@ -1936,8 +1865,8 @@ def get(self, path_or_url, as_text=True): Get a file from a `path_or_url` through the cache. `path_or_url` can be a path or a URL to a file. """ - filename = os.path.basename(path_or_url.strip("/")) - cached = os.path.join(self.directory, filename) + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) @@ -1948,32 +1877,23 @@ def get(self, path_or_url, as_text=True): else: return get_local_file_content(path=cached, as_text=as_text) - def put(self, filename, content): - """ - Put in the cache the `content` of `filename`. - """ - cached = os.path.join(self.directory, filename) - wmode = "wb" if isinstance(content, bytes) else "w" - with open(cached, wmode) as fo: - fo.write(content) - def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if path_or_url.startswith("file://") or ( - path_or_url.startswith("/") and os.path.exists(path_or_url) - ): - return get_local_file_content(path=path_or_url, as_text=as_text) - - elif path_or_url.startswith("https://"): + if path_or_url.startswith("https://"): if TRACE: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + else: raise Exception(f"Unsupported URL scheme: {path_or_url}") @@ -2016,6 +1936,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header + print(f" DOWNLOADING {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -2039,76 +1960,11 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def get_url_content_if_modified( - url, - md5, - _delay=0, -): - """ - Return fetched content bytes at `url` or None if the md5 has not changed. - Retries multiple times to fetch if there is a HTTP 429 throttling response - and this with an increasing delay. - """ - time.sleep(_delay) - headers = None - if md5: - etag = f'"{md5}"' - headers = {"If-None-Match": f"{etag}"} - - # using a GET with stream=True ensure we get the the final header from - # several redirects and that we can ignore content there. A HEAD request may - # not get us this last header - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: - status = response.status_code - if status == requests.codes.too_many_requests and _delay < 20: # NOQA - # too many requests: start waiting with some exponential delay - _delay = (_delay * 2) or 1 - return get_url_content_if_modified(url=url, md5=md5, _delay=_delay) - - elif status == requests.codes.not_modified: # NOQA - # all is well, the md5 is the same - return None - - elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") - - return response.content - - -def get_remote_headers(url): - """ - Fetch and return a mapping of HTTP headers of `url`. - """ - headers, _content = get_remote_file_content(url, headers_only=True) - return headers - - -def fetch_and_save_filename_from_paths_or_urls( - filename, - paths_or_urls, - dest_dir=THIRDPARTY_DIR, +def fetch_content_from_path_or_url_through_cache( + path_or_url, as_text=True, + cache=Cache(), ): - """ - Return the content from fetching the `filename` file name found in the - `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an - Exception on errors. Treats the content as text if `as_text` is True - otherwise as binary. - """ - path_or_url = get_link_for_filename( - filename=filename, - paths_or_urls=paths_or_urls, - ) - - return fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, - path_or_url=path_or_url, - as_text=as_text, - ) - - -def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()): """ Return the content from fetching at path or URL. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as @@ -2118,23 +1974,24 @@ def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cach Note: the `cache` argument is a global, though it does not really matter since it does not hold any state which is only kept on disk. """ - if cache: - return cache.get(path_or_url=path_or_url, as_text=as_text) - else: - return get_file_content(path_or_url=path_or_url, as_text=as_text) + return cache.get(path_or_url=path_or_url, as_text=as_text) -def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True): +def fetch_and_save_path_or_url( + filename, + dest_dir, + path_or_url, + as_text=True, +): """ Return the content from fetching the `filename` file name at URL or path and save to `dest_dir`. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as binary. """ - if through_cache: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text) - else: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) - + content = fetch_content_from_path_or_url_through_cache( + path_or_url=path_or_url, + as_text=as_text, + ) output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" with open(output, wmode) as fo: @@ -2143,398 +2000,66 @@ def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, th ################################################################################ -# -# Sync and fix local thirdparty directory for various issues and gaps -# +# Requirements processing ################################################################################ -def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): - """ - Given a thirdparty dir, fetch missing source distributions from our remote - repo or PyPI. Return a list of (name, version) tuples for source - distribution that were not found - """ - not_found = [] - local_packages = get_local_packages(directory=dest_dir) - remote_repo = get_remote_repo() - pypi_repo = get_pypi_repo() - - for package in local_packages: - if not package.sdist: - print(f"Finding sources for: {package.name}=={package.version}: ", end="") - try: - pypi_package = pypi_repo.get_package(name=package.name, version=package.version) - - if pypi_package and pypi_package.sdist: - print(f"Fetching sources from Pypi") - pypi_package.fetch_sdist(dest_dir=dest_dir) - continue - else: - remote_package = remote_repo.get_package( - name=package.name, version=package.version - ) - - if remote_package and remote_package.sdist: - print(f"Fetching sources from Remote") - remote_package.fetch_sdist(dest_dir=dest_dir) - continue - - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - print(f"No sources found") - not_found.append( - ( - package.name, - package.version, - ) - ) - - return not_found - - -def fetch_missing_wheels( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, +def get_required_remote_packages( + requirements_file="requirements.txt", + index_url=PYPI_SIMPLE_URL, ): """ - Given a thirdparty dir fetch missing wheels for all known combos of Python - versions and OS. Return a list of tuple (Package, Environment) for wheels - that were not found locally or remotely. + Yield tuple of (name, version, PypiPackage) for packages listed in the + `requirements_file` requirements file and found in the PyPI index + ``index_url`` URL. """ - local_packages = get_local_packages(directory=dest_dir) - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - packages_and_envts = itertools.product(local_packages, environments) + required_name_versions = load_requirements(requirements_file=requirements_file) + return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - not_fetched = [] - fetched_filenames = set() - for package, envt in packages_and_envts: - filename = package.fetch_wheel( - environment=envt, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) - - if filename: - fetched_filenames.add(filename) - else: - not_fetched.append( - ( - package, - envt, - ) - ) - - return not_fetched - - -def build_missing_wheels( - packages_and_envts, - build_remotely=False, - with_deps=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, +def get_required_packages( + required_name_versions, + index_url=PYPI_SIMPLE_URL, ): """ - Build all wheels in a list of tuple (Package, Environment) and save in - `dest_dir`. Return a list of tuple (Package, Environment), and a list of - built wheel filenames. + Yield tuple of (name, version) or a PypiPackage for package name/version + listed in the ``required_name_versions`` list and found in the PyPI index + ``index_url`` URL. """ - - not_built = [] - built_filenames = [] - - packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) - - for package, pkg_envts in packages_and_envts: - - envts = [envt for _pkg, envt in pkg_envts] - python_versions = sorted(set(e.python_version for e in envts)) - operating_systems = sorted(set(e.operating_system for e in envts)) - built = None - try: - built = build_wheels( - requirements_specifier=package.specifier, - with_deps=with_deps, - build_remotely=build_remotely, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=TRACE, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - print(".") - except Exception as e: - import traceback - - print("#############################################################") - print("############# WHEEL BUILD FAILED ######################") - traceback.print_exc() - print() - print("#############################################################") - - if not built: - for envt in pkg_envts: - not_built.append((package, envt)) - else: - for bfn in built: - print(f" --> Built wheel: {bfn}") - built_filenames.append(bfn) - - return not_built, built_filenames - - -################################################################################ -# -# Functions to handle remote or local repo used to "find-links" -# -################################################################################ - - -def get_paths_or_urls(links_url): - if links_url.startswith("https:"): - paths_or_urls = find_links_from_release_url(links_url) - else: - paths_or_urls = find_links_from_dir(links_url) - return paths_or_urls - - -def find_links_from_dir(directory=THIRDPARTY_DIR): - """ - Return a list of path to files in `directory` for any file that ends with - any of the extension in the list of `extensions` strings. - """ - base = os.path.abspath(directory) - files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] - return files - - -get_links = re.compile('href="([^"]+)"').findall - - -def find_links_from_release_url(links_url=REMOTE_LINKS_URL): - """ - Return a list of download link URLs found in the HTML page at `links_url` - URL that starts with the `prefix` string and ends with any of the extension - in the list of `extensions` strings. Use the `base_url` to prefix the links. - """ - if TRACE: - print(f"Finding links for {links_url}") - - plinks_url = urllib.parse.urlparse(links_url) - - base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - if TRACE: - print(f"Base URL {base_url}") + print("get_required_packages", index_url) - _headers, text = get_remote_file_content(links_url) - links = [] - for link in get_links(text): - if not link.endswith(EXTENSIONS): - continue - - plink = urllib.parse.urlsplit(link) - - if plink.scheme: - # full URL kept as-is - url = link - - if plink.path.startswith("/"): - # absolute link - url = f"{base_url}{link}" - - else: - # relative link - url = f"{links_url}/{link}" + repo = get_pypi_repo(index_url=index_url) + for name, version in required_name_versions: if TRACE: - print(f"Adding URL: {url}") - - links.append(url) - - if TRACE: - print(f"Found {len(links)} links at {links_url}") - return links - - -def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): - """ - Return a list of download link URLs found in a PyPI simple index for package name. - with the list of `extensions` strings. Use the `simple_url` PyPI url. - """ - if TRACE: - print(f"Finding links for {simple_url}") - - name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip("/") - simple_url = f"{simple_url}/{name}" - - _headers, text = get_remote_file_content(simple_url) - links = get_links(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - links = [l for l in links if l.endswith(EXTENSIONS)] - return links - - -def get_link_for_filename(filename, paths_or_urls): - """ - Return a link for `filename` found in the `links` list of URLs or paths. Raise an - exception if no link is found or if there are more than one link for that - file name. - """ - path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] - if not path_or_url: - raise Exception(f"Missing link to file: {filename}") - if not len(path_or_url) == 1: - raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) - return path_or_url[0] + print(" get_required_packages: name:", name, "version:", version) + yield repo.get_package(name, version) ################################################################################ -# -# Requirements processing -# +# Functions to update or fetch ABOUT and license files ################################################################################ -class MissingRequirementException(Exception): - pass - - -def get_required_packages(required_name_versions): - """ - Return a tuple of (remote packages, PyPI packages) where each is a mapping - of {(name, version): PypiPackage} for packages listed in the - `required_name_versions` list of (name, version) tuples. Raise a - MissingRequirementException with a list of missing (name, version) if a - requirement cannot be satisfied remotely or in PyPI. - """ - remote_repo = get_remote_repo() - - remote_packages = { - (name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions - } - - pypi_repo = get_pypi_repo() - pypi_packages = { - (name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions - } - - # remove any empty package (e.g. that do not exist in some place) - remote_packages = {nv: p for nv, p in remote_packages.items() if p} - pypi_packages = {nv: p for nv, p in pypi_packages.items() if p} - - # check that we are not missing any - repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys()) - missing_name_versions = required_name_versions.difference(repos_name_versions) - if missing_name_versions: - raise MissingRequirementException(sorted(missing_name_versions)) - - return remote_packages, pypi_packages - - -def get_required_remote_packages( - requirements_file="requirements.txt", - force_pinned=True, - remote_links_url=REMOTE_LINKS_URL, +def clean_about_files( + dest_dir=THIRDPARTY_DIR, ): """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI-like link repo - ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a - local directory path to a wheels directory if this is not a a URL. - """ - required_name_versions = load_requirements( - requirements_file=requirements_file, - force_pinned=force_pinned, - ) - - if remote_links_url.startswith("https://"): - repo = get_remote_repo(remote_links_url=remote_links_url) - else: - # a local path - assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" - repo = get_local_repo(directory=remote_links_url) - - for name, version in required_name_versions: - if version: - yield name, version, repo.get_package(name, version) - else: - yield name, version, repo.get_latest_version(name) - - -def update_requirements(name, version=None, requirements_file="requirements.txt"): - """ - Upgrade or add `package_name` with `new_version` to the `requirements_file` - requirements file. Write back requirements sorted with name and version - canonicalized. Note: this cannot deal with hashed or unpinned requirements. - Do nothing if the version already exists as pinned. + Given a thirdparty dir, clean ABOUT files """ - normalized_name = NameVer.normalize_name(name) - - is_updated = False - updated_name_versions = [] - for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False): - - existing_normalized_name = NameVer.normalize_name(existing_name) - - if normalized_name == existing_normalized_name: - if version != existing_version: - is_updated = True - updated_name_versions.append( - ( - existing_normalized_name, - existing_version, - ) - ) - - if is_updated: - updated_name_versions = sorted(updated_name_versions) - nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - - with open(requirements_file, "w") as fo: - fo.write(nvs) - - -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): - """ - Hash all the requirements found in the `requirements_file` - requirements file based on distributions available in `dest_dir` - """ - local_repo = get_local_repo(directory=dest_dir) - packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version - hashed = [] - for name, version in load_requirements(requirements_file, force_pinned=True): - package = packages_by_normalized_name_version.get((name, version)) - if not package: - raise Exception(f"Missing required package {name}=={version}") - hashed.append(package.specifier_with_hashes) - - with open(requirements_file, "w") as fo: - fo.write("\n".join(hashed)) - + local_packages = get_local_packages(directory=dest_dir) + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) -################################################################################ -# -# Functions to update or fetch ABOUT and license files -# -################################################################################ + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) -def add_fetch_or_update_about_and_license_files( - dest_dir=THIRDPARTY_DIR, - include_remote=True, - strip_classifiers=False, -): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2544,32 +2069,28 @@ def add_fetch_or_update_about_and_license_files( - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files - - TODO: make API calls to fetch package data from DejaCode - - The process consists in load and iterate on every package distributions, - collect data and then acsk to save. """ - local_packages = get_local_packages(directory=dest_dir) - local_repo = get_local_repo(directory=dest_dir) - - remote_repo = get_remote_repo() - def get_other_dists(_package, _dist): """ - Return a list of all the dists from package that are not the `dist` object + Return a list of all the dists from `_package` that are not the `_dist` + object """ return [d for d in _package.get_distributions() if d != _dist] + selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + for local_package in local_packages: for local_dist in local_package.get_distributions(): local_dist.load_about_data(dest_dir=dest_dir) local_dist.set_checksums(dest_dir=dest_dir) - if strip_classifiers and "classifiers" in local_dist.extra_data: - local_dist.extra_data.pop("classifiers", None) - local_dist.save_about_and_notice_files(dest_dir) - # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) @@ -2588,16 +2109,16 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version + # try to get another version of the same package that is not our version other_local_packages = [ p - for p in local_repo.get_versions(local_package.name) + for p in packages_by_name[local_package.name] if p.version != local_package.version ] - latest_local_version = other_local_packages and other_local_packages[-1] - if latest_local_version: - latest_local_dists = list(latest_local_version.get_distributions()) + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2615,9 +2136,35 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - if include_remote: - # lets try to fetch remotely - local_dist.load_remote_about_data() + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get a latest version of the same package that is not our version + other_remote_packages = [ + p + for p in selfhosted_repo.get_versions(local_package.name) + if p.version != local_package.version + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): @@ -2625,33 +2172,6 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version - other_remote_packages = [ - p - for p in remote_repo.get_versions(local_package.name) - if p.version != local_package.version - ] - - latest_version = other_remote_packages and other_remote_packages[-1] - if latest_version: - latest_dists = list(latest_version.get_distributions()) - for remote_dist in latest_dists: - remote_dist.load_remote_about_data() - if not remote_dist.has_key_metadata(): - # there is not much value to get other data if we are missing the key ones - continue - else: - local_dist.update_from_other_dist(remote_dist) - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - break - - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) - continue - # try to get data from pkginfo (no license though) local_dist.load_pkginfo_data(dest_dir=dest_dir) @@ -2661,8 +2181,6 @@ def get_other_dists(_package, _dist): lic_errs = local_dist.fetch_license_files(dest_dir) - # TODO: try to get data from dejacode - if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") if lic_errs: @@ -2680,9 +2198,9 @@ def get_other_dists(_package, _dist): def call(args, verbose=TRACE): """ Call args in a subprocess and display output on the fly if ``trace`` is True. - Return or raise stdout, stderr, returncode + Return a tuple of (returncode, stdout, stderr) """ - if TRACE: + if TRACE_DEEP: print("Calling:", " ".join(args)) with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" @@ -2700,312 +2218,78 @@ def call(args, verbose=TRACE): stdout, stderr = process.communicate() if not stdout.strip(): stdout = "\n".join(stdouts) + return process.returncode, stdout, stderr - returncode = process.returncode - - if returncode == 0: - return returncode, stdout, stderr - else: - raise Exception(returncode, stdout, stderr) - -def add_or_upgrade_built_wheels( - name, - version=None, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=TRACE, - remote_build_log_file=None, -): - """ - Add or update package `name` and `version` as a binary wheel saved in - `dest_dir`. Use the latest version if `version` is None. Return the a list - of the collected, fetched or built wheel file names or an empty list. - - Use the provided lists of `python_versions` (e.g. "36", "39") and - `operating_systems` (e.g. linux, windows or macos) to decide which specific - wheel to fetch or build. - - Include wheels for all dependencies if `with_deps` is True. - Build remotely is `build_remotely` is True. - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - assert name, "Name is required" - ver = version and f"=={version}" or "" - print(f"\nAdding wheels for package: {name}{ver}") - - if verbose: - print("python_versions:", python_versions) - print("operating_systems:", operating_systems) - - wheel_filenames = [] - # a mapping of {req specifier: {mapping build_wheels kwargs}} - wheels_to_build = {} - for python_version, operating_system in itertools.product(python_versions, operating_systems): - print( - f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" - ) - environment = Environment.from_pyver_and_os(python_version, operating_system) - - # Check if requested wheel already exists locally for this version - local_repo = get_local_repo(directory=dest_dir) - local_package = local_repo.get_package(name=name, version=version) - - has_local_wheel = False - if version and local_package: - for wheel in local_package.get_supported_wheels(environment): - has_local_wheel = True - wheel_filenames.append(wheel.filename) - break - if has_local_wheel: - print(f" local wheel exists: {wheel.filename}") - continue - - if not version: - pypi_package = get_pypi_repo().get_latest_version(name) - version = pypi_package.version - - # Check if requested wheel already exists remotely or in Pypi for this version - wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir - ) - if verbose: - print(" fetching package wheel:", wheel_filename) - if wheel_filename: - wheel_filenames.append(wheel_filename) - - # the wheel is not available locally, remotely or in Pypi - # we need to build binary from sources - requirements_specifier = f"{name}=={version}" - to_build = wheels_to_build.get(requirements_specifier) - if to_build: - to_build["python_versions"].append(python_version) - to_build["operating_systems"].append(operating_system) - else: - wheels_to_build[requirements_specifier] = dict( - requirements_specifier=requirements_specifier, - python_versions=[python_version], - operating_systems=[operating_system], - dest_dir=dest_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - for build_wheels_kwargs in wheels_to_build.values(): - bwheel_filenames = build_wheels(**build_wheels_kwargs) - wheel_filenames.extend(bwheel_filenames) - - return sorted(set(wheel_filenames)) - - -def build_wheels( - requirements_specifier, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=False, - remote_build_log_file=None, -): - """ - Given a pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) for all - `python_versions` and `operating_systems` combinations and save them - back in `dest_dir` and return a list of built wheel file names. - - Include wheels for all dependencies if `with_deps` is True. - - First try to build locally to process pure Python wheels, and fall back to - build remotey on all requested Pythons and operating systems. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - all_pure, builds = build_wheels_locally_if_pure_python( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - verbose=verbose, - dest_dir=dest_dir, - ) - for local_build in builds: - print(f"Built wheel: {local_build}") - - if all_pure: - return builds - - if build_remotely: - remote_builds = build_wheels_remotely_on_multiple_platforms( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=verbose, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - builds.extend(remote_builds) - - return builds - - -def build_wheels_remotely_on_multiple_platforms( - requirements_specifier, - with_deps=False, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - verbose=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) including wheels for - all dependencies for all `python_versions` and `operating_systems` - combinations and save them back in `dest_dir` and return a list of built - wheel file names. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - check_romp_is_configured() - pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = "" if with_deps else "--no-deps" - verbose = "--verbose" if verbose else "" - - if remote_build_log_file: - # zero seconds, no wait, log to file instead - wait_build_for = "0" - else: - wait_build_for = DEFAULT_ROMP_BUILD_WAIT - - romp_args = [ - "romp", - "--interpreter", - "cpython", - "--architecture", - "x86_64", - "--check-period", - wait_build_for, # in seconds - ] - - if remote_build_log_file: - romp_args += ["--build-log-file", remote_build_log_file] - - romp_args += pyos_options + [ - "--artifact-paths", - "*.whl", - "--artifact", - "artifacts.tar.gz", - "--command", - f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " - f"python -m pip {verbose} wheel {deps} {requirements_specifier}", - ] - - if verbose: - romp_args.append("--verbose") - - print(f"Building wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(romp_args)) - call(romp_args) - wheel_filenames = [] - if not remote_build_log_file: - wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) - for wfn in wheel_filenames: - print(f" built wheel: {wfn}") - return wheel_filenames - - -def fetch_remotely_built_wheels( - remote_build_log_file, +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, dest_dir=THIRDPARTY_DIR, - no_wait=False, - verbose=False, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, ): """ - Given a ``remote_build_log_file`` file path with a JSON lines log of a - remote build, fetch the built wheels and move them to ``dest_dir``. Return a - list of built wheel file names. - """ - wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT # in seconds - - romp_args = [ - "romp-fetch", - "--build-log-file", - remote_build_log_file, - "--check-period", - wait, + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", ] - if verbose: - romp_args.append("--verbose") - - print(f"Fetching built wheels from log file: {remote_build_log_file}") - print(f"Using command:", " ".join(romp_args)) - - call(romp_args, verbose=verbose) - - wheel_filenames = [] - - for art in os.listdir(os.getcwd()): - if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art): - continue - - print(f" Processing artifact archive: {art}") - wheel_fns = extract_tar(art, dest_dir) - for wfn in wheel_fns: - print(f" Retrieved built wheel: {wfn}") - wheel_filenames.extend(wheel_fns) - return wheel_filenames - - -def get_romp_pyos_options( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, -): - """ - Return a list of CLI options for romp - For example: - >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', - ... '--version', '3.9', '--version', '3.10', '--platform', 'linux', - ... '--platform', 'macos', '--platform', 'windows'] - >>> assert get_romp_pyos_options() == expected - """ - python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))] - pyos_options = list( - itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) - ) + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") - pyos_options += list( - itertools.chain.from_iterable( - ("--platform", plat) for plat in sorted(set(operating_systems)) - ) - ) + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) - return pyos_options + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) -def check_romp_is_configured(): - # these environment variable must be set before - has_envt = ( - os.environ.get("ROMP_BUILD_REQUEST_URL") - and os.environ.get("ROMP_DEFINITION_ID") - and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") - and os.environ.get("ROMP_USERNAME") - ) + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") - if not has_envt: - raise Exception( - "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " - "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " - "are required enironment variables." - ) + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error def build_wheels_locally_if_pure_python( @@ -3064,95 +2348,6 @@ def build_wheels_locally_if_pure_python( return all_pure, pure_built -# TODO: Use me -def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): - """ - Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its - tags for PyPI compatibility and making it smaller if possible. Return the - name of the new wheel if renamed or the existing new name otherwise. - """ - if is_pure_wheel(wheel_filename): - print(f"Pure wheel: {wheel_filename}, nothing to do.") - return wheel_filename - - original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") - awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] - call(awargs) - - audited = os.listdir(wheel_dir) - if not audited: - # cannot optimize wheel - return wheel_filename - - assert len(audited) == 1 - new_wheel_name = audited[0] - - new_wheel_loc = os.path.join(wheel_dir, new_wheel_name) - - # this needs to go now - os.remove(original_wheel_loc) - - if new_wheel_name == wheel_filename: - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel = Wheel.from_filename(new_wheel_name) - non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) - new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] - if not new_wheel.platforms: - print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel_cleaned_filename = new_wheel.to_filename() - new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename) - os.rename(new_wheel_loc, new_wheel_cleaned_loc) - return new_wheel_cleaned_filename - - -def extract_tar( - location, - dest_dir=THIRDPARTY_DIR, -): - """ - Extract a tar archive at `location` in the `dest_dir` directory. Return a - list of extracted locations (either directories or files). - """ - with open(location, "rb") as fi: - with tarfile.open(fileobj=fi) as tar: - members = list(tar.getmembers()) - tar.extractall(dest_dir, members=members) - - return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] - - -def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): - """ - Fetch the binary wheel for package `name` and `version` and save in - `dest_dir`. Use the provided `environment` Environment to determine which - specific wheel to fetch. - - Return the fetched wheel file name on success or None if it was not fetched. - Trying fetching from our own remote repo, then from PyPI. - """ - wheel_filename = None - remote_package = get_remote_package(name=name, version=version) - if TRACE: - print(" remote_package:", remote_package) - if remote_package: - wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - if wheel_filename: - return wheel_filename - - pypi_package = get_pypi_package(name=name, version=version) - if TRACE: - print(" pypi_package:", pypi_package) - if pypi_package: - wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - return wheel_filename - - def check_about(dest_dir=THIRDPARTY_DIR): try: subprocess.check_output(f"about check {dest_dir}".split()) @@ -3195,6 +2390,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ if not declared_licenses: return try: