Add scripts from scancode-toolkit/etc/release/ #33

Signed-off-by: Jono Yang <[email protected]>

etc/scripts/bootstrap.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import itertools
+
+import click
+
+import utils_thirdparty
+from utils_thirdparty import Environment
+from utils_thirdparty import PypiPackage
+
+
+@click.command()
+
+@click.option('-r', '--requirements-file',
+    type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
+    metavar='FILE',
+    multiple=True,
+    default=['requirements.txt'],
+    show_default=True,
+    help='Path to the requirements file(s) to use for thirdparty packages.',
+)
+@click.option('-d', '--thirdparty-dir',
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
+    metavar='DIR',
+    default=utils_thirdparty.THIRDPARTY_DIR,
+    show_default=True,
+    help='Path to the thirdparty directory where wheels are built and '
+         'sources, ABOUT and LICENSE files fetched.',
+)
+@click.option('-p', '--python-version',
+    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
+    metavar='PYVER',
+    default=utils_thirdparty.PYTHON_VERSIONS,
+    show_default=True,
+    multiple=True,
+    help='Python version(s) to use for this build.',
+)
+@click.option('-o', '--operating-system',
+    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
+    metavar='OS',
+    default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
+    multiple=True,
+    show_default=True,
+    help='OS(ses) to use for this build: one of linux, mac or windows.',
+)
+@click.option('-l', '--latest-version',
+    is_flag=True,
+    help='Get the latest version of all packages, ignoring version specifiers.',
+)
+@click.option('--sync-dejacode',
+    is_flag=True,
+    help='Synchronize packages with DejaCode.',
+)
+@click.option('--with-deps',
+    is_flag=True,
+    help='Also include all dependent wheels.',
+)
+@click.help_option('-h', '--help')
+def bootstrap(
+    requirements_file,
+    thirdparty_dir,
+    python_version,
+    operating_system,
+    with_deps,
+    latest_version,
+    sync_dejacode,
+    build_remotely=False,
+):
+    """
+    Boostrap a thirdparty Python packages directory from pip requirements.
+
+    Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for
+    the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible
+    with all the provided ``--python-version`` PYVER(s) and ```--operating_system``
+    OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and
+    .LICENSE files.
+
+    Optionally ignore version specifiers and use the ``--latest-version`` 
+    of everything.
+
+    Sources and wheels are fetched with attempts first from PyPI, then our remote repository.
+    If missing wheels are built as needed.
+    """
+    # rename variables for clarity since these are lists
+    requirements_files = requirements_file
+    python_versions = python_version
+    operating_systems = operating_system
+
+    # create the environments we need
+    evts = itertools.product(python_versions, operating_systems)
+    environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
+
+    # collect all packages to process from requirements files
+    # this will fail with an exception if there are packages we cannot find
+
+    required_name_versions = set()
+
+    for req_file in requirements_files:
+        nvs = utils_thirdparty.load_requirements(
+            requirements_file=req_file, force_pinned=False)
+        required_name_versions.update(nvs)
+    if latest_version:
+        required_name_versions = set((name, None) for name, _ver in required_name_versions)
+
+    print(f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES')
+
+    # fetch all available wheels, keep track of missing
+    # start with local, then remote, then PyPI
+
+    print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS')
+    # list of all the wheel filenames either pre-existing, fetched or built
+    # updated as we progress
+    available_wheel_filenames = []
+
+    local_packages_by_namever = {
+        (p.name, p.version): p
+        for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir)
+    }
+
+    # list of (name, version, environment) not local and to fetch
+    name_version_envt_to_fetch = []
+
+    # start with a local check
+    for (name, version), envt in itertools.product(required_name_versions, environments):
+        local_pack = local_packages_by_namever.get((name, version,))
+        if local_pack:
+            supported_wheels = list(local_pack.get_supported_wheels(environment=envt))
+            if supported_wheels:
+                available_wheel_filenames.extend(w.filename for w in supported_wheels)
+                print(f'====> No fetch or build needed. '
+                      f'Local wheel already available for {name}=={version} '
+                      f'on os: {envt.operating_system} for Python: {envt.python_version}')
+                continue
+
+        name_version_envt_to_fetch.append((name, version, envt,))
+
+    print(f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS')
+
+    # list of (name, version, environment) not fetch and to build
+    name_version_envt_to_build = []
+
+    # then check if the wheel can be fetched without building from remote and Pypi
+    for name, version, envt in name_version_envt_to_fetch:
+
+        fetched_fwn = utils_thirdparty.fetch_package_wheel(
+            name=name,
+            version=version,
+            environment=envt,
+            dest_dir=thirdparty_dir,
+        )
+
+        if fetched_fwn:
+            available_wheel_filenames.append(fetched_fwn)
+        else:
+            name_version_envt_to_build.append((name, version, envt,))
+
+    # At this stage we have all the wheels we could obtain without building
+    for name, version, envt in name_version_envt_to_build:
+        print(f'====> Need to build wheels for {name}=={version} on os: '
+              f'{envt.operating_system} for Python: {envt.python_version}')
+
+    packages_and_envts_to_build = [
+        (PypiPackage(name, version), envt)
+        for name, version, envt in name_version_envt_to_build
+    ]
+
+    print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS')
+
+    package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels(
+        packages_and_envts=packages_and_envts_to_build,
+        build_remotely=build_remotely,
+        with_deps=with_deps,
+        dest_dir=thirdparty_dir,
+)
+    if wheel_filenames_built:
+        available_wheel_filenames.extend(available_wheel_filenames)
+
+    for pack, envt in package_envts_not_built:
+        print(
+            f'====> FAILED to build any wheel for {pack.name}=={pack.version} '
+            f'on os: {envt.operating_system} for Python: {envt.python_version}'
+        )
+
+    print(f'==> FETCHING SOURCE DISTRIBUTIONS')
+    # fetch all sources, keep track of missing
+    # This is a list of (name, version)
+    utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
+
+    print(f'==> FETCHING ABOUT AND LICENSE FILES')
+    utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
+
+    ############################################################################
+    if sync_dejacode:
+        print(f'==> SYNC WITH DEJACODE')
+        # try to fetch from DejaCode any missing ABOUT
+        # create all missing DejaCode packages
+        pass
+
+    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+
+
+if __name__ == '__main__':
+    bootstrap()

etc/scripts/build_wheels.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import click
+
+import utils_thirdparty
+
+
+@click.command()
+
+@click.option('-n', '--name',
+    type=str,
+    metavar='PACKAGE_NAME',
+    required=True,
+    help='Python package name to add or build.',
+)
+@click.option('-v', '--version',
+    type=str,
+    default=None,
+    metavar='VERSION',
+    help='Python package version to add or build.',
+)
+@click.option('-d', '--thirdparty-dir',
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
+    metavar='DIR',
+    default=utils_thirdparty.THIRDPARTY_DIR,
+    show_default=True,
+    help='Path to the thirdparty directory where wheels are built.',
+)
+@click.option('-p', '--python-version',
+    type=click.Choice(utils_thirdparty.PYTHON_VERSIONS),
+    metavar='PYVER',
+    default=utils_thirdparty.PYTHON_VERSIONS,
+    show_default=True,
+    multiple=True,
+    help='Python version to use for this build.',
+)
+@click.option('-o', '--operating-system',
+    type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS),
+    metavar='OS',
+    default=tuple(utils_thirdparty.PLATFORMS_BY_OS),
+    multiple=True,
+    show_default=True,
+    help='OS to use for this build: one of linux, mac or windows.',
+)
+@click.option('--build-remotely',
+    is_flag=True,
+    help='Build missing wheels remotely.',
+)
+@click.option('--with-deps',
+    is_flag=True,
+    help='Also include all dependent wheels.',
+)
+@click.option('--verbose',
+    is_flag=True,
+    help='Provide verbose output.',
+)
+@click.help_option('-h', '--help')
+def build_wheels(
+    name,
+    version,
+    thirdparty_dir,
+    python_version,
+    operating_system,
+    with_deps,
+    build_remotely,
+    verbose,
+):
+    """
+    Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and
+    optional VERSION. Build wheels compatible with all the `--python-version`
+    PYVER(s) and `--operating_system` OS(s).
+
+    Build native wheels remotely if needed when `--build-remotely` and include
+    all dependencies with `--with-deps`.
+    """
+    utils_thirdparty.add_or_upgrade_built_wheels(
+        name=name,
+        version=version,
+        python_versions=python_version,
+        operating_systems=operating_system,
+        dest_dir=thirdparty_dir,
+        build_remotely=build_remotely,
+        with_deps=with_deps,
+        verbose=verbose,
+    )
+
+
+if __name__ == '__main__':
+    build_wheels()

etc/scripts/check_thirdparty.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import click
+
+import utils_thirdparty
+
+
+@click.command()
+
+@click.option('-d', '--thirdparty-dir',
+    type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
+    required=True,
+    help='Path to the thirdparty directory to check.',
+)
+@click.help_option('-h', '--help')
+def check_thirdparty_dir(thirdparty_dir):
+    """
+    Check a thirdparty directory for problems.
+    """
+    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+
+
+if __name__ == '__main__':
+    check_thirdparty_dir()