Improve release scripts #3040

Some release archves where missing critical wheels.
The solution was to:

1. ensure that we have effectively missing wheels published and made
available from our PyPI repo when building a release

2. Improve the fetch_thirdparty script to fall back on fetching a
source distribution when no wheel is found. Fail loudly if neither a
wheel not an sdist is found.

3. Ensure that some source-only packages are always included as sources


Signed-off-by: Philippe Ombredanne <[email protected]>

etc/scripts/fetch_thirdparty.py

@@ -12,6 +12,7 @@
 import itertools
 import os
 import sys
+from collections import defaultdict
 
 import click
 
@@ -205,10 +206,9 @@ def fetch_thirdparty(
             repos.append(repo)
 
     wheels_fetched = []
-    wheels_not_found = []
+    wheels_or_sdist_not_found = defaultdict(list)
 
     sdists_fetched = []
-    sdists_not_found = []
 
     for name, version in sorted(required_name_versions):
         nv = name, version
@@ -227,11 +227,11 @@ def fetch_thirdparty(
                 if fwfns:
                     wheels_fetched.extend(fwfns)
                 else:
-                    wheels_not_found.append(f"{name}=={version} for: {environment}")
+                    wheels_or_sdist_not_found[f"{name}=={version}"].append(environment)
                     if TRACE:
                         print(f"      NOT FOUND")
 
-        if sdists:
+        if sdists or f"{name}=={version}" in wheels_or_sdist_not_found:
             if TRACE:
                 print(f"  ==> Fetching sdist: {name}=={version}")
             fetched = utils_thirdparty.download_sdist(
@@ -243,19 +243,26 @@ def fetch_thirdparty(
             if fetched:
                 sdists_fetched.append(fetched)
             else:
-                sdists_not_found.append(f"{name}=={version}")
+                wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist")
                 if TRACE:
                     print(f"      NOT FOUND")
 
-    if wheels and wheels_not_found:
-        print(f"==> MISSING WHEELS")
-        for wh in wheels_not_found:
-            print(f"  {wh}")
+    for nv, dists in wheels_or_sdist_not_found.items():
+        print()
+        print(nv, dists)
+        print()
+        sdist_missing = "sdist" in dists
+        wheels_missing = any(d for d in dists if d != "sdist")
 
-    if sdists and sdists_not_found:
-        print(f"==> MISSING SDISTS")
-        for sd in sdists_not_found:
-            print(f"  {sd}")
+        if wheels and wheels_missing:
+            print(f"==> MISSING WHEELS: {nv}:", ", ".join(map(str, environments)))
+
+        if sdists and sdist_missing:
+            print(f"==> MISSING SDIST: {nv}")
+
+        if wheels_missing and sdist_missing:
+            print(f"==> CRITICAL ERROR: MISSING BOTH wheel and dists: {nv}")
+            raise Exception(f"==> CRITICAL ERROR: MISSING BOTH wheel and dists: {nv}")
 
     print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
     utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)

requirements-native.txt

@@ -4,3 +4,4 @@ lxml==4.9.1
 MarkupSafe==2.1.1
 pyahocorasick==2.0.0b1
 PyYAML==6.0
+libfwsi-python==20220123