diff --git a/cleanup_bconds.py b/cleanup_bconds.py new file mode 100644 index 0000000..6b5904b --- /dev/null +++ b/cleanup_bconds.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +Remove unused bcond entries from config.toml. + +For packages not in todo.pkgs: +- "just bootstrap" packages: check koji for ~bootstrap builds since 2026-06-03 +- Other packages: check distgit for Bootstrap/Rebuilt commits + +Unused bconds are removed from config.toml in place. +""" + +import argparse +import asyncio +import os +import shutil +import sys +import tempfile +import time + +import tomllib + +def log(*args, **kwargs): + kwargs.setdefault('file', sys.stderr) + kwargs.setdefault('flush', True) + return print(*args, **kwargs) + + +def load_todo_pkgs(config): + path = config['todo']['path'] + max_age_minutes = config['todo']['max_age_minutes'] + mtime = os.path.getmtime(path) + age_seconds = time.time() - mtime + if age_seconds > max_age_minutes * 60: + age_minutes = int(age_seconds / 60) + sys.exit(f'{path} is {age_minutes} minutes old (> {max_age_minutes} min). Refresh it first.') + + with open(path) as f: + return {line.strip() for line in f if line.strip()} + + +def is_just_bootstrap(entries): + for entry in entries: + if entry.get('withouts') or entry.get('replacements'): + return False + withs = entry.get('withs', []) + if list(withs) != ['bootstrap']: + return False + return True + + +def remove_bcond_blocks(content, packages_to_remove): + lines = content.splitlines(keepends=True) + result = [] + i = 0 + while i < len(lines): + stripped = lines[i].strip() + + if stripped.startswith('[[bconds.') and stripped.endswith(']]'): + key = stripped[len('[[bconds.'):-len(']]')].strip() + if key in packages_to_remove: + i += 1 + # skip key-value lines + while i < len(lines): + s = lines[i].strip() + if s and not s.startswith('#') and not s.startswith('['): + i += 1 + else: + break + # skip trailing blank lines + while i < len(lines) and lines[i].strip() == '': + i += 1 + continue + + result.append(lines[i]) + i += 1 + + return ''.join(result) + + +async def run_cmd(*cmd, cwd=None): + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + stdout, stderr = await proc.communicate() + return proc.returncode, stdout.decode(), stderr.decode() + + +async def check_distgit(pkg_name, tmpdir, config): + clone_dir = os.path.join(tmpdir, pkg_name) + clone_depth = config['distgit']['clone_depth'] + clone_retries = config['distgit']['clone_retries'] + + for attempt in range(1, clone_retries + 1): + rc, stdout, stderr = await run_cmd( + 'fedpkg', 'clone', '-a', pkg_name, clone_dir, '--', f'--depth={clone_depth}' + ) + if rc == 0: + break + log(f' WARNING: fedpkg clone failed for {pkg_name} (attempt {attempt}/{clone_retries}): {stderr.strip()}') + if attempt < clone_retries: + await asyncio.sleep(2 ** attempt) + if os.path.isdir(clone_dir): + shutil.rmtree(clone_dir) + else: + return False + + rc, stdout, stderr = await run_cmd( + 'git', '-C', clone_dir, 'log', + f'--author={config["distgit"]["author"]}', '--format=%s', '--all', + ) + if rc != 0: + log(f' WARNING: git log failed for {pkg_name}: {stderr.strip()}') + return False + + bootstrap_msg = config['distgit']['bootstrap_commit_message'] + rebuild_msg = config['distgit']['commit_message'] + has_bootstrap = False + has_rebuild = False + for subject in stdout.splitlines(): + if subject == bootstrap_msg: + has_bootstrap = True + elif subject == rebuild_msg: + has_rebuild = True + + # Unused: no bootstrap commit but was rebuilt directly + return not has_bootstrap and has_rebuild + + +async def check_koji(pkg_name, config): + builds_after = config['koji']['builds_after'] + bootstrap_marker = config['koji']['bootstrap_marker'] + rc, stdout, stderr = await run_cmd( + 'koji', 'list-builds', f'--package={pkg_name}', f'--after={builds_after}' + ) + if rc != 0: + log(f' WARNING: koji list-builds failed for {pkg_name}: {stderr.strip()}') + return False + + for line in stdout.splitlines(): + if bootstrap_marker in line: + return False # bootstrap build exists, was used + + return True # no bootstrap builds found, unused + + +async def check_package(pkg_name, entries, tmpdir, config, sem): + async with sem: + just_bootstrap = is_just_bootstrap(entries) + + if just_bootstrap: + log(f' {pkg_name}: checking koji for ~bootstrap builds...') + unused = await check_koji(pkg_name, config) + else: + log(f' {pkg_name}: checking distgit for Bootstrap/Rebuilt commits...') + unused = await check_distgit(pkg_name, tmpdir, config) + + status = 'UNUSED' if unused else 'used' + log(f' {pkg_name}: {status}') + return pkg_name, unused + + +async def main(): + parser = argparse.ArgumentParser(description='Remove unused bconds from config.toml') + parser.add_argument( + '-j', '--jobs', + type=int, + default=os.cpu_count(), + help=f'Max parallel tasks (default: {os.cpu_count()})', + ) + args = parser.parse_args() + + with open('config.toml', 'rb') as f: + config = tomllib.load(f) + + todo_pkgs = load_todo_pkgs(config) + log(f'Loaded {len(todo_pkgs)} packages from {config["todo"]["path"]}') + + bconds = config['bconds'] + + to_check = {} + skipped_todo = 0 + for pkg_name in bconds: + if pkg_name.strip() in todo_pkgs: + skipped_todo += 1 + else: + to_check[pkg_name] = bconds[pkg_name] + + log(f'Skipping {skipped_todo} packages in todo.pkgs') + log(f'Checking {len(to_check)} packages with {args.jobs} parallel jobs...') + + sem = asyncio.Semaphore(args.jobs) + to_remove = set() + + def collect_result(task): + try: + pkg_name, unused = task.result() + except (asyncio.CancelledError, Exception): + return + if unused: + to_remove.add(pkg_name) + + try: + with tempfile.TemporaryDirectory(prefix='cleanup_bconds_') as tmpdir: + tasks = [] + for pkg_name, entries in to_check.items(): + task = asyncio.ensure_future(check_package( + pkg_name.strip(), entries, + tmpdir, config, sem, + )) + task.add_done_callback(collect_result) + tasks.append(task) + await asyncio.gather(*tasks) + except (KeyboardInterrupt, asyncio.CancelledError): + log(f'\nInterrupted. {len(to_remove)} unused bconds found so far.') + finally: + if not to_remove: + log('No unused bconds found.') + return + + log(f'\nRemoving {len(to_remove)} unused bcond entries:') + for pkg_name in sorted(to_remove): + log(f' - {pkg_name}') + + with open('config.toml') as f: + content = f.read() + content = remove_bcond_blocks(content, to_remove) + with open('config.toml', 'w') as f: + f.write(content) + + log(f'\nDone. Removed {len(to_remove)} entries from config.toml.') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/config.toml b/config.toml index db08449..a08de2d 100644 --- a/config.toml +++ b/config.toml @@ -35,12 +35,20 @@ target = 'rawhide' # target = 'f45-python' bootstrap_target = 'f45-python-bootstrap' +builds_after = '2026-06-03' +bootstrap_marker = '~bootstrap' [distgit] branch = "rawhide" commit_message = "Rebuilt for Python 3.15" bootstrap_commit_message = "Bootstrap for Python 3.15" author = "Python Maint " +clone_depth = 10 +clone_retries = 3 + +[todo] +path = "todo.pkgs" +max_age_minutes = 60 [architectures] repoquery = "x86_64" # used for repository querying @@ -96,12 +104,6 @@ withouts = ["check"] [[bconds.python-setuptools_scm]] withouts = ["tests"] -[[bconds.python-py]] -withouts = ["docs", "tests"] - -[[bconds.python-chardet]] -withouts = ["doc"] - [[bconds.python-pbr]] withs = ["bootstrap"] withouts = ["tests"] @@ -163,9 +165,6 @@ withs = ["bootstrap"] [[bconds.python-jinja2]] withouts = ["docs", "asyncio_tests"] -[[bconds.python-sphinx_rtd_theme]] -withs = ["bootstrap"] - [[bconds.python-sphinx-basic-ng]] withs = ["bootstrap"] @@ -202,21 +201,12 @@ withouts = ["tests"] [[bconds.python-flit-core]] withs = ["bootstrap"] -[[bconds.python-async-timeout]] -withouts = ["tests"] - -[[bconds.python-jupyter-client]] -withs = ["bootstrap"] - [[bconds.python-jupyter-server]] withs = ["bootstrap"] [[bconds.python-matplotlib]] withouts = ["check"] -[[bconds.ipython]] -withouts = ["check","doc"] - [[bconds.python-ipykernel]] withs = ["bootstrap"] @@ -226,24 +216,12 @@ withouts = ["optional_tests"] [[bconds.python-nbconvert]] withouts = ["check", "doc"] -[[bconds.python-nbclient]] -withs = ["bootstrap"] - -[[bconds.python-pyquery]] -withouts = ["tests"] - -[[bconds.python-cherrypy]] -withouts = ["tests"] - [[bconds.freeipa-healthcheck]] withouts = ["tests"] [[bconds.python-Traits]] withs = ["bootstrap"] -[[bconds.python-pcodedmp]] -withs = ["bootstrap"] - [[bconds.python-libcst]] withs = ["bootstrap"] @@ -278,15 +256,6 @@ withs = ["bootstrap"] [[bconds.python-astropy]] withouts = ["check"] -[[bconds.python-pyerfa]] -withouts = ["tests"] - -[[bconds.python-aiohttp]] -withouts = ["tests"] - -[[bconds.python-azure-core]] -withouts = ["tests"] - # this bcond is disabled in spec on purpose # it might be enabled in the future, so leaving this here, commend out #[[bconds.python-azure-common]] @@ -295,9 +264,6 @@ withouts = ["tests"] [[bconds.python-subprocess-tee]] withouts = ["tests"] -[[bconds.python-typeguard]] -withs = ["bootstrap"] - [[bconds.python-tox]] withs = ["bootstrap"] @@ -310,24 +276,15 @@ withs = ["bootstrap"] [[bconds.python-contourpy]] withs = ["bootstrap"] -[[bconds.python-msrest]] -withouts = ["tests"] - [[bconds.python-oletools]] withs = ["bootstrap"] -[[bconds.python-google-api-core]] -withouts = ["tests"] - [[bconds.fonttools]] withouts = ["tests", "plot_extra", "symfont_extra", "ufo_extra", "woff_extra", "graphite_extra", "interpolatable_extra"] [[bconds.conda]] withs = ["bootstrap"] -[[bconds.python-conda-libmamba-solver]] -withouts = ["tests"] - [[bconds.python-conda-index]] withs = ["bootstrap"] @@ -361,18 +318,12 @@ withs = ["bootstrap"] [[bconds.python-pysocks]] replacements = {with_python3_tests = "0"} -[[bconds.ara]] -replacements = {with_docs = "0"} - [[bconds.python-httpretty]] replacements = {run_tests = "0"} [[bconds.python-ruamel-yaml]] withs = ["bootstrap"] -[[bconds.python-rq]] -withouts = ["tests"] - [[bconds.python-markdown-it-py]] withouts = ["plugins"] @@ -388,9 +339,6 @@ withouts = ["docs", "tests"] [[bconds.python-executing]] withs = ["bootstrap"] -[[bconds.python-editables]] -withouts = ["doc"] - [[bconds.python-jsonschema-specifications]] replacements = {with_doc = "0"} @@ -400,12 +348,6 @@ withs = ["bootstrap"] [[bconds.python-oslo-config]] withs = ["bootstrap"] -[[bconds.python-neutronclient]] -replacements = {with_doc = "0"} - -[[bconds.python-glanceclient]] -replacements = {with_doc = "0"} - [[bconds.python-domdf-python-tools]] withouts = ["tests"] @@ -460,9 +402,6 @@ withouts = ["check"] [[bconds.python-tabulate]] withs = ["bootstrap"] -[[bconds.python-repoze-sphinx-autointerface]] -withouts = ["tests"] - [[bconds.python-zope-exceptions]] withouts = ["tests"] @@ -487,15 +426,6 @@ withouts = ["tests"] [[bconds.python-humanfriendly]] withs = ["bootstrap"] -[[bconds.python-cascadio]] -withs = ["bootstrap"] - -[[bconds.python-pint]] -withs = ["bootstrap"] - -[[bconds.python-qcengine]] -withouts = ["tests"] - [[bconds.python-astropy-iers-data]] withouts = ["tests"] @@ -511,9 +441,6 @@ withs = ["bootstrap"] [[bconds.python-pydantic-core]] withouts = ["inline_snapshot_tests"] -[[bconds.python-optking]] -withouts = ["tests"] - [[bconds.python-django5]] withouts = ["tests"] @@ -544,9 +471,6 @@ withs = ["bootstrap"] [[bconds.python-mkdocs-material-extensions]] withouts = ["tests"] -[[bconds.python-zope-testing]] -withouts = ["tests"] - [[bconds.python-xnat]] withs = ["bootstrap"] @@ -556,9 +480,6 @@ withs = ["bootstrap"] [[bconds.llvm]] withouts = ["pgo"] -[[bconds.llvm21 ]] -withouts = ["pgo"] - [[bconds.python-pathspec]] withouts = ["tests"] @@ -622,9 +543,6 @@ withouts = ["tests"] [[bconds.pyproj]] withouts = ["xarray"] -[[bconds.python-distlib]] -withouts = ["check"] - [[bconds.python-flit]] withouts = ["tests"] @@ -655,9 +573,6 @@ withs = ["bootstrap"] [[bconds.python-versioningit]] withouts = ["pydantic_tests"] -[[bconds.python-scikit-image]] -withouts = ["check"] - [[bconds.python-semantic_version]] withs = ["bootstrap"] @@ -667,9 +582,6 @@ withs = ["bootstrap"] [[bconds.python-trimesh]] withs = ["bootstrap"] -[[bconds.python-scikit-image]] -withs = ["bootstrap"] - [[bconds.python-xarray]] withs = ["bootstrap"]