From e001d107c2299010c010d50525d6964074558b40 Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 19 Aug 2022 17:11:56 +0200 Subject: [PATCH] move packages.{clone_pkbuilds(),discover_packages(),init_pkgbuilds()} to packages.pkgbuild also rename discover_packages() to discover_pkgbuilds() --- image.py | 4 +- packages/__init__.py | 102 +++---------------------------------------- packages/pkgbuild.py | 99 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 100 deletions(-) diff --git a/image.py b/image.py index 7f0fd75..72df728 100644 --- a/image.py +++ b/image.py @@ -15,7 +15,7 @@ from config import config, Profile from distro.distro import get_base_distro, get_kupfer_https from exec.cmd import run_root_cmd, generate_cmd_su from exec.file import root_write_file, root_makedir, makedir -from packages import build_enable_qemu_binfmt, discover_packages, build_packages +from packages import build_enable_qemu_binfmt, discover_pkgbuilds, build_packages from ssh import copy_ssh_keys from wrapper import enforce_wrap @@ -400,7 +400,7 @@ def cmd_build(profile_name: str = None, if local_repos and build_pkgs: logging.info("Making sure all packages are built") - repo = discover_packages() + repo = discover_pkgbuilds() build_packages(repo, [p for name, p in repo.items() if name in packages], arch, try_download=not no_download_pkgs) image_path = block_target or get_image_path(device, flavour) diff --git a/packages/__init__.py b/packages/__init__.py index cef5f08..1551159 100644 --- a/packages/__init__.py +++ b/packages/__init__.py @@ -5,7 +5,6 @@ import os import shutil import subprocess from copy import deepcopy -from joblib import Parallel, delayed from glob import glob from urllib.error import HTTPError from urllib.request import urlopen @@ -23,7 +22,7 @@ from ssh import run_ssh_command, scp_put_files from wrapper import enforce_wrap from utils import git -from .pkgbuild import Pkgbuild, parse_pkgbuild +from .pkgbuild import discover_pkgbuilds, init_pkgbuilds, Pkgbuild pacman_cmd = [ 'pacman', @@ -45,38 +44,6 @@ def get_makepkg_env(): } -def clone_pkbuilds(pkgbuilds_dir: str, repo_url: str, branch: str, interactive=False, update=True): - git_dir = os.path.join(pkgbuilds_dir, '.git') - if not os.path.exists(git_dir): - logging.info('Cloning branch {branch} from {repo}') - result = git(['clone', '-b', branch, repo_url, pkgbuilds_dir]) - if result.returncode != 0: - raise Exception('Error cloning pkgbuilds') - else: - result = git(['--git-dir', git_dir, 'branch', '--show-current'], capture_output=True) - current_branch = result.stdout.decode().strip() - if current_branch != branch: - logging.warning(f'pkgbuilds repository is on the wrong branch: {current_branch}, requested: {branch}') - if interactive and click.confirm('Would you like to switch branches?', default=False): - result = git(['switch', branch], dir=pkgbuilds_dir) - if result.returncode != 0: - raise Exception('failed switching branches') - if update: - if interactive: - if not click.confirm('Would you like to try updating the PKGBUILDs repo?'): - return - result = git(['pull'], pkgbuilds_dir) - if result.returncode != 0: - raise Exception('failed to update pkgbuilds') - - -def init_pkgbuilds(interactive=False): - pkgbuilds_dir = config.get_path('pkgbuilds') - repo_url = config.file['pkgbuilds']['git_repo'] - branch = config.file['pkgbuilds']['git_branch'] - clone_pkbuilds(pkgbuilds_dir, repo_url, branch, interactive=interactive, update=False) - - def init_prebuilts(arch: Arch, dir: str = None): """Ensure that all `constants.REPOSITORIES` inside `dir` exist""" prebuilts_dir = dir or config.get_package_dir(arch) @@ -104,65 +71,6 @@ def init_prebuilts(arch: Arch, dir: str = None): raise Exception(f'Failed to create local repo {repo}') -_pkgbuilds_cache = dict[str, Pkgbuild]() -_pkgbuilds_scanned: bool = False - - -def discover_packages(parallel: bool = True, lazy: bool = True) -> dict[str, Pkgbuild]: - global _pkgbuilds_cache, _pkgbuilds_scanned - if lazy and _pkgbuilds_scanned: - logging.debug("Reusing cached pkgbuilds repo") - return _pkgbuilds_cache.copy() - pkgbuilds_dir = config.get_path('pkgbuilds') - packages: dict[str, Pkgbuild] = {} - paths = [] - init_pkgbuilds(interactive=False) - for repo in REPOSITORIES: - for dir in os.listdir(os.path.join(pkgbuilds_dir, repo)): - paths.append(os.path.join(repo, dir)) - - results = [] - - logging.info("Parsing PKGBUILDs") - - logging.debug(f"About to parse pkgbuilds. verbosity: {config.runtime['verbose']}") - if parallel: - chunks = (Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(parse_pkgbuild)(path, config) for path in paths)) - else: - chunks = (parse_pkgbuild(path, config) for path in paths) - - for pkglist in chunks: - results += pkglist - - logging.debug('Building package dictionary!') - for package in results: - for name in [package.name] + package.replaces: - if name in packages: - logging.warning(f'Overriding {packages[package.name]} with {package}') - packages[name] = package - - # This filters the deps to only include the ones that are provided in this repo - for package in packages.values(): - package.local_depends = package.depends.copy() - for dep in package.depends.copy(): - found = dep in packages - for p in packages.values(): - if found: - break - if dep in p.names(): - logging.debug(f'Found {p.name} that provides {dep}') - found = True - break - if not found: - logging.debug(f'Removing {dep} from dependencies') - package.local_depends.remove(dep) - - _pkgbuilds_cache.clear() - _pkgbuilds_cache.update(packages) - _pkgbuilds_scanned = True - return packages - - def filter_packages(repo: dict[str, Pkgbuild], paths: Iterable[str], allow_empty_results=True, use_paths=True, use_names=True) -> Iterable[Pkgbuild]: if 'all' in paths: return list(repo.values()) @@ -718,7 +626,7 @@ def build_enable_qemu_binfmt(arch: Arch, repo: dict[str, Pkgbuild] = None): return enforce_wrap() if not repo: - repo = discover_packages() + repo = discover_pkgbuilds() # build qemu-user, binfmt, crossdirect build_packages_by_paths( @@ -784,7 +692,7 @@ def build( raise Exception(f'Unknown architecture "{arch}". Choices: {", ".join(ARCHES)}') enforce_wrap() config.enforce_config_loaded() - repo: dict[str, Pkgbuild] = discover_packages() + repo: dict[str, Pkgbuild] = discover_pkgbuilds() if arch != config.runtime['arch']: build_enable_qemu_binfmt(arch, repo=repo) @@ -884,7 +792,7 @@ def cmd_clean(what: Iterable[str] = ['all'], force: bool = False, noop: bool = F def cmd_list(): enforce_wrap() logging.info('Discovering packages.') - packages = discover_packages() + packages = discover_pkgbuilds() logging.info(f'Done! {len(packages)} Pkgbuilds:') for p in set(packages.values()): print( @@ -906,7 +814,7 @@ def cmd_check(paths): return False paths = list(paths) - packages = filter_packages(discover_packages(), paths, allow_empty_results=False) + packages = filter_packages(discover_pkgbuilds(), paths, allow_empty_results=False) for package in packages: name = package.name diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index b271a97..d9bcf0a 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -1,15 +1,53 @@ from __future__ import annotations +import click +import logging +import multiprocessing import os import subprocess +from constants import REPOSITORIES +from joblib import Parallel, delayed from typing import Optional, Sequence from config import config, ConfigStateHolder from exec.cmd import run_cmd from constants import Arch, MAKEPKG_CMD from distro.package import PackageInfo -from logger import logging, setup_logging +from logger import setup_logging +from utils import git + + +def clone_pkbuilds(pkgbuilds_dir: str, repo_url: str, branch: str, interactive=False, update=True): + git_dir = os.path.join(pkgbuilds_dir, '.git') + if not os.path.exists(git_dir): + logging.info('Cloning branch {branch} from {repo}') + result = git(['clone', '-b', branch, repo_url, pkgbuilds_dir]) + if result.returncode != 0: + raise Exception('Error cloning pkgbuilds') + else: + result = git(['--git-dir', git_dir, 'branch', '--show-current'], capture_output=True) + current_branch = result.stdout.decode().strip() + if current_branch != branch: + logging.warning(f'pkgbuilds repository is on the wrong branch: {current_branch}, requested: {branch}') + if interactive and click.confirm('Would you like to switch branches?', default=False): + result = git(['switch', branch], dir=pkgbuilds_dir) + if result.returncode != 0: + raise Exception('failed switching branches') + if update: + if interactive: + if not click.confirm('Would you like to try updating the PKGBUILDs repo?'): + return + result = git(['pull'], pkgbuilds_dir) + if result.returncode != 0: + raise Exception('failed to update pkgbuilds') + + +def init_pkgbuilds(interactive=False): + pkgbuilds_dir = config.get_path('pkgbuilds') + repo_url = config.file['pkgbuilds']['git_repo'] + branch = config.file['pkgbuilds']['git_branch'] + clone_pkbuilds(pkgbuilds_dir, repo_url, branch, interactive=interactive, update=False) class Pkgbuild(PackageInfo): @@ -171,3 +209,62 @@ def parse_pkgbuild(relative_pkg_dir: str, _config: Optional[ConfigStateHolder] = if not (pkg.version == base_package.version): raise Exception(f'Subpackage malformed! Versions differ! base: {base_package}, subpackage: {pkg}') return results + + +_pkgbuilds_cache = dict[str, Pkgbuild]() +_pkgbuilds_scanned: bool = False + + +def discover_pkgbuilds(parallel: bool = True, lazy: bool = True) -> dict[str, Pkgbuild]: + global _pkgbuilds_cache, _pkgbuilds_scanned + if lazy and _pkgbuilds_scanned: + logging.debug("Reusing cached pkgbuilds repo") + return _pkgbuilds_cache.copy() + pkgbuilds_dir = config.get_path('pkgbuilds') + packages: dict[str, Pkgbuild] = {} + paths = [] + init_pkgbuilds(interactive=False) + for repo in REPOSITORIES: + for dir in os.listdir(os.path.join(pkgbuilds_dir, repo)): + paths.append(os.path.join(repo, dir)) + + results = [] + + logging.info("Parsing PKGBUILDs") + + logging.debug(f"About to parse pkgbuilds. verbosity: {config.runtime['verbose']}") + if parallel: + chunks = (Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(parse_pkgbuild)(path, config) for path in paths)) + else: + chunks = (parse_pkgbuild(path) for path in paths) + + for pkglist in chunks: + results += pkglist + + logging.debug('Building package dictionary!') + for package in results: + for name in [package.name] + package.replaces: + if name in packages: + logging.warning(f'Overriding {packages[package.name]} with {package}') + packages[name] = package + + # This filters the deps to only include the ones that are provided in this repo + for package in packages.values(): + package.local_depends = package.depends.copy() + for dep in package.depends.copy(): + found = dep in packages + for p in packages.values(): + if found: + break + if dep in p.names(): + logging.debug(f'Found {p.name} that provides {dep}') + found = True + break + if not found: + logging.debug(f'Removing {dep} from dependencies') + package.local_depends.remove(dep) + + _pkgbuilds_cache.clear() + _pkgbuilds_cache.update(packages) + _pkgbuilds_scanned = True + return packages