From 97598cd59719632eb99148bac6e6851985484151 Mon Sep 17 00:00:00 2001 From: jld3103 Date: Sun, 8 Aug 2021 18:24:31 +0200 Subject: [PATCH] Improve package discovery and add PKGBUILD formatting check --- packages.py | 183 ++++++++++++++++++++++++++++++++++++++++++----- requirements.txt | 1 + 2 files changed, 167 insertions(+), 17 deletions(-) diff --git a/packages.py b/packages.py index 997ae16..1f5cc1c 100644 --- a/packages.py +++ b/packages.py @@ -1,3 +1,4 @@ +from cmath import log from logger import * import atexit import click @@ -7,6 +8,7 @@ import os import shutil import subprocess from chroot import create_chroot +from joblib import Parallel, delayed makepkg_env = os.environ.copy() | { 'LANG': 'C', @@ -24,6 +26,7 @@ makepkg_cmd = ['makepkg', '--needed'] pacman_cmd = ['pacman', + '-Syuu', '--noconfirm', '--overwrite=*', '--needed', ] @@ -110,8 +113,7 @@ def setup_chroot(chroot_path='/chroot/root'): logging.info('Updating root chroot') result = subprocess.run(pacman_cmd + - ['-Syuu', - '--root', chroot_path, + ['--root', chroot_path, '--arch', 'aarch64', '--config', chroot_path+'/etc/pacman.conf']) if result.returncode != 0: @@ -139,7 +141,7 @@ def setup_chroot(chroot_path='/chroot/root'): exit(1) -def discover_packages() -> dict[str, Package]: +def discover_packages(path: str) -> dict[str, Package]: packages = {} paths = [] @@ -149,9 +151,9 @@ def discover_packages() -> dict[str, Package]: for dir2 in os.listdir(os.path.join('device', dir1)): paths.append(os.path.join('device', dir1, dir2)) - for path in paths: - logging.debug(f'Discovered {path}') - package = Package(path) + results = Parallel(n_jobs=multiprocessing.cpu_count()*4)( + delayed(Package)(path) for path in paths) + for package in results: packages[package.name] = package # This filters the deps to only include the ones that are provided in this repo @@ -169,6 +171,30 @@ def discover_packages() -> dict[str, Package]: if not found: package.local_depends.remove(dep) + selection = [] + for package in packages.values(): + if path == 'all' or package.path == path: + selection.append(package) + for dep in package.local_depends: + if dep in packages: + selection.append(packages[dep]) + else: + found = False + for p in packages.values(): + for name in p.names: + if dep == name: + selection.append(p) + found = True + break + if found: + break + if not found: + logging.fatal( + f'Could not find package for "{dep}"') + exit(1) + selection = list(set(selection)) + packages = {package.name: package for package in selection} + return packages @@ -246,7 +272,9 @@ def setup_dependencies_and_sources(package: Package): if package.mode == 'cross': for p in package.depends: result = subprocess.run( - pacman_cmd + ['-S', p], stderr=subprocess.DEVNULL) + pacman_cmd + [p], + stderr=subprocess.DEVNULL, + ) if result.returncode != 0: logging.fatal( f'Failed to setup dependencies for {package.path}') @@ -365,16 +393,7 @@ def cmd_build(verbose, path): check_prebuilts() - packages = discover_packages() - - if path != 'all': - selection = [] - for package in packages.values(): - if package.path == path: - # TODO: currently matches through package.name only, no provides - selection += [packages[pkg] - for pkg in package.local_depends] + [package] - packages = {package.name: package for package in selection} + packages = discover_packages(path) package_order = generate_package_order(list(packages.values())) need_build = [] @@ -412,5 +431,135 @@ def cmd_clean(verbose): exit(1) +@click.command(name='check') +@verbose_option +@click.argument('path') +def cmd_check(verbose, path): + setup_logging(verbose) + + packages = discover_packages(path) + + for name in packages: + package = packages[name] + + is_git_package = False + if name.endswith('-git'): + is_git_package = True + + mode_key = '_mode' + pkgbase_key = 'pkgbase' + pkgname_key = 'pkgname' + commit_key = '_commit' + source_key = 'source' + sha256sums_key = 'sha256sums' + required = { + mode_key: True, + pkgbase_key: False, + pkgname_key: True, + 'pkgdesc': False, + 'pkgver': True, + 'pkgrel': True, + 'arch': True, + 'license': True, + 'url': False, + 'provides': False, + 'conflicts': False, + 'depends': False, + 'optdepends': False, + 'makedepends': False, + 'install': False, + 'options': False, + commit_key: is_git_package, + source_key: False, + sha256sums_key: False, + } + + with open(os.path.join(package.path, 'PKGBUILD'), 'r') as file: + lines = file.read().split('\n') + if len(lines) == 0: + logging.fatal(f'Empty PKGBUILD for {package.path}') + exit(1) + line_index = 0 + key_index = 0 + hold_key = False + while True: + line = lines[line_index] + + if line.startswith('_') and not line.startswith(mode_key) and not line.startswith(commit_key): + line_index += 1 + continue + + formatted = True + next_key = False + next_line = False + reason = "" + + if hold_key: + next_line = True + else: + if key_index < len(required): + key = list(required)[key_index] + if line.startswith(key): + if key == pkgbase_key: + required[pkgname_key] = False + if key == source_key: + required[sha256sums_key] = True + next_key = True + next_line = True + elif key in required and not required[key]: + next_key = True + + if line.endswith('=('): + hold_key = True + if line == ')': + hold_key = False + next_key = True + + if line.startswith(' ') or line == ')': + next_line = True + + if line.startswith(' ') and not line.startswith(' '): + formatted = False + reason = 'Multiline variables should be indented with 4 spaces' + + if '"' in line and not '$' in line and not ' ' in line: + formatted = False + reason = f'Found literal " although no "$" or " " was found in the line justifying the usage of a literal "' + + if '\'' in line: + formatted = False + reason = 'Found literal \' although either a literal " or no qoutes should be used' + + if ('=(' in line and ' ' in line and not line.endswith('=(')) or (hold_key and line.endswith(')')): + formatted = False + reason = f'Multiple elements in a list need to be in separate lines' + + if formatted and not next_key and not next_line: + if key_index == len(required): + if lines[line_index] == '': + break + else: + formatted = False + reason = 'Expected final emtpy line after all variables' + else: + formatted = False + reason = f'Expected to find "{key}"' + + if not formatted: + logging.fatal( + f'Line {line_index+1} in {os.path.join(package.path, "PKGBUILD")} is not formatted correctly: "{line}"') + if reason != "": + logging.fatal(reason) + exit(1) + + if next_key and not hold_key: + key_index += 1 + if next_line: + line_index += 1 + + logging.info(f'{package.path} nicely formatted!') + + cmd_packages.add_command(cmd_build) cmd_packages.add_command(cmd_clean) +cmd_packages.add_command(cmd_check) diff --git a/requirements.txt b/requirements.txt index 70f45e4..aa5625f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ click==8.0.1 appdirs==1.4.4 +joblib==1.0.1