diff --git a/integration_tests.py b/integration_tests.py index 556a4f3..2edeb68 100644 --- a/integration_tests.py +++ b/integration_tests.py @@ -10,13 +10,16 @@ from constants import SRCINFO_METADATA_FILE from exec.cmd import run_cmd from exec.file import get_temp_dir from logger import setup_logging -from packages.cli import cmd_build, cmd_clean, cmd_update +from packages.cli import SRCINFO_CACHE_FILES, cmd_build, cmd_clean, cmd_update from utils import git_get_branch tempdir = None config.try_load_file() setup_logging(True) +PKG_TEST_PATH = 'device/device-sdm845-oneplus-enchilada' +PKG_TEST_NAME = 'device-sdm845-xiaomi-beryllium-ebbg' + @pytest.fixture() def ctx() -> click.Context: @@ -54,7 +57,7 @@ def test_packages_update(ctx: click.Context): for branch, may_fail in branches.items(): config.file.pkgbuilds.git_branch = branch try: - ctx.invoke(cmd_update, non_interactive=True, switch_branch=True, discard_changes=True) + ctx.invoke(cmd_update, non_interactive=True, switch_branch=True, discard_changes=True, init_caches=False) except Exception as ex: print(f'may_fail: {may_fail}; Exception: {ex}') if not may_fail: @@ -73,15 +76,20 @@ def test_packages_clean(ctx: click.Context): ctx.invoke(cmd_clean, what=['git'], force=True) -def build_pkgs(_ctx: click.Context, query: list[str], arch: str = 'aarch64'): - _ctx.invoke(cmd_build, paths=query, arch=arch) +def test_packages_cache_init(ctx: click.Context): + ctx.invoke(cmd_update, non_interactive=True, switch_branch=False, discard_changes=False, init_caches=True) + + for f in SRCINFO_CACHE_FILES: + assert os.path.exists(os.path.join(config.get_path('pkgbuilds'), PKG_TEST_PATH, f)) + + +def build_pkgs(_ctx: click.Context, query: list[str], arch: str = 'aarch64', **kwargs): + _ctx.invoke(cmd_build, paths=query, arch=arch, **kwargs) def test_packages_build_by_path(ctx: click.Context): - name = 'device/device-sdm845-oneplus-enchilada' - build_pkgs(ctx, [name]) + build_pkgs(ctx, [PKG_TEST_PATH], force=True) def test_split_package_build_by_name(ctx: click.Context): - name = 'device-sdm845-xiaomi-beryllium-ebbg' - build_pkgs(ctx, [name]) + build_pkgs(ctx, [PKG_TEST_NAME]) diff --git a/packages/cli.py b/packages/cli.py index e6f0a18..fca39a2 100644 --- a/packages/cli.py +++ b/packages/cli.py @@ -1,4 +1,5 @@ import click +import json import logging import os @@ -6,17 +7,18 @@ from glob import glob from typing import Iterable, Optional from config.state import config -from constants import Arch, ARCHES, REPOSITORIES, SRCINFO_INITIALISED_FILE -from exec.file import remove_file +from constants import Arch, ARCHES, REPOSITORIES, SRCINFO_FILE, SRCINFO_INITIALISED_FILE, SRCINFO_METADATA_FILE, SRCINFO_TARBALL_FILE, SRCINFO_TARBALL_URL +from exec.cmd import run_cmd, shell_quote, CompletedProcess +from exec.file import get_temp_dir, makedir, remove_file from devices.device import get_profile_device -from distro.distro import get_kupfer_local +from distro.distro import get_kupfer_local, get_kupfer_url from distro.package import LocalPackage from net.ssh import run_ssh_command, scp_put_files -from utils import git +from utils import download_file, git, sha256sum from wrapper import check_programs_wrap, enforce_wrap from .build import build_packages_by_paths -from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, init_pkgbuilds +from .pkgbuild import discover_pkgbuilds, filter_pkgbuilds, get_pkgbuild_dirs, init_pkgbuilds def build( @@ -46,31 +48,127 @@ def build( ) +def init_pkgbuild_caches(clean_src_dirs: bool = True, remote_branch: Optional[str] = None): + + def read_srcinitialised_checksum(src_initialised): + with open(src_initialised) as fd: + d = json.load(fd) + if isinstance(d, dict): + return d.get('PKGBUILD', '!!!ERROR!!!') + raise Exception("JSON content not a dictionary!") + + # get_kupfer_url() resolves repo branch variable in url + url = get_kupfer_url(url=SRCINFO_TARBALL_URL, branch=remote_branch) + cachetar = os.path.join(config.get_path('packages'), SRCINFO_TARBALL_FILE) + makedir(os.path.dirname(cachetar)) + logging.info(f"Updating PKGBUILD caches from {url}" + (", pruning outdated src/ directories" if clean_src_dirs else "")) + updated = download_file(cachetar, url) + logging.info("Cache tarball was " + ('downloaded successfully' if updated else 'already up to date')) + tmpdir = get_temp_dir() + logging.debug(f"Extracting {cachetar} to {tmpdir}") + res = run_cmd(['tar', 'xf', cachetar], cwd=tmpdir) + assert isinstance(res, CompletedProcess) + if res.returncode: + raise Exception(f"failed to extract srcinfo cache archive '{cachetar}'") + pkgbuild_dirs = get_pkgbuild_dirs() + for pkg in pkgbuild_dirs: + logging.info(f"{pkg}: analyzing cache") + pkgdir = os.path.join(config.get_path('pkgbuilds'), pkg) + srcdir = os.path.join(pkgdir, 'src') + src_initialised = os.path.join(pkgdir, SRCINFO_INITIALISED_FILE) + cachedir = os.path.join(tmpdir, pkg) + pkgbuild_checksum = sha256sum(os.path.join(pkgdir, 'PKGBUILD')) + copy_files: set[str] = {SRCINFO_FILE, SRCINFO_INITIALISED_FILE, SRCINFO_METADATA_FILE} + if os.path.exists(src_initialised): + try: + if read_srcinitialised_checksum(src_initialised) == pkgbuild_checksum: + copy_files.remove(SRCINFO_INITIALISED_FILE) + for f in copy_files.copy(): + fpath = os.path.join(pkgdir, f) + if os.path.exists(fpath): + copy_files.remove(f) + if not copy_files: + logging.info(f"{pkg}: SRCINFO cache already up to date") + continue + except Exception as ex: + logging.warning(f"{pkg}: Something went wrong parsing {SRCINFO_INITIALISED_FILE}, treating as outdated!:\n{ex}") + if clean_src_dirs and os.path.exists(srcdir): + logging.info(f"{pkg}: outdated src/ detected, removing") + remove_file(srcdir, recursive=True) + remove_file(src_initialised) + if not os.path.exists(cachedir): + logging.info(f"{pkg}: not found in remote repo cache, skipping") + continue + cache_initialised = os.path.join(cachedir, SRCINFO_INITIALISED_FILE) + try: + if read_srcinitialised_checksum(cache_initialised) != pkgbuild_checksum: + logging.info(f"{pkg}: PKGBUILD checksum differs from remote repo cache, skipping") + continue + except Exception as ex: + logging.warning(f"{pkg}: Failed to parse the remote repo's cached {SRCINFO_INITIALISED_FILE}, skipping!:\n{ex}") + continue + if not copy_files: + continue + logging.info(f"{pkg}: Copying srcinfo cache from remote repo") + logging.debug(f'{pkg}: copying {copy_files}') + copy_files_list = [shell_quote(os.path.join(cachedir, f)) for f in copy_files] + res = run_cmd(f"cp {' '.join(copy_files_list)} {shell_quote(pkgdir)}/") + assert isinstance(res, CompletedProcess) + if res.returncode: + raise Exception(f"{pkg}: failed to copy cache contents from {cachedir}") + + +non_interactive_flag = click.option('--non-interactive', is_flag=True) +init_caches_flag = click.option( + '--init-caches/--no-init-caches', + is_flag=True, + default=True, + show_default=True, + help="Fill PKGBUILDs caches from HTTPS repo where checksums match", +) +remove_outdated_src_flag = click.option( + '--clean-src-dirs/--no-clean-src-dirs', + is_flag=True, + default=True, + show_default=True, + help="Remove outdated src/ directories to avoid problems", +) + + @click.group(name='packages') def cmd_packages(): """Build and manage packages and PKGBUILDs""" -non_interactive_flag = click.option('--non-interactive', is_flag=True) - - @cmd_packages.command(name='update') @non_interactive_flag -@click.option('--switch-branch', is_flag=True, help="Force the branch to be corrected even in non-interactive mode") -@click.option('--discard-changes', is_flag=True, help="When switching branches, discard any locally changed conflicting files") -def cmd_update(non_interactive: bool = False, switch_branch: bool = False, discard_changes: bool = False): +@init_caches_flag +@remove_outdated_src_flag +def cmd_update( + non_interactive: bool = False, + init_caches: bool = False, + clean_src_dirs: bool = True, + switch_branch: bool = False, + discard_changes: bool = False, +): """Update PKGBUILDs git repo""" init_pkgbuilds(interactive=not non_interactive, lazy=False, update=True, switch_branch=switch_branch, discard_changes=discard_changes) - logging.info("Refreshing SRCINFO caches") + if init_caches: + init_pkgbuild_caches(clean_src_dirs=clean_src_dirs) + logging.info("Refreshing outdated SRCINFO caches") discover_pkgbuilds(lazy=False) @cmd_packages.command(name='init') @non_interactive_flag +@init_caches_flag +@remove_outdated_src_flag @click.option('-u', '--update', is_flag=True, help='Use git pull to update the PKGBUILDs') -def cmd_init(non_interactive: bool = False, update: bool = False): +def cmd_init(non_interactive: bool = False, init_caches: bool = True, clean_src_dirs: bool = True, update: bool = False): "Ensure PKGBUILDs git repo is checked out locally" init_pkgbuilds(interactive=not non_interactive, lazy=False, update=update, switch_branch=False) + if init_caches: + init_pkgbuild_caches(clean_src_dirs=clean_src_dirs) @cmd_packages.command(name='build')