From 3b8242be193119854025e3a158fedc41bdb8d33e Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Fri, 9 Sep 2022 20:50:27 +0200 Subject: [PATCH] packages: parse_pkgbuilds: cache `makepkg --printsrcinfo` output to SRCINFO, tie to PKGBUILD checksum in srcinfo_meta.json --- constants.py | 3 ++ packages/pkgbuild.py | 111 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 22 deletions(-) diff --git a/constants.py b/constants.py index 2e9738e..e44fd48 100644 --- a/constants.py +++ b/constants.py @@ -194,3 +194,6 @@ MAKEPKG_CMD = [ '--ignorearch', '--needed', ] + +SRCINFO_FILE = 'SRCINFO' +SRCINFO_METADATA_FILE = 'srcinfo_meta.json' diff --git a/packages/pkgbuild.py b/packages/pkgbuild.py index 9b464ec..62172da 100644 --- a/packages/pkgbuild.py +++ b/packages/pkgbuild.py @@ -1,6 +1,7 @@ from __future__ import annotations import click +import json import logging import multiprocessing import os @@ -11,11 +12,12 @@ from typing import Iterable, Optional from config import config, ConfigStateHolder from constants import REPOSITORIES +from dataclass import DataClass from exec.cmd import run_cmd -from constants import Arch, MAKEPKG_CMD +from constants import Arch, MAKEPKG_CMD, SRCINFO_FILE, SRCINFO_METADATA_FILE from distro.package import PackageInfo from logger import setup_logging -from utils import git +from utils import git, sha256sum from wrapper import check_programs_wrap @@ -203,7 +205,22 @@ class SubPkgbuild(Pkgbuild): self.pkgbase.refresh_sources(lazy=lazy) -def parse_pkgbuild(relative_pkg_dir: str, _config: Optional[ConfigStateHolder] = None, sources_refreshed: bool = False) -> list[Pkgbuild]: +class SrcinfoMetaFile(DataClass): + + class SrcinfoMetaChecksums(DataClass): + PKGBUILD: str + SRCINFO: str + + checksums: SrcinfoMetaChecksums + build_mode: str + + +def parse_pkgbuild( + relative_pkg_dir: str, + _config: Optional[ConfigStateHolder] = None, + force_refresh_srcinfo: bool = False, + sources_refreshed: bool = False, +) -> list[Pkgbuild]: """ Since function may run in a different subprocess, we need to be passed the config via parameter """ @@ -215,27 +232,72 @@ def parse_pkgbuild(relative_pkg_dir: str, _config: Optional[ConfigStateHolder] = pkgbuilds_dir = config.get_path('pkgbuilds') pkgdir = os.path.join(pkgbuilds_dir, relative_pkg_dir) filename = os.path.join(pkgdir, 'PKGBUILD') - logging.debug(f"Parsing {filename}") mode = None - with open(filename, 'r') as file: - for line in file.read().split('\n'): - if line.startswith('_mode='): - mode = line.split('=')[1] - break - if mode not in ['host', 'cross']: - raise Exception((f'{relative_pkg_dir}/PKGBUILD has {"no" if mode is None else "an invalid"} mode configured') + - (f': "{mode}"' if mode is not None else '')) + srcinfo_file = os.path.join(pkgdir, SRCINFO_FILE) + srcinfo_meta_file = os.path.join(pkgdir, SRCINFO_METADATA_FILE) + refresh = force_refresh_srcinfo or not os.path.exists(srcinfo_meta_file) + if not refresh and not os.path.exists(srcinfo_meta_file): + logging.debug(f"{relative_pkg_dir}: {SRCINFO_METADATA_FILE} doesn't exist, running makepkg --printsrcinfo") + refresh = True + if not refresh: + try: + with open(srcinfo_meta_file, 'r') as meta_fd: + metadata_raw = json.load(meta_fd) + metadata = SrcinfoMetaFile.fromDict(metadata_raw, validate=True) + except Exception as ex: + logging.debug(f"{relative_pkg_dir}: something went wrong parsing json from {srcinfo_meta_file}," + f"running makepkg instead instead: {ex}") + refresh = True + # validate checksums + if not refresh: + assert metadata and metadata.checksums + for filename, checksum in metadata.checksums.items(): + file_sum = sha256sum(os.path.join(pkgdir, filename)) + if file_sum != checksum: + logging.debug(f"{relative_pkg_dir}: Checksums for {filename} don't match") + refresh = True + break + # checksums are valid! + logging.debug(f'{relative_pkg_dir}: srcinfo cache hit!') + mode = metadata.build_mode + with open(srcinfo_file, 'r') as srcinfo_fd: + lines = srcinfo_fd.read().split('\n') + + # do the actual refresh + if refresh: + logging.debug(f"Parsing {filename}") + with open(filename, 'r') as file: + for line in file.read().split('\n'): + if line.startswith('_mode='): + mode = line.split('=')[1] + break + if mode not in ['host', 'cross']: + err = 'an invalid' if mode else 'no' + raise Exception(f'{relative_pkg_dir}/PKGBUILD has {err} mode configured' + (f": {repr(mode)}" if mode is not None else "")) + + srcinfo_proc = run_cmd( + MAKEPKG_CMD + ['--printsrcinfo'], + cwd=pkgdir, + stdout=subprocess.PIPE, + ) + assert (isinstance(srcinfo_proc, subprocess.CompletedProcess)) + output = srcinfo_proc.stdout.decode('utf-8') + lines = output.split('\n') + with open(srcinfo_file, 'w') as srcinfo_fd: + srcinfo_fd.write(output) + checksums = {os.path.basename(p): sha256sum(p) for p in [filename, srcinfo_file]} + metadata = SrcinfoMetaFile.fromDict({ + 'build_mode': mode, + 'checksums': checksums, + }, validate=True) + with open(srcinfo_meta_file, 'w') as meta_fd: + json.dump(metadata, meta_fd) + + assert mode base_package = Pkgbase(relative_pkg_dir, sources_refreshed=sources_refreshed) base_package.mode = mode base_package.repo = relative_pkg_dir.split('/')[0] - srcinfo = run_cmd( - MAKEPKG_CMD + ['--printsrcinfo'], - cwd=pkgdir, - stdout=subprocess.PIPE, - ) - assert (isinstance(srcinfo, subprocess.CompletedProcess)) - lines = srcinfo.stdout.decode('utf-8').split('\n') current: Pkgbuild = base_package multi_pkgs = False @@ -286,11 +348,16 @@ _pkgbuilds_paths = dict[str, list[Pkgbuild]]() _pkgbuilds_scanned: bool = False -def get_pkgbuild_by_path(relative_path: str, lazy: bool = True, _config: Optional[ConfigStateHolder] = None) -> list[Pkgbuild]: +def get_pkgbuild_by_path( + relative_path: str, + force_refresh_srcinfo: bool = False, + lazy: bool = True, + _config: Optional[ConfigStateHolder] = None, +) -> list[Pkgbuild]: global _pkgbuilds_cache, _pkgbuilds_paths - if lazy and relative_path in _pkgbuilds_paths: + if lazy and not force_refresh_srcinfo and relative_path in _pkgbuilds_paths: return _pkgbuilds_paths[relative_path] - parsed = parse_pkgbuild(relative_path, _config=_config) + parsed = parse_pkgbuild(relative_path, force_refresh_srcinfo=force_refresh_srcinfo, _config=_config) _pkgbuilds_paths[relative_path] = parsed for pkg in parsed: _pkgbuilds_cache[pkg.name] = pkg