packages/pkgbuild: cache parsed pkgbuilds by path, add get_pkgbuild_by_path(), Pkgbuild.update(pkgb)

packages/device.py

@@ -6,7 +6,7 @@ from typing import Optional
 from config import config
 from constants import Arch, ARCHES
 from config.scheme import DataClass, munchclass
-from .pkgbuild import discover_pkgbuilds, _pkgbuilds_cache, Pkgbuild, parse_pkgbuild
+from .pkgbuild import discover_pkgbuilds, get_pkgbuild_by_path, _pkgbuilds_cache, Pkgbuild
 
 DEVICE_DEPRECATIONS = {
     "oneplus-enchilada": "sdm845-oneplus-enchilada",
@@ -105,8 +105,7 @@ def get_device(name: str, pkgbuilds: Optional[dict[str, Pkgbuild]] = None, lazy:
             else:
                 relative_path = os.path.join('device', pkgname)
                 assert os.path.exists(os.path.join(config.get_path('pkgbuilds'), relative_path))
-                pkgbuild = [p for p in parse_pkgbuild(relative_path, _config=config) if p.name == pkgname][0]
+                pkgbuild = [p for p in get_pkgbuild_by_path(relative_path, lazy=lazy, _config=config) if p.name == pkgname][0]
-                _pkgbuilds_cache[pkgname] = pkgbuild
         device = parse_device_pkg(pkgbuild)
         if lazy:
             _device_cache[name] = device

packages/pkgbuild.py

@@ -6,11 +6,11 @@ import multiprocessing
 import os
 import subprocess
 
-from constants import REPOSITORIES
 from joblib import Parallel, delayed
 from typing import Optional, Sequence
 
 from config import config, ConfigStateHolder
+from constants import REPOSITORIES
 from exec.cmd import run_cmd
 from constants import Arch, MAKEPKG_CMD
 from distro.package import PackageInfo
@@ -102,6 +102,19 @@ class Pkgbuild(PackageInfo):
         """updates `self.version` from `self.pkgver` and `self.pkgrel`"""
         self.version = f'{self.pkgver}-{self.pkgrel}'
 
+    def update(self, pkg: Pkgbuild):
+        self.version = pkg.version
+        self.arches = list(pkg.arches)
+        self.depends = list(pkg.depends)
+        self.provides = list(pkg.provides)
+        self.replaces = list(pkg.replaces)
+        self.local_depends = list(pkg.local_depends)
+        self.repo = pkg.repo
+        self.mode = pkg.mode
+        self.path = pkg.path
+        self.pkgver = pkg.pkgver
+        self.pkgrel = pkg.pkgrel
+        self.update_version()
 
 class Pkgbase(Pkgbuild):
     subpackages: Sequence[SubPkgbuild]
@@ -110,6 +123,21 @@ class Pkgbase(Pkgbuild):
         self.subpackages = list(subpackages)
         super().__init__(relative_path, **args)
 
+    def update(self, pkg: Pkgbuild):
+        if not isinstance(pkg, Pkgbase):
+            raise Exception(f"Tried to update pkgbase {self.name} with non-base pkg {pkg}")
+        Pkgbuild.update(self, pkg)
+        sub_dict = {p.name: p for p in self.subpackages}
+        self.subpackages.clear()
+        for new_pkg in pkg.subpackages:
+            name = new_pkg.name
+            if name not in sub_dict:
+                sub_dict[name] = new_pkg
+            else:
+                sub_dict[name].update(new_pkg)
+            updated = sub_dict[name]
+            self.subpackages.append(updated)
+
 
 class SubPkgbuild(Pkgbuild):
     pkgbase: Pkgbase
@@ -119,18 +147,10 @@ class SubPkgbuild(Pkgbuild):
         self.name = name
         self.pkgbase = pkgbase
 
-        self.version = pkgbase.version
+        self.update(pkgbase)
-        self.arches = pkgbase.arches
+
-        self.depends = list(pkgbase.depends)
         self.provides = []
         self.replaces = []
-        self.local_depends = list(pkgbase.local_depends)
-        self.repo = pkgbase.repo
-        self.mode = pkgbase.mode
-        self.path = pkgbase.path
-        self.pkgver = pkgbase.pkgver
-        self.pkgrel = pkgbase.pkgrel
-        self.update_version()
 
 
 def parse_pkgbuild(relative_pkg_dir: str, _config: Optional[ConfigStateHolder] = None) -> Sequence[Pkgbuild]:
@@ -214,9 +234,21 @@ def parse_pkgbuild(relative_pkg_dir: str, _config: Optional[ConfigStateHolder] =
 
 
 _pkgbuilds_cache = dict[str, Pkgbuild]()
+_pkgbuilds_paths = dict[str, list[Pkgbuild]]()
 _pkgbuilds_scanned: bool = False
 
 
+def get_pkgbuild_by_path(relative_path: str, lazy: bool = True, _config: Optional[config] = None) -> list[Pkgbuild]:
+    global _pkgbuilds_cache, _pkgbuilds_paths
+    if lazy and relative_path in _pkgbuilds_paths:
+        return _pkgbuilds_paths[relative_path]
+    parsed = parse_pkgbuild(relative_path, _config=_config)
+    _pkgbuilds_paths[relative_path] = parsed
+    for pkg in parsed:
+        _pkgbuilds_cache[pkg.name] = pkg
+    return parsed
+
+
 def discover_pkgbuilds(parallel: bool = True, lazy: bool = True) -> dict[str, Pkgbuild]:
     global _pkgbuilds_cache, _pkgbuilds_scanned
     if lazy and _pkgbuilds_scanned:
@@ -230,17 +262,29 @@ def discover_pkgbuilds(parallel: bool = True, lazy: bool = True) -> dict[str, Pk
         for dir in os.listdir(os.path.join(pkgbuilds_dir, repo)):
             paths.append(os.path.join(repo, dir))
 
-    results = []
-
     logging.info("Parsing PKGBUILDs")
 
-    logging.debug(f"About to parse pkgbuilds. verbosity: {config.runtime['verbose']}")
+    results = []
     if parallel:
-        chunks = (Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(parse_pkgbuild)(path, config) for path in paths))
+        paths_filtered = paths
+        if lazy:
+            # filter out cached packages as the caches don't cross process boundaries
+            paths_filtered = []
+            for p in paths:
+                if p in _pkgbuilds_paths:
+                    # use cache
+                    results += _pkgbuilds_paths[p]
+                else:
+                    paths_filtered += [p]
+        chunks = (Parallel(n_jobs=multiprocessing.cpu_count() * 4)(
+            delayed(get_pkgbuild_by_path)(path, lazy=lazy, _config=config) for path in paths_filtered))
     else:
-        chunks = (parse_pkgbuild(path) for path in paths)
+        chunks = (get_pkgbuild_by_path(path, lazy=lazy) for path in paths)
 
+    _pkgbuilds_paths.clear()
+    # one list of packages per path
     for pkglist in chunks:
+        _pkgbuilds_paths[pkglist[0].path] = pkglist
         results += pkglist
 
     logging.debug('Building package dictionary!')

packages/test_device.py

@@ -42,7 +42,7 @@ ONEPLUS_ENCHILADA_PKG = f'device-{ONEPLUS_ENCHILADA}'
 def enchilada_pkgbuild(initialise_pkgbuilds_dir: ConfigStateHolder):
     config = initialise_pkgbuilds_dir
     config.try_load_file()
-    return parse_pkgbuild(os.path.join('device', ONEPLUS_ENCHILADA_PKG), config)[0]
+    return parse_pkgbuild(os.path.join('device', ONEPLUS_ENCHILADA_PKG), _config=config)[0]
 
 
 def validate_oneplus_enchilada(d: Device):