distro/keyrings: extraction works

distro/distro.py

@@ -49,6 +49,15 @@ class Distro(Generic[RepoType]):
             results.update(repo.packages)
         return results
 
+    def find_package(self, name: str) -> Optional[BinaryPackageType]:
+        for repo in self.repos.values():
+            if not repo.scanned:
+                repo.scan()
+            p = repo.packages.get(name, None)
+            if p:
+                return p
+        return None
+
     def repos_config_snippet(self, extra_repos: Mapping[str, RepoInfo] = {}) -> str:
         extras: list[Repo] = [
             Repo(name, url_template=info.url_template, arch=self.arch, options=info.options, scan=False) for name, info in extra_repos.items()

distro/keyring.py

@@ -9,7 +9,7 @@ from constants import Arch, KEYRINGS_KEY, KEYRINGS_LOCAL_KEY
 from distro.repo_config import get_repo_config
 from exec.cmd import CompletedProcess, run_cmd
 from exec.file import makedir, remove_file
-from utils import extract_files_from_tar_generator, read_files_from_tar_recursive
+from utils import extract_files_from_tar_generator, read_files_from_tar_recursive, sha256sum
 
 from .distro import Distro, get_base_distro, get_kupfer_local, get_kupfer_https
 from .package import BinaryPackage
@@ -64,7 +64,7 @@ def init_keyring_gpg_dir(
 ) -> dict[str, bool]:
     base_dir = base_dir or get_keyring_path(arch, distro_type)
     gpg_dir = get_keyring_gpg_path(base_dir)
-    exists = os.path.exists(gpg_dir)
+    exists = os.path.exists(os.path.join(gpg_dir, 'trustdb.gpg'))
     if exists and not lazy:
         remove_file(gpg_dir)
         exists = False
@@ -109,7 +109,7 @@ def init_keyring_dist_dir(
     pkg_names: list[str] = []
     distro: Distro
     if distro_type == DistroType.BASE:
-        pkg_names = repo_config.base_distros.get(arch, {}).get(KEYRINGS_KEY, None) or []
+        pkg_names = repo_config.base_distros.get(arch, {}).get(KEYRINGS_KEY, None) or []  # type: ignore[call-overload]
         distro = get_base_distro(arch, scan=False)
     elif distro_type == DistroType.LOCAL:
         pkg_name = repo_config.get(KEYRINGS_LOCAL_KEY, None)
@@ -119,9 +119,9 @@ def init_keyring_dist_dir(
         pkg_names = repo_config.get(KEYRINGS_KEY, None) or []
         distro = get_kupfer_https(arch, scan=False)
     logging.debug(f"Acquiring keyrings from {distro}: {pkg_names}")
-    dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, base_dir)
+    dist_pkgs, changed = acquire_dist_pkgs(pkg_names, distro, dist_dir)
-    if lazy and dist_pkgs and not changed and os.path.exists(dist_dir):  # and keyring_is_created(arch, distro_type):
+    #if lazy and dist_pkgs and not changed and os.path.exists(dist_dir):  # and keyring_is_created(arch, distro_type):
-        return {name: (val[0], False) for name, val in dist_pkgs.items()}
+    #    return {name: (get_keyring_dist_path(base_dir, name), False) for name, val in dist_pkgs.items()}
 
     makedir(dist_dir)
     dist_dirs = []
@@ -131,26 +131,25 @@ def init_keyring_dist_dir(
         _dir = os.path.join(dist_dir, name)
         results[name] = _dir, False
         if lazy and not changed and os.path.exists(_dir):
+            logging.debug(f"Skipping extracting keyring pkg for {name}: dir exists and file unchanged")
             continue
         extract_keyring_pkg(dist_pkg, _dir)
         dist_dirs.append(_dir)
-        results[name] = dist_pkg, True
+        results[name] = _dir, True
     return results
 
 
 def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str) -> tuple[dict[str, tuple[str, bool]], bool]:
     if not keyring_packages:
         return {}, False
-    pkgs = {}
+    pkgs: dict[str, BinaryPackage] = {}
     not_found = []
-    distro.scan(lazy=True)
+    pkg: Optional[BinaryPackage]
-    repos: dict[str, BinaryPackage] = distro.get_packages()
-    pkg: BinaryPackage
     for name in keyring_packages:
-        if name not in repos:
+        pkg = distro.find_package(name)
+        if not pkg:
             not_found.append(name)
             continue
-        pkg = repos[name]
         pkgs[name] = pkg
     if not_found:
         raise Exception(f"Keyring packages for {distro.arch} not found: {not_found}")
@@ -158,22 +157,25 @@ def acquire_dist_pkgs(keyring_packages: list[str], distro: Distro, dist_dir: str
     changed = False
     results = {}
     for name in pkgs:
-        assert isinstance(pkgs[name], BinaryPackage)
         pkg = pkgs[name]
         assert PKGNAME_MARKER in pkg.filename
         comp_ext = pkg.filename.rsplit(PKGNAME_MARKER, 1)[1]
-        target_path, _changed = pkg.acquire(dist_dir, f'{name}.tar{comp_ext}')
+        filename = f'{name}.tar{comp_ext}'
+        filepath = os.path.join(dist_dir, filename)
+        checksum = None if not os.path.exists(filepath) else sha256sum(filepath)
+        target_path, _changed = pkg.acquire(dist_dir, filename)
+        _changed = _changed and checksum != sha256sum(filepath)
         results[name] = target_path, _changed
         if _changed:
-            logging.debug(f"{target_path} changed")
             changed = True
+            logging.debug(f"{target_path} changed")
     return results, changed
 
 
 def extract_keyring_pkg(pkg_path: str, dest_path: str):
-    makedir(dest_path)
+    logging.debug(f"Extracting {pkg_path} to {dest_path}")
     extract_files_from_tar_generator(
-        read_files_from_tar_recursive(pkg_path, PKG_KEYRING_FOLDER),
+        read_files_from_tar_recursive(pkg_path, [PKG_KEYRING_FOLDER]),
         dest_path,
         remove_prefix=PKG_KEYRING_FOLDER,
     )
@@ -183,8 +185,8 @@ def get_keyring_path(arch: Arch, distro_type: DistroType, *extra_paths) -> str:
     return os.path.join(config.get_path('pacman'), KEYRING_DIR, arch, KEYRING_LOCATIONS[distro_type], *extra_paths)
 
 
-def get_keyring_dist_path(base_dir: str) -> str:
+def get_keyring_dist_path(base_dir: str, *name) -> str:
-    return os.path.join(base_dir, KEYRING_DIST_DIR)
+    return os.path.join(base_dir, KEYRING_DIST_DIR, *name)
 
 
 def get_keyring_gpg_path(base_dir: str) -> str:

distro/package.py

@@ -94,5 +94,5 @@ class RemotePackage(BinaryPackage):
 
         logging.info(f"Trying to download package {url}")
         changed = download_file(dest_file_path, url)
-        logging.info(f"{self.filename} downloaded from repos")
+        logging.info(f"{self.filename} {'already ' if not changed else ''}downloaded from repos")
         return dest_file_path, changed

distro/repo_config.py

@@ -123,7 +123,7 @@ REPOS_CONFIG_DEFAULTS_DICT = {
     BASEDISTROS_KEY: {
         arch: {
             REMOTEURL_KEY: None,
-            KEYRINGS_KEY: arch_def.get(KEYRINGS_KEY, None).copy(),
+            KEYRINGS_KEY: arch_def[KEYRINGS_KEY].copy() if KEYRINGS_KEY in arch_def else None,
             'repos': {
                 k: {
                     'remote_url': v

requirements.txt

@@ -10,3 +10,4 @@ requests
 python-dateutil
 enlighten
 PyYAML
+zstandard