let's break all the things! (use distros to generate repos)

Signed-off-by: InsanePrawn <insane.prawny@gmail.com>

boot.py

@@ -4,7 +4,6 @@ from image import get_device_and_flavour, get_image_name, dump_bootimg, dump_lk2
 from fastboot import fastboot_boot, fastboot_erase_dtbo
 from constants import BOOT_STRATEGIES, FLASH_PARTS, FASTBOOT, JUMPDRIVE, JUMPDRIVE_VERSION
 import click
-import logging
 
 LK2ND = FLASH_PARTS['LK2ND']
 BOOTIMG = FLASH_PARTS['BOOTIMG']

chroot.py

@@ -3,6 +3,7 @@ import subprocess
 import os
 import shutil
 from config import config
+from distro import get_base_distros, RepoInfo
 
 
 def get_chroot_path(chroot_name, override_basepath: str = None) -> str:
@@ -15,11 +16,12 @@ def create_chroot(
     arch='aarch64',
     packages=['base'],
     pacman_conf='/app/local/etc/pacman.conf',
-    extra_repos={},
+    extra_repos: dict[str, RepoInfo] = {},
     chroot_base_path: str = None,
 ):
     base_chroot = f'base_{arch}'
     chroot_path = get_chroot_path(chroot_name, override_basepath=chroot_base_path)
+    base_distro = get_base_distros()[arch]
     pacman_conf_target = chroot_path + '/etc/pacman.conf'
 
     # copy base_chroot instead of creating from scratch every time
@@ -42,14 +44,10 @@ def create_chroot(
             exit(1)
 
     os.makedirs(chroot_path + '/etc', exist_ok=True)
-    shutil.copyfile(pacman_conf, pacman_conf_target)
 
-    extra_conf = ''
+    conf_text = base_distro.get_pacman_conf(extra_repos)
-    for repo_name, repo_options in extra_repos.items():
+    with open(pacman_conf_target, 'w') as file:
-        extra_conf += f'\n\n[{repo_name}]\n'
+        file.write(conf_text)
-        extra_conf += '\n'.join(['%s = %s' % (name, value) for name, value in repo_options.items()])
-    with open(pacman_conf_target, 'a') as file:
-        file.write(extra_conf)
 
     result = subprocess.run([
         'pacstrap',

constants.py

@@ -71,3 +71,5 @@ BASE_DISTROS = {
         },
     },
 }
+
+KUPFER_HTTPS = 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo'

distro.py

@@ -1,5 +1,5 @@
 from copy import deepcopy
-from constants import ARCHES, BASE_DISTROS
+from constants import ARCHES, BASE_DISTROS, REPOSITORIES, KUPFER_HTTPS
 from config import config
 
 
@@ -28,19 +28,28 @@ class PackageInfo:
         self.resolved_url = resolved_url
 
 
-class Repo:
+class RepoInfo:
-    name: str
+    options: dict[str, str] = {}
     url_template: str
+
+    def __init__(self, url_template: str, options: dict[str, str] = {}):
+        self.url_template = url_template
+        self.options.update(options)
+
+
+class Repo(RepoInfo):
+    name: str
     resolved_url: str
     arch: str
     packages: dict[str, PackageInfo]
-    options: dict[str, str]
     remote: bool
+    scanned: bool = False
 
     def scan(self):
-        self.resolved_url = resolve_url(self.url_template, repo_name=self.repo_name, arch=self.arch)
+        self.resolved_url = resolve_url(self.url_template, repo_name=self.name, arch=self.arch)
         self.remote = not self.resolved_url.startswith('file://')
         # TODO
+        self.scanned = True
 
     def __init__(self, name: str, url_template: str, arch: str, options={}, scan=True):
         self.name = name
@@ -54,14 +63,8 @@ class Repo:
         options = {'Server': self.url_template} | self.options.items()
         return ('[%s]\n' % self.name) + '\n'.join([f"{key} = {value}" for key, value in options])
 
-
+    def get_RepoInfo(self):
-class RepoInfo:
+        return RepoInfo(url_template=self.url_template, options=self.options)
-    options: dict[str, str] = {}
-    url_template: str
-
-    def __init__(self, url_template: str, options: dict[str, str] = {}):
-        self.url_template = url_template
-        self.options.update(options)
 
 
 class Distro:
@@ -89,11 +92,85 @@ class Distro:
             for package in repo.packages:
                 results[package.name] = package
 
-    def config_snippet(self) -> str:
+    def _repos_config_snippet(self, extra_repos: dict[str, RepoInfo] = {}) -> str:
-        return '\n'.join(repo.config_snippet() for repo in self.repos)
+        extras = [Repo(name, url_template=info.url_template, options=info.options, scan=False) for name, info in extra_repos.items()]
+        return '\n'.join(repo.config_snippet() for repo in (self.repos.values + extras))
+
+    def get_pacman_conf(self, extra_repos: dict[str, RepoInfo] = []):
+        header = f'''
+#
+# /etc/pacman.conf
+#
+# See the pacman.conf(5) manpage for option and repository directives
+
+#
+# GENERAL OPTIONS
+#
+[options]
+# The following paths are commented out with their default values listed.
+# If you wish to use different paths, uncomment and update the paths.
+#RootDir     = /
+#DBPath      = /var/lib/pacman/
+#CacheDir    = /var/cache/pacman/pkg/
+#LogFile     = /var/log/pacman.log
+#GPGDir      = /etc/pacman.d/gnupg/
+#HookDir     = /etc/pacman.d/hooks/
+HoldPkg     = pacman glibc
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
+#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
+#CleanMethod = KeepInstalled
+Architecture = {self.arch}
+
+# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
+#IgnorePkg   =
+#IgnoreGroup =
+
+#NoUpgrade   =
+#NoExtract   =
+
+# Misc options
+#UseSyslog
+Color
+#NoProgressBar
+CheckSpace
+VerbosePkgLists
+ParallelDownloads = 8
+
+# By default, pacman accepts packages signed by keys that its local keyring
+# trusts (see pacman-key and its man page), as well as unsigned packages.
+SigLevel    = Required DatabaseOptional
+LocalFileSigLevel = Optional
+#RemoteFileSigLevel = Required
+
+# NOTE: You must run `pacman-key --init` before first using pacman; the local
+# keyring can then be populated with the keys of all official Arch Linux ARM
+# packagers with `pacman-key --populate archlinuxarm`.
+
+#
+# REPOSITORIES
+#   - can be defined here or included from another file
+#   - pacman will search repositories in the order defined here
+#   - local/custom mirrors can be added here or in separate files
+#   - repositories listed first will take precedence when packages
+#     have identical names, regardless of version number
+#   - URLs will have $repo replaced by the name of the current repo
+#   - URLs will have $arch replaced by the name of the architecture
+#
+# Repository entries are of the format:
+#       [repo-name]
+#       Server = ServerName
+#       Include = IncludePath
+#
+# The header [repo-name] is crucial - it must be present and
+# uncommented to enable the repo.
+#
+
+'''
+        return header + self._repos_config_snippet(extra_repos)
 
 
 _base_distros: dict[str, Distro] = None
+_kupfer_distros: dict[str, Distro] = {}
 
 
 def get_base_distros() -> dict[str, Distro]:
@@ -105,3 +182,22 @@ def get_base_distros() -> dict[str, Distro]:
             _distros[arch] = Distro(arch=arch, repo_infos=repos, scan=False)
         _base_distros = _distros
     return _base_distros
+
+
+def get_kupfer(arch: str, url_template: str) -> Distro:
+    global _kupfer_distros
+    if arch not in _kupfer_distros:
+        repos = {name: RepoInfo(url_template=url_template, options={'SigLevel': 'Never'}) for name in REPOSITORIES}
+        _kupfer_distros[arch] = Distro(
+            arch=arch,
+            repo_infos=repos,
+        )
+    return _kupfer_distros[arch]
+
+
+def get_kupfer_https(arch: str) -> Distro:
+    return get_kupfer(arch, KUPFER_HTTPS)
+
+
+def get_kupfer_local(arch: str) -> Distro:
+    return get_kupfer(arch, f"file://{config.file['paths']['packages']}/$repo")

image.py

@@ -6,6 +6,7 @@ from logger import logging
 from chroot import create_chroot, create_chroot_user, get_chroot_path, run_chroot_cmd
 from constants import DEVICES, FLAVOURS, REPOSITORIES
 from config import config
+from distro import get_kupfer_https, get_kupfer_local
 
 
 def get_device_and_flavour(profile=None) -> tuple[str, str]:
@@ -108,6 +109,9 @@ def cmd_build():
     post_cmds = FLAVOURS[flavour].get('post_cmds', [])
     image_name = get_image_name(device, flavour)
 
+    # TODO: PARSE DEVICE ARCH
+    arch = 'aarch64'
+
     if not os.path.exists(image_name):
         result = subprocess.run([
             'fallocate',
@@ -133,11 +137,10 @@ def cmd_build():
     mount_rootfs_image(image_name, rootfs_mount)
 
     packages_dir = config.file['paths']['packages']
-    if os.path.exists(packages_dir):
+    if os.path.exists(os.path.join(packages_dir, 'main')):
-        url = f'file://{packages_dir}/$repo'
+        extra_repos = get_kupfer_local(arch).repos
     else:
-        url = 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo'
+        extra_repos = get_kupfer_https(arch).repos
-    extra_repos = {repo: {'Server': url} for repo in REPOSITORIES}
     packages = ['base', 'base-kupfer'] + DEVICES[device] + FLAVOURS[flavour]['packages'] + profile['pkgs_include']
     create_chroot(
         chroot_name,

logger.py

@@ -1,7 +1,6 @@
 import click
 import logging
 import sys
-from traceback import format_exc as get_trace
 
 
 def setup_logging(verbose: bool):

main.py

@@ -28,7 +28,7 @@ def cli(verbose: bool = False, config_file: str = None, no_wrapper: bool = False
 def main():
     try:
         return cli(prog_name='kupferbootstrap')
-    except Exception as err:
+    except Exception:
         logging.fatal(get_trace())
         exit(1)
 

packages.py

@@ -9,6 +9,7 @@ from constants import REPOSITORIES
 from config import config
 from chroot import create_chroot
 from joblib import Parallel, delayed
+from distro import RepoInfo, get_kupfer_local
 
 makepkg_env = os.environ.copy() | {
     'LANG': 'C',
@@ -295,17 +296,11 @@ def check_package_version_built(package: Package) -> bool:
 def setup_build_chroot(arch='aarch64', extra_packages=[]) -> str:
     chroot_name = f'build_{arch}'
     logging.info(f'Initializing {arch} build chroot')
-    extra_repos = {}
-    for repo in REPOSITORIES:
-        extra_repos[repo] = {
-            'Server': f"file://{config.file['paths']['packages']}/{repo}",
-            'SigLevel': 'Never',
-        }
     chroot_path = create_chroot(
         chroot_name,
         packages=['base-devel', 'git'] + extra_packages,
         pacman_conf='/app/local/etc/pacman.conf',
-        extra_repos=extra_repos,
+        extra_repos=get_kupfer_local(arch).repos,
     )
 
     logging.info(f'Updating chroot {chroot_name}')