mirror of
https://gitlab.com/kupfer/kupferbootstrap.git
synced 2025-02-23 05:35:44 -05:00
431 lines
15 KiB
Python
431 lines
15 KiB
Python
from logger import *
|
|
import atexit
|
|
import click
|
|
import logging
|
|
import multiprocessing
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
|
|
makepkg_env = os.environ.copy() | {
|
|
'LANG': 'C',
|
|
'MAKEFLAGS': f'-j{multiprocessing.cpu_count()}',
|
|
}
|
|
|
|
makepkg_cmd = ['makepkg',
|
|
'--config', '/app/src/makepkg.conf',
|
|
'--noconfirm',
|
|
'--ignorearch',
|
|
'--needed']
|
|
|
|
pacman_cmd = ['pacman',
|
|
'--noconfirm',
|
|
'--overwrite=*',
|
|
'--needed', ]
|
|
|
|
|
|
class Package:
|
|
names = []
|
|
depends = []
|
|
local_depends = None
|
|
repo = ''
|
|
mode = ''
|
|
has_pkgver = False
|
|
|
|
def __init__(self, path: str) -> None:
|
|
self.path = path
|
|
self._loadinfo()
|
|
|
|
def _loadinfo(self):
|
|
result = subprocess.run(makepkg_cmd+['--printsrcinfo'],
|
|
cwd=self.path,
|
|
stdout=subprocess.PIPE)
|
|
lines = result.stdout.decode('utf-8').split('\n')
|
|
names = []
|
|
depends = []
|
|
for line in lines:
|
|
if line.startswith('pkgbase') or line.startswith('\tpkgname') or line.startswith('\tprovides'):
|
|
names.append(line.split(' = ')[1])
|
|
if line.startswith('\tdepends') or line.startswith('\tmakedepends') or line.startswith('\tcheckdepends') or line.startswith('\toptdepends'):
|
|
depends.append(line.split(' = ')[1].split('=')[0])
|
|
self.names = list(set(names))
|
|
self.depends = list(set(depends))
|
|
|
|
self.repo = self.path.split('/')[0]
|
|
|
|
mode = ''
|
|
with open(os.path.join(self.path, 'PKGBUILD'), 'r') as file:
|
|
for line in file.read().split('\n'):
|
|
if line.startswith('_mode='):
|
|
mode = line.split('=')[1]
|
|
break
|
|
if mode not in ['host', 'cross']:
|
|
logging.fatal(
|
|
f'Package {self.path} has an invalid mode configured: \'{mode}\'')
|
|
exit(1)
|
|
self.mode = mode
|
|
|
|
has_pkgver = False
|
|
with open(os.path.join(self.path, 'PKGBUILD'), 'r') as file:
|
|
for line in file.read().split('\n'):
|
|
if line.startswith('pkgver()'):
|
|
has_pkgver = True
|
|
break
|
|
self.has_pkgver = has_pkgver
|
|
|
|
|
|
def check_prebuilts():
|
|
if not os.path.exists('prebuilts'):
|
|
os.makedirs('prebuilts')
|
|
for repo in ['main', 'device']:
|
|
if not os.path.exists(os.path.join('prebuilts', repo)):
|
|
os.makedirs(os.path.join('prebuilts', repo))
|
|
for ext1 in ['db', 'files']:
|
|
for ext2 in ['', '.tar.xz']:
|
|
if not os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext1}{ext2}')):
|
|
result = subprocess.run(['tar',
|
|
'-czf',
|
|
f'{repo}.{ext1}{ext2}',
|
|
'-T', '/dev/null'],
|
|
cwd=os.path.join('prebuilts', repo))
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed create prebuilt repos')
|
|
exit(1)
|
|
|
|
|
|
def setup_chroot():
|
|
if not os.path.exists('/chroot/root'):
|
|
logging.info('Initializing root chroot')
|
|
result = subprocess.run(['mkarchroot',
|
|
'-s',
|
|
'-C', '/app/src/pacman.conf',
|
|
'-f', '/etc/locale.gen',
|
|
'/chroot/root',
|
|
'base-devel'])
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed to initialize root chroot')
|
|
shutil.rmtree('/chroot/root')
|
|
exit(1)
|
|
else:
|
|
logging.info('Updating root chroot')
|
|
result = subprocess.run(pacman_cmd +
|
|
['-Syuu',
|
|
'--root', '/chroot/root',
|
|
'--arch', 'aarch64',
|
|
'--config', '/app/src/pacman.conf'])
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed to update root chroot')
|
|
exit(1)
|
|
|
|
shutil.copyfile('/app/src/pacman.conf', '/app/src/pacman_copy.conf')
|
|
with open('/app/src/pacman_copy.conf', 'a') as file:
|
|
file.write('\n\n[main]\nServer = file:///src/prebuilts/main')
|
|
file.write('\n\n[device]\nServer = file:///src/prebuilts/device')
|
|
shutil.copyfile('/app/src/pacman_copy.conf',
|
|
'/chroot/root/etc/pacman.conf')
|
|
shutil.copyfile('/etc/pacman.d/aarch64_mirrorlist',
|
|
'/chroot/root/etc/pacman.d/aarch64_mirrorlist')
|
|
result = subprocess.run(pacman_cmd +
|
|
['-Sy',
|
|
'--root', '/chroot/root',
|
|
'--arch', 'aarch64',
|
|
'--config', '/chroot/root/etc/pacman.conf'])
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed to update root chroot')
|
|
exit(1)
|
|
|
|
with open('/chroot/root/usr/bin/makepkg', 'r') as file:
|
|
data = file.read()
|
|
data = data.replace('EUID == 0', 'EUID == -1')
|
|
with open('/chroot/root/usr/bin/makepkg', 'w') as file:
|
|
file.write(data)
|
|
|
|
with open('/chroot/root/etc/makepkg.conf', 'r') as file:
|
|
data = file.read()
|
|
data = data.replace('xz -c', 'xz -T0 -c')
|
|
data = data.replace(' check ', ' !check ')
|
|
with open('/chroot/root/etc/makepkg.conf', 'w') as file:
|
|
file.write(data)
|
|
|
|
logging.info('Syncing chroot copy')
|
|
result = subprocess.run(
|
|
['rsync', '-a', '--delete', '-q', '-W', '-x', '/chroot/root/', '/chroot/copy'])
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed to sync chroot copy')
|
|
exit(1)
|
|
|
|
|
|
def discover_packages() -> list[Package]:
|
|
packages = []
|
|
paths = []
|
|
|
|
for dir in os.listdir('main'):
|
|
paths.append(os.path.join('main', dir))
|
|
for dir1 in os.listdir('device'):
|
|
for dir2 in os.listdir(os.path.join('device', dir1)):
|
|
paths.append(os.path.join('device', dir1, dir2))
|
|
|
|
for path in paths:
|
|
logging.debug(f'Discovered {path}')
|
|
packages.append(Package(path))
|
|
|
|
# This filters the deps to only include the ones that are provided in this repo
|
|
for package in packages:
|
|
package.local_depends = package.depends.copy()
|
|
for dep in package.depends.copy():
|
|
found = False
|
|
for p in packages:
|
|
for name in p.names:
|
|
if dep == name:
|
|
found = True
|
|
break
|
|
if found:
|
|
break
|
|
if not found:
|
|
package.local_depends.remove(dep)
|
|
|
|
return packages
|
|
|
|
|
|
def generate_package_order(packages: list[Package]) -> list[Package]:
|
|
unsorted = packages.copy()
|
|
sorted = []
|
|
|
|
"""
|
|
It goes through all unsorted packages and checks if the dependencies have already been sorted.
|
|
If that is true, the package itself is added to the sorted packages
|
|
"""
|
|
while len(unsorted) > 0:
|
|
for package in unsorted.copy():
|
|
if len(package.local_depends) == 0:
|
|
sorted.append(package)
|
|
unsorted.remove(package)
|
|
for package in sorted:
|
|
for name in package.names:
|
|
for p in unsorted:
|
|
for dep in p.local_depends.copy():
|
|
if name == dep:
|
|
p.local_depends.remove(name)
|
|
|
|
return sorted
|
|
|
|
|
|
def update_package_version_and_sources(package: Package):
|
|
"""
|
|
This updates the package version and the sources.
|
|
It is done here already, because doing it while host-compiling takes longer.
|
|
We decided to even pin the commit of every -git package so this won't update any version, but it would if possible.
|
|
"""
|
|
cmd = makepkg_cmd+['--nobuild', '--noprepare', '--nodeps', '--skipinteg']
|
|
if not package.has_pkgver:
|
|
cmd.append('--noextract')
|
|
logging.info(f'Updating package version for {package.path}')
|
|
result = subprocess.run(cmd,
|
|
env=makepkg_env,
|
|
cwd=package.path)
|
|
if result.returncode != 0:
|
|
logging.fatal(f'Failed to update package version for {package.path}')
|
|
exit(1)
|
|
|
|
|
|
def check_package_version_built(package: Package) -> bool:
|
|
built = True
|
|
|
|
result = subprocess.run(makepkg_cmd +
|
|
['--nobuild',
|
|
'--noprepare',
|
|
'--packagelist'],
|
|
env=makepkg_env,
|
|
cwd=package.path,
|
|
capture_output=True)
|
|
if result.returncode != 0:
|
|
logging.fatal(f'Failed to get package list for {package.path}')
|
|
exit(1)
|
|
|
|
for line in result.stdout.decode('utf-8').split('\n'):
|
|
if line != "":
|
|
file = os.path.basename(line)
|
|
if not os.path.exists(os.path.join('prebuilts', package.repo, file)):
|
|
built = False
|
|
|
|
return built
|
|
|
|
|
|
def setup_dependencies_and_sources(package: Package):
|
|
logging.info(f'Setting up dependencies and sources for {package.path}')
|
|
|
|
"""
|
|
To make cross-compilation work for almost every package, the host needs to have the dependencies installed
|
|
so that the build tools can be used
|
|
"""
|
|
if package.mode == 'cross':
|
|
for p in package.depends:
|
|
subprocess.run(pacman_cmd + ['-Sw', p],
|
|
stderr=subprocess.DEVNULL)
|
|
|
|
result = subprocess.run(makepkg_cmd +
|
|
['--nobuild',
|
|
'--skipinteg',
|
|
'--holdver',
|
|
'--syncdeps'],
|
|
env=makepkg_env,
|
|
cwd=package.path)
|
|
if result.returncode != 0:
|
|
logging.fatal(
|
|
f'Failed to setup dependencies and sources for {package.path}')
|
|
exit(1)
|
|
|
|
|
|
def build_package(package: Package):
|
|
makepkg_compile_opts = ['--noextract',
|
|
'--skipinteg',
|
|
'--holdver',
|
|
'--nodeps']
|
|
|
|
if package.mode == 'cross':
|
|
logging.info(f'Cross-compiling {package.path}')
|
|
result = subprocess.run(makepkg_cmd+makepkg_compile_opts,
|
|
env=makepkg_env | {
|
|
'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'},
|
|
cwd=package.path)
|
|
if result.returncode != 0:
|
|
logging.fatal(
|
|
f'Failed to cross-compile package {package.path}')
|
|
exit(1)
|
|
else:
|
|
logging.info(f'Host-compiling {package.path}')
|
|
|
|
def umount():
|
|
subprocess.run(['umount', '-lc', '/chroot/copy'],
|
|
stderr=subprocess.DEVNULL)
|
|
atexit.register(umount)
|
|
|
|
result = subprocess.run(
|
|
['mount', '-o', 'bind', '/chroot/copy', '/chroot/copy'])
|
|
if result.returncode != 0:
|
|
logging.fatal('Failed to bind mount chroot to itself')
|
|
exit(1)
|
|
|
|
os.makedirs('/chroot/copy/src')
|
|
result = subprocess.run(
|
|
['mount', '-o', 'bind', '.', '/chroot/copy/src'])
|
|
if result.returncode != 0:
|
|
logging.fatal(
|
|
f'Failed to bind mount folder to chroot')
|
|
exit(1)
|
|
|
|
env = []
|
|
for key in makepkg_env:
|
|
env.append(f'{key}={makepkg_env[key]}')
|
|
result = subprocess.run(
|
|
['arch-chroot', '/chroot/copy', '/bin/bash', '-c', f'cd /src/{package.path} && {" ".join(env)} makepkg --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}'])
|
|
if result.returncode != 0:
|
|
logging.fatal(f'Failed to host-compile package {package.path}')
|
|
exit(1)
|
|
|
|
umount()
|
|
|
|
|
|
def add_package_to_repo(package: Package):
|
|
logging.info(f'Adding {package.path} to repo')
|
|
dir = os.path.join('prebuilts', package.repo)
|
|
if not os.path.exists(dir):
|
|
os.mkdir(dir)
|
|
|
|
for file in os.listdir(package.path):
|
|
# Forced extension by makepkg.conf
|
|
if file.endswith('.pkg.tar.xz'):
|
|
shutil.move(
|
|
os.path.join(package.path, file),
|
|
os.path.join(dir, file),
|
|
)
|
|
result = subprocess.run(['repo-add',
|
|
'--remove',
|
|
'--new',
|
|
'--prevent-downgrade',
|
|
os.path.join(
|
|
'prebuilts',
|
|
package.repo,
|
|
f'{package.repo}.db.tar.xz',
|
|
),
|
|
os.path.join(dir, file),
|
|
])
|
|
if result.returncode != 0:
|
|
logging.fatal(f'Failed add package {package.path} to repo')
|
|
exit(1)
|
|
for repo in ['main', 'device']:
|
|
for ext in ['db', 'files']:
|
|
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz')):
|
|
os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}'))
|
|
shutil.copyfile(
|
|
os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz'),
|
|
os.path.join('prebuilts', repo, f'{repo}.{ext}')
|
|
)
|
|
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old')):
|
|
os.unlink(os.path.join('prebuilts', repo,
|
|
f'{repo}.{ext}.tar.xz.old'))
|
|
|
|
|
|
@click.group(name='packages')
|
|
def cmd_packages():
|
|
pass
|
|
|
|
|
|
@click.command(name='build')
|
|
@verbose_option
|
|
@click.argument('path')
|
|
def cmd_build(verbose, path):
|
|
setup_logging(verbose)
|
|
|
|
check_prebuilts()
|
|
|
|
if path == 'all':
|
|
packages = generate_package_order(discover_packages())
|
|
need_build = []
|
|
|
|
for package in packages:
|
|
update_package_version_and_sources(package)
|
|
if not check_package_version_built(package):
|
|
need_build.append(package)
|
|
|
|
if len(need_build) == 0:
|
|
logging.info('Everything built already')
|
|
return
|
|
logging.info('Building %s', ', '.join(
|
|
map(lambda x: x.path, need_build)))
|
|
with open('.last_built', 'w') as file:
|
|
file.write('\n'.join(
|
|
map(lambda x: x.path, need_build)))
|
|
|
|
for package in need_build:
|
|
setup_chroot()
|
|
setup_dependencies_and_sources(package)
|
|
build_package(package)
|
|
add_package_to_repo(package)
|
|
else:
|
|
package = Package(path)
|
|
update_package_version_and_sources(package)
|
|
if not check_package_version_built(package):
|
|
with open('.last_built', 'w') as file:
|
|
file.write(package.path)
|
|
setup_chroot()
|
|
setup_dependencies_and_sources(package)
|
|
build_package(package)
|
|
add_package_to_repo(package)
|
|
|
|
|
|
@click.command(name='clean')
|
|
@verbose_option
|
|
def cmd_clean(verbose):
|
|
setup_logging(verbose)
|
|
result = subprocess.run(['git',
|
|
'clean',
|
|
'-dffX',
|
|
'main', 'device'])
|
|
if result.returncode != 0:
|
|
logging.fatal(f'Failed to git clean')
|
|
exit(1)
|
|
|
|
|
|
cmd_packages.add_command(cmd_build)
|
|
cmd_packages.add_command(cmd_clean)
|