parchlinux-mobile/parchbootstrap
1
0
Fork 0
forked from parchlinux-mobile/kupferbootstrap
Code Issues Pull requests Activity

Apply formatting

Browse source
This commit is contained in:
jld3103 2021-08-08 18:32:42 +02:00
parent 8b943a903a
commit 98040d8a31
10 changed files with 375 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

13
boot.py
View file

@ -1,11 +1,10 @@
import os import os
import urllib.request import urllib.request
from image import get_device_and_flavour, get_image_name from image import get_device_and_flavour, get_image_name
from logger import * from logger import logging, setup_logging, verbose_option
import click import click
import subprocess import subprocess
FASTBOOT = 'fastboot' FASTBOOT = 'fastboot'
JUMPDRIVE = 'jumpdrive' JUMPDRIVE = 'jumpdrive'
@ -20,9 +19,12 @@ boot_strategies = {
def dump_bootimg(image_name: str) -> str: def dump_bootimg(image_name: str) -> str:
path = '/tmp/boot.img' path = '/tmp/boot.img'
result = subprocess.run(['debugfs', result = subprocess.run([
'debugfs',
image_name, image_name,
'-R', f'dump /boot/boot.img {path}']) '-R',
f'dump /boot/boot.img {path}',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Faild to dump boot.img') logging.fatal(f'Faild to dump boot.img')
exit(1) exit(1)
@ -43,8 +45,7 @@ def cmd_boot(verbose, type):
if type == JUMPDRIVE: if type == JUMPDRIVE:
file = f'boot-{device}.img' file = f'boot-{device}.img'
path = os.path.join('/var/cache/jumpdrive', file) path = os.path.join('/var/cache/jumpdrive', file)
urllib.request.urlretrieve( urllib.request.urlretrieve(f'https://github.com/dreemurrs-embedded/Jumpdrive/releases/download/{jumpdrive_version}/{file}', path)
f'https://github.com/dreemurrs-embedded/Jumpdrive/releases/download/{jumpdrive_version}/{file}', path)
else: else:
path = dump_bootimg(image_name) path = dump_bootimg(image_name)

2
cache.py
View file

@ -1,5 +1,5 @@
import shutil import shutil
from logger import * from logger import setup_logging, verbose_option
import click import click
import os import os

32
chroot.py
View file

@ -4,7 +4,13 @@ import os
import shutil import shutil
def create_chroot(chroot_path, packages=['base'], pacman_conf='/app/local/etc/pacman.conf', chroot_base_path='/chroot', extra_repos={}): def create_chroot(
chroot_path,
packages=['base'],
pacman_conf='/app/local/etc/pacman.conf',
chroot_base_path='/chroot',
extra_repos={},
):
pacman_conf_target = chroot_path + '/etc/pacman.conf' pacman_conf_target = chroot_path + '/etc/pacman.conf'
os.makedirs(chroot_path + '/etc', exist_ok=True) os.makedirs(chroot_path + '/etc', exist_ok=True)
@ -13,18 +19,15 @@ def create_chroot(chroot_path, packages=['base'], pacman_conf='/app/local/etc/pa
extra_conf = '' extra_conf = ''
for repo_name, repo_options in extra_repos.items(): for repo_name, repo_options in extra_repos.items():
extra_conf += f'\n\n[{repo_name}]\n' extra_conf += f'\n\n[{repo_name}]\n'
extra_conf += '\n'.join(['%s = %s' % (name, value) extra_conf += '\n'.join(['%s = %s' % (name, value) for name, value in repo_options.items()])
for name, value in repo_options.items()])
with open(pacman_conf_target, 'a') as file: with open(pacman_conf_target, 'a') as file:
file.write(extra_conf) file.write(extra_conf)
result = subprocess.run(['pacstrap', result = subprocess.run(['pacstrap', '-C', pacman_conf_target, '-c', '-G', chroot_path] + packages + [
'-C', pacman_conf_target, '--needed',
'-c', '--overwrite=*',
'-G', '-yyuu',
chroot_path] ])
+ packages
+ ['--needed', '--overwrite=*', '-yyuu'])
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed to install system') logging.fatal('Failed to install system')
exit(1) exit(1)
@ -33,8 +36,7 @@ def create_chroot(chroot_path, packages=['base'], pacman_conf='/app/local/etc/pa
def create_chroot_user(chroot_path): def create_chroot_user(chroot_path):
user = 'kupfer' user = 'kupfer'
password = '123456' password = '123456'
groups = ['network', 'video', 'audio', 'optical', 'storage', groups = ['network', 'video', 'audio', 'optical', 'storage', 'input', 'scanner', 'games', 'lp', 'rfkill', 'wheel']
'input', 'scanner', 'games', 'lp', 'rfkill', 'wheel']
install_script = '\n'.join([ install_script = '\n'.join([
f'if ! id -u "{user}" >/dev/null 2>&1; then', f'if ! id -u "{user}" >/dev/null 2>&1; then',
f' useradd -m {user}', f' useradd -m {user}',
@ -43,11 +45,13 @@ def create_chroot_user(chroot_path):
f'echo "{user}:{password}" | chpasswd', f'echo "{user}:{password}" | chpasswd',
f'chown {user}:{user} /home/{user} -R', f'chown {user}:{user} /home/{user} -R',
]) ])
result = subprocess.run(['arch-chroot', result = subprocess.run([
'arch-chroot',
chroot_path, chroot_path,
'/bin/bash', '/bin/bash',
'-c', '-c',
install_script]) install_script,
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed to setup user') logging.fatal('Failed to setup user')
exit(1) exit(1)

32
flash.py
View file

@ -3,7 +3,7 @@ from boot import dump_bootimg
import os import os
import subprocess import subprocess
import click import click
from logger import * from logger import logging, setup_logging, verbose_option
ROOTFS = 'rootfs' ROOTFS = 'rootfs'
BOOTIMG = 'bootimg' BOOTIMG = 'bootimg'
@ -29,8 +29,7 @@ def cmd_flash(verbose, what, location):
logging.info(f'You need to specify a location to flash {what} to') logging.info(f'You need to specify a location to flash {what} to')
exit(1) exit(1)
if location not in locations: if location not in locations:
logging.info( logging.info(f'Invalid location {location}. Choose one of {", ".join(locations)} for location')
f'Invalid location {location}. Choose one of {", ".join(locations)} for location')
exit(1) exit(1)
dir = '/dev/disk/by-id' dir = '/dev/disk/by-id'
@ -38,41 +37,50 @@ def cmd_flash(verbose, what, location):
sanitized_file = file.replace('-', '').replace('_', '').lower() sanitized_file = file.replace('-', '').replace('_', '').lower()
if f'jumpdrive{location.split("-")[0]}' in sanitized_file: if f'jumpdrive{location.split("-")[0]}' in sanitized_file:
path = os.path.realpath(os.path.join(dir, file)) path = os.path.realpath(os.path.join(dir, file))
result = subprocess.run(['lsblk', result = subprocess.run(['lsblk', path, '-o', 'SIZE'], capture_output=True)
path,
'-o', 'SIZE'],
capture_output=True)
if result.returncode != 0: if result.returncode != 0:
logging.info(f'Failed to lsblk {path}') logging.info(f'Failed to lsblk {path}')
exit(1) exit(1)
if result.stdout == b'SIZE\n 0B\n': if result.stdout == b'SIZE\n 0B\n':
logging.info( logging.info(
f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect') f'Disk {path} has a size of 0B. That probably means it is not available (e.g. no microSD inserted or no microSD card slot installed in the device) or corrupt or defect'
)
exit(1) exit(1)
if location.endswith('-file'): if location.endswith('-file'):
logging.fatal('Not implemented yet') logging.fatal('Not implemented yet')
exit() exit()
else: else:
result = subprocess.run(['dd', result = subprocess.run([
'dd',
f'if={image_name}', f'if={image_name}',
f'of={path}', f'of={path}',
'bs=20M', 'bs=20M',
'iflag=direct', 'iflag=direct',
'oflag=direct', 'oflag=direct',
'status=progress']) 'status=progress',
])
if result.returncode != 0: if result.returncode != 0:
logging.info(f'Failed to flash {image_name} to {path}') logging.info(f'Failed to flash {image_name} to {path}')
exit(1) exit(1)
elif what == BOOTIMG: elif what == BOOTIMG:
result = subprocess.run(['fastboot', 'erase', 'dtbo']) result = subprocess.run([
'fastboot',
'erase',
'dtbo',
])
if result.returncode != 0: if result.returncode != 0:
logging.info(f'Failed to erase dtbo') logging.info(f'Failed to erase dtbo')
exit(1) exit(1)
path = dump_bootimg(image_name) path = dump_bootimg(image_name)
result = subprocess.run(['fastboot', 'flash', 'boot', path]) result = subprocess.run([
'fastboot',
'flash',
'boot',
path,
])
if result.returncode != 0: if result.returncode != 0:
logging.info(f'Failed to flash boot.img') logging.info(f'Failed to flash boot.img')
exit(1) exit(1)

48
forwarding.py
View file

@ -1,6 +1,6 @@
import click import click
import subprocess import subprocess
from logger import * from logger import logging, setup_logging, verbose_option
@click.command(name='forwarding') @click.command(name='forwarding')
@ -8,31 +8,51 @@ from logger import *
def cmd_forwarding(verbose): def cmd_forwarding(verbose):
setup_logging(verbose) setup_logging(verbose)
result = subprocess.run(['sysctl', 'net.ipv4.ip_forward=1']) result = subprocess.run([
'sysctl',
'net.ipv4.ip_forward=1',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to enable ipv4 forward via sysctl') logging.fatal(f'Failed to enable ipv4 forward via sysctl')
exit(1) exit(1)
result = subprocess.run(['iptables', '-P', 'FORWARD', 'ACCEPT']) result = subprocess.run([
'iptables',
'-P',
'FORWARD',
'ACCEPT',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed set iptables rule') logging.fatal(f'Failed set iptables rule')
exit(1) exit(1)
result = subprocess.run(['iptables', result = subprocess.run([
'-A', 'POSTROUTING', 'iptables',
'-t', 'nat', '-A',
'-j', 'MASQUERADE', 'POSTROUTING',
'-s', '172.16.42.0/24']) '-t',
'nat',
'-j',
'MASQUERADE',
'-s',
'172.16.42.0/24',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed set iptables rule') logging.fatal(f'Failed set iptables rule')
exit(1) exit(1)
result = subprocess.run(['ssh', result = subprocess.run([
'-o', 'GlobalKnownHostsFile=/dev/null', 'ssh',
'-o', 'UserKnownHostsFile=/dev/null', '-o',
'-o', 'StrictHostKeyChecking=no', 'GlobalKnownHostsFile=/dev/null',
'-t', 'kupfer@172.16.42.1', '-o',
'sudo route add default gw 172.16.42.2']) 'UserKnownHostsFile=/dev/null',
'-o',
'StrictHostKeyChecking=no',
'-t',
'kupfer@172.16.42.1',
'sudo route add default gw 172.16.42.2',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to add gateway over ssh') logging.fatal(f'Failed to add gateway over ssh')
exit(1) exit(1)

72
image.py
View file

@ -1,12 +1,8 @@
import atexit import atexit
from logging import root
import os import os
import shutil
import signal
import subprocess import subprocess
import time
import click import click
from logger import * from logger import logging, setup_logging, verbose_option
from chroot import create_chroot, create_chroot_user from chroot import create_chroot, create_chroot_user
devices = { devices = {
@ -24,12 +20,10 @@ flavours = {
def get_device_and_flavour() -> tuple[str, str]: def get_device_and_flavour() -> tuple[str, str]:
if not os.path.exists('.device'): if not os.path.exists('.device'):
logging.fatal( logging.fatal(f'Please set the device using \'kupferbootstrap image device ...\'')
f'Please set the device using \'kupferbootstrap image device ...\'')
exit(1) exit(1)
if not os.path.exists('.flavour'): if not os.path.exists('.flavour'):
logging.fatal( logging.fatal(f'Please set the flavour using \'kupferbootstrap image flavour ...\'')
f'Please set the flavour using \'kupferbootstrap image flavour ...\'')
exit(1) exit(1)
with open('.device', 'r') as file: with open('.device', 'r') as file:
@ -50,14 +44,24 @@ def mount_rootfs_image(path):
os.makedirs(rootfs_mount) os.makedirs(rootfs_mount)
def umount(): def umount():
subprocess.run(['umount', '-lc', rootfs_mount], subprocess.run(
stderr=subprocess.DEVNULL) [
'umount',
'-lc',
rootfs_mount,
],
stderr=subprocess.DEVNULL,
)
atexit.register(umount) atexit.register(umount)
result = subprocess.run(['mount', result = subprocess.run([
'-o', 'loop', 'mount',
'-o',
'loop',
path, path,
rootfs_mount]) rootfs_mount,
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to loop mount {path} to {rootfs_mount}') logging.fatal(f'Failed to loop mount {path} to {rootfs_mount}')
exit(1) exit(1)
@ -82,8 +86,7 @@ def cmd_device(verbose, device):
break break
if device not in devices: if device not in devices:
logging.fatal( logging.fatal(f'Unknown device {device}. Pick one from:\n{", ".join(devices.keys())}')
f'Unknown device {device}. Pick one from:\n{", ".join(devices.keys())}')
exit(1) exit(1)
logging.info(f'Setting device to {device}') logging.info(f'Setting device to {device}')
@ -99,8 +102,7 @@ def cmd_flavour(verbose, flavour):
setup_logging(verbose) setup_logging(verbose)
if flavour not in flavours: if flavour not in flavours:
logging.fatal( logging.fatal(f'Unknown flavour {flavour}. Pick one from:\n{", ".join(flavours.keys())}')
f'Unknown flavour {flavour}. Pick one from:\n{", ".join(flavours.keys())}')
exit(1) exit(1)
logging.info(f'Setting flavour to {flavour}') logging.info(f'Setting flavour to {flavour}')
@ -118,31 +120,47 @@ def cmd_build(verbose):
image_name = get_image_name(device, flavour) image_name = get_image_name(device, flavour)
if not os.path.exists(image_name): if not os.path.exists(image_name):
result = subprocess.run(['fallocate', result = subprocess.run([
'-l', '4G', 'fallocate',
image_name]) '-l',
'4G',
image_name,
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to allocate {image_name}') logging.fatal(f'Failed to allocate {image_name}')
exit(1) exit(1)
result = subprocess.run(['mkfs.ext4', result = subprocess.run([
'-L', 'kupfer', 'mkfs.ext4',
image_name]) '-L',
'kupfer',
image_name,
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to create ext4 filesystem on {image_name}') logging.fatal(f'Failed to create ext4 filesystem on {image_name}')
exit(1) exit(1)
rootfs_mount = mount_rootfs_image(image_name) rootfs_mount = mount_rootfs_image(image_name)
create_chroot(rootfs_mount, packages=(['base', 'base-kupfer'] + devices[device] + flavours[flavour]), pacman_conf='/app/local/etc/pacman.conf', extra_repos={'main': { create_chroot(
'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo'}, 'device': {'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo'}}) rootfs_mount,
packages=(['base', 'base-kupfer'] + devices[device] + flavours[flavour]),
pacman_conf='/app/local/etc/pacman.conf',
extra_repos={
'main': {
'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo',
},
'device': {
'Server': 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo',
},
},
)
create_chroot_user(rootfs_mount) create_chroot_user(rootfs_mount)
""" """
This doesn't work, because the mount isn't passed through to the real host This doesn't work, because the mount isn't passed through to the real host
""" """
""" """
@click.command(name='inspect') @click.command(name='inspect')
@verbose_option @verbose_option

4
logger.py
View file

@ -11,7 +11,7 @@ def setup_logging(verbose: bool):
stream=sys.stdout, stream=sys.stdout,
format='%(asctime)s %(levelname)s: %(message)s', format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %H:%M:%S', datefmt='%m/%d/%Y %H:%M:%S',
level=level level=level,
) )
@ -19,5 +19,5 @@ verbose_option = click.option(
'-v', '-v',
'--verbose', '--verbose',
is_flag=True, is_flag=True,
help='Enables verbose logging' help='Enables verbose logging',
) )

218
packages.py
View file

@ -1,5 +1,4 @@
from cmath import log from logger import setup_logging, verbose_option
from logger import *
import atexit import atexit
import click import click
import logging import logging
@ -15,21 +14,24 @@ makepkg_env = os.environ.copy() | {
'MAKEFLAGS': f'-j{multiprocessing.cpu_count()}', 'MAKEFLAGS': f'-j{multiprocessing.cpu_count()}',
} }
makepkg_cross_env = makepkg_env | { makepkg_cross_env = makepkg_env | {'PACMAN': '/app/local/bin/pacman_aarch64'}
'PACMAN': '/app/local/bin/pacman_aarch64'
}
makepkg_cmd = ['makepkg', makepkg_cmd = [
'--config', '/app/local/etc/makepkg.conf', 'makepkg',
'--config',
'/app/local/etc/makepkg.conf',
'--noconfirm', '--noconfirm',
'--ignorearch', '--ignorearch',
'--needed'] '--needed',
]
pacman_cmd = ['pacman', pacman_cmd = [
'pacman',
'-Syuu', '-Syuu',
'--noconfirm', '--noconfirm',
'--overwrite=*', '--overwrite=*',
'--needed', ] '--needed',
]
class Package: class Package:
@ -46,9 +48,11 @@ class Package:
self._loadinfo() self._loadinfo()
def _loadinfo(self): def _loadinfo(self):
result = subprocess.run(makepkg_cmd+['--printsrcinfo'], result = subprocess.run(
makepkg_cmd + ['--printsrcinfo'],
cwd=self.path, cwd=self.path,
stdout=subprocess.PIPE) stdout=subprocess.PIPE,
)
lines = result.stdout.decode('utf-8').split('\n') lines = result.stdout.decode('utf-8').split('\n')
names = [] names = []
depends = [] depends = []
@ -73,8 +77,7 @@ class Package:
mode = line.split('=')[1] mode = line.split('=')[1]
break break
if mode not in ['host', 'cross']: if mode not in ['host', 'cross']:
logging.fatal( logging.fatal(f'Package {self.path} has an invalid mode configured: \'{mode}\'')
f'Package {self.path} has an invalid mode configured: \'{mode}\'')
exit(1) exit(1)
self.mode = mode self.mode = mode
@ -96,11 +99,16 @@ def check_prebuilts():
for ext1 in ['db', 'files']: for ext1 in ['db', 'files']:
for ext2 in ['', '.tar.xz']: for ext2 in ['', '.tar.xz']:
if not os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext1}{ext2}')): if not os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext1}{ext2}')):
result = subprocess.run(['tar', result = subprocess.run(
[
'tar',
'-czf', '-czf',
f'{repo}.{ext1}{ext2}', f'{repo}.{ext1}{ext2}',
'-T', '/dev/null'], '-T',
cwd=os.path.join('prebuilts', repo)) '/dev/null',
],
cwd=os.path.join('prebuilts', repo),
)
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed create prebuilt repos') logging.fatal('Failed create prebuilt repos')
exit(1) exit(1)
@ -108,14 +116,29 @@ def check_prebuilts():
def setup_chroot(chroot_path='/chroot/root'): def setup_chroot(chroot_path='/chroot/root'):
logging.info('Initializing root chroot') logging.info('Initializing root chroot')
create_chroot(chroot_path, packages=['base-devel'], pacman_conf='/app/local/etc/pacman.conf', extra_repos={ create_chroot(
'main': {'Server': 'file:///src/prebuilts/main'}, 'device': {'Server': 'file:///src/prebuilts/device'}}) chroot_path,
packages=['base-devel'],
pacman_conf='/app/local/etc/pacman.conf',
extra_repos={
'main': {
'Server': 'file:///src/prebuilts/main',
},
'device': {
'Server': 'file:///src/prebuilts/device',
},
},
)
logging.info('Updating root chroot') logging.info('Updating root chroot')
result = subprocess.run(pacman_cmd + result = subprocess.run(pacman_cmd + [
['--root', chroot_path, '--root',
'--arch', 'aarch64', chroot_path,
'--config', chroot_path+'/etc/pacman.conf']) '--arch',
'aarch64',
'--config',
chroot_path + '/etc/pacman.conf',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed to update root chroot') logging.fatal('Failed to update root chroot')
exit(1) exit(1)
@ -134,8 +157,16 @@ def setup_chroot(chroot_path='/chroot/root'):
file.write(data) file.write(data)
logging.info('Syncing chroot copy') logging.info('Syncing chroot copy')
result = subprocess.run( result = subprocess.run([
['rsync', '-a', '--delete', '-q', '-W', '-x', '/chroot/root/', '/chroot/copy']) 'rsync',
'-a',
'--delete',
'-q',
'-W',
'-x',
'/chroot/root/',
'/chroot/copy',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed to sync chroot copy') logging.fatal('Failed to sync chroot copy')
exit(1) exit(1)
@ -151,8 +182,7 @@ def discover_packages(path: str) -> dict[str, Package]:
for dir2 in os.listdir(os.path.join('device', dir1)): for dir2 in os.listdir(os.path.join('device', dir1)):
paths.append(os.path.join('device', dir1, dir2)) paths.append(os.path.join('device', dir1, dir2))
results = Parallel(n_jobs=multiprocessing.cpu_count()*4)( results = Parallel(n_jobs=multiprocessing.cpu_count() * 4)(delayed(Package)(path) for path in paths)
delayed(Package)(path) for path in paths)
for package in results: for package in results:
packages[package.name] = package packages[package.name] = package
@ -189,8 +219,7 @@ def discover_packages(path: str) -> dict[str, Package]:
if found: if found:
break break
if not found: if not found:
logging.fatal( logging.fatal(f'Could not find package for "{dep}"')
f'Could not find package for "{dep}"')
exit(1) exit(1)
selection = list(set(selection)) selection = list(set(selection))
packages = {package.name: package for package in selection} packages = {package.name: package for package in selection}
@ -201,7 +230,6 @@ def discover_packages(path: str) -> dict[str, Package]:
def generate_package_order(packages: list[Package]) -> list[Package]: def generate_package_order(packages: list[Package]) -> list[Package]:
unsorted = packages.copy() unsorted = packages.copy()
sorted = [] sorted = []
""" """
It goes through all unsorted packages and checks if the dependencies have already been sorted. It goes through all unsorted packages and checks if the dependencies have already been sorted.
If that is true, the package itself is added to the sorted packages If that is true, the package itself is added to the sorted packages
@ -227,13 +255,20 @@ def update_package_version_and_sources(package: Package):
It is done here already, because doing it while host-compiling takes longer. It is done here already, because doing it while host-compiling takes longer.
We decided to even pin the commit of every -git package so this won't update any version, but it would if possible. We decided to even pin the commit of every -git package so this won't update any version, but it would if possible.
""" """
cmd = makepkg_cmd+['--nobuild', '--noprepare', '--nodeps', '--skipinteg'] cmd = makepkg_cmd + [
'--nobuild',
'--noprepare',
'--nodeps',
'--skipinteg',
]
if not package.has_pkgver: if not package.has_pkgver:
cmd.append('--noextract') cmd.append('--noextract')
logging.info(f'Updating package version for {package.path}') logging.info(f'Updating package version for {package.path}')
result = subprocess.run(cmd, result = subprocess.run(
cmd,
env=makepkg_cross_env, env=makepkg_cross_env,
cwd=package.path) cwd=package.path,
)
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to update package version for {package.path}') logging.fatal(f'Failed to update package version for {package.path}')
exit(1) exit(1)
@ -242,13 +277,16 @@ def update_package_version_and_sources(package: Package):
def check_package_version_built(package: Package) -> bool: def check_package_version_built(package: Package) -> bool:
built = True built = True
result = subprocess.run(makepkg_cmd + result = subprocess.run(
['--nobuild', makepkg_cmd + [
'--nobuild',
'--noprepare', '--noprepare',
'--packagelist'], '--packagelist',
],
env=makepkg_cross_env, env=makepkg_cross_env,
cwd=package.path, cwd=package.path,
capture_output=True) capture_output=True,
)
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to get package list for {package.path}') logging.fatal(f'Failed to get package list for {package.path}')
exit(1) exit(1)
@ -264,7 +302,6 @@ def check_package_version_built(package: Package) -> bool:
def setup_dependencies_and_sources(package: Package): def setup_dependencies_and_sources(package: Package):
logging.info(f'Setting up dependencies and sources for {package.path}') logging.info(f'Setting up dependencies and sources for {package.path}')
""" """
To make cross-compilation work for almost every package, the host needs to have the dependencies installed To make cross-compilation work for almost every package, the host needs to have the dependencies installed
so that the build tools can be used so that the build tools can be used
@ -276,63 +313,89 @@ def setup_dependencies_and_sources(package: Package):
stderr=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
) )
if result.returncode != 0: if result.returncode != 0:
logging.fatal( logging.fatal(f'Failed to setup dependencies for {package.path}')
f'Failed to setup dependencies for {package.path}')
exit(1) exit(1)
result = subprocess.run(makepkg_cmd + result = subprocess.run(
['--nobuild', makepkg_cmd + [
'--nobuild',
'--holdver', '--holdver',
'--syncdeps'], '--syncdeps',
],
env=makepkg_cross_env, env=makepkg_cross_env,
cwd=package.path) cwd=package.path,
)
if result.returncode != 0: if result.returncode != 0:
logging.fatal( logging.fatal(f'Failed to check sources for {package.path}')
f'Failed to check sources for {package.path}')
exit(1) exit(1)
def build_package(package: Package): def build_package(package: Package):
makepkg_compile_opts = ['--noextract', makepkg_compile_opts = [
'--noextract',
'--skipinteg', '--skipinteg',
'--holdver', '--holdver',
'--nodeps'] '--nodeps',
]
if package.mode == 'cross': if package.mode == 'cross':
logging.info(f'Cross-compiling {package.path}') logging.info(f'Cross-compiling {package.path}')
result = subprocess.run(makepkg_cmd+makepkg_compile_opts, result = subprocess.run(
env=makepkg_cross_env | { makepkg_cmd + makepkg_compile_opts,
'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}, env=makepkg_cross_env | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'},
cwd=package.path) cwd=package.path,
)
if result.returncode != 0: if result.returncode != 0:
logging.fatal( logging.fatal(f'Failed to cross-compile package {package.path}')
f'Failed to cross-compile package {package.path}')
exit(1) exit(1)
else: else:
logging.info(f'Host-compiling {package.path}') logging.info(f'Host-compiling {package.path}')
def umount(): def umount():
subprocess.run(['umount', '-lc', '/chroot/copy'], subprocess.run(
stderr=subprocess.DEVNULL) [
'umount',
'-lc',
'/chroot/copy',
],
stderr=subprocess.DEVNULL,
)
atexit.register(umount) atexit.register(umount)
result = subprocess.run( result = subprocess.run([
['mount', '-o', 'bind', '/chroot/copy', '/chroot/copy']) 'mount',
'-o',
'bind',
'/chroot/copy',
'/chroot/copy',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal('Failed to bind mount chroot to itself') logging.fatal('Failed to bind mount chroot to itself')
exit(1) exit(1)
os.makedirs('/chroot/copy/src') os.makedirs('/chroot/copy/src')
result = subprocess.run( result = subprocess.run([
['mount', '-o', 'bind', '.', '/chroot/copy/src']) 'mount',
'-o',
'bind',
'.',
'/chroot/copy/src',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal( logging.fatal(f'Failed to bind mount folder to chroot')
f'Failed to bind mount folder to chroot')
exit(1) exit(1)
env = [f'{key}={value}' for key, value in makepkg_env.items()] env = [f'{key}={value}' for key, value in makepkg_env.items()]
result = subprocess.run( result = subprocess.run([
['arch-chroot', '/chroot/copy', '/usr/bin/env'] + env + ['/bin/bash', '-c', f'cd /src/{package.path} && makepkg --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}']) 'arch-chroot',
'/chroot/copy',
'/usr/bin/env',
] + env + [
'/bin/bash',
'-c',
f'cd /src/{package.path} && makepkg --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to host-compile package {package.path}') logging.fatal(f'Failed to host-compile package {package.path}')
exit(1) exit(1)
@ -353,7 +416,8 @@ def add_package_to_repo(package: Package):
os.path.join(package.path, file), os.path.join(package.path, file),
os.path.join(dir, file), os.path.join(dir, file),
) )
result = subprocess.run(['repo-add', result = subprocess.run([
'repo-add',
'--remove', '--remove',
'--new', '--new',
'--prevent-downgrade', '--prevent-downgrade',
@ -371,13 +435,9 @@ def add_package_to_repo(package: Package):
for ext in ['db', 'files']: for ext in ['db', 'files']:
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz')): if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz')):
os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}')) os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}'))
shutil.copyfile( shutil.copyfile(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz'), os.path.join('prebuilts', repo, f'{repo}.{ext}'))
os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz'),
os.path.join('prebuilts', repo, f'{repo}.{ext}')
)
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old')): if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old')):
os.unlink(os.path.join('prebuilts', repo, os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old'))
f'{repo}.{ext}.tar.xz.old'))
@click.group(name='packages') @click.group(name='packages')
@ -405,11 +465,9 @@ def cmd_build(verbose, path):
if len(need_build) == 0: if len(need_build) == 0:
logging.info('Everything built already') logging.info('Everything built already')
return return
logging.info('Building %s', ', '.join( logging.info('Building %s', ', '.join(map(lambda x: x.path, need_build)))
map(lambda x: x.path, need_build)))
with open('.last_built', 'w') as file: with open('.last_built', 'w') as file:
file.write('\n'.join( file.write('\n'.join(map(lambda x: x.path, need_build)))
map(lambda x: x.path, need_build)))
for package in need_build: for package in need_build:
setup_chroot() setup_chroot()
@ -422,10 +480,13 @@ def cmd_build(verbose, path):
@verbose_option @verbose_option
def cmd_clean(verbose): def cmd_clean(verbose):
setup_logging(verbose) setup_logging(verbose)
result = subprocess.run(['git', result = subprocess.run([
'git',
'clean', 'clean',
'-dffX', '-dffX',
'main', 'device']) 'main',
'device',
])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to git clean') logging.fatal(f'Failed to git clean')
exit(1) exit(1)
@ -546,8 +607,7 @@ def cmd_check(verbose, path):
reason = f'Expected to find "{key}"' reason = f'Expected to find "{key}"'
if not formatted: if not formatted:
logging.fatal( logging.fatal(f'Line {line_index+1} in {os.path.join(package.path, "PKGBUILD")} is not formatted correctly: "{line}"')
f'Line {line_index+1} in {os.path.join(package.path, "PKGBUILD")} is not formatted correctly: "{line}"')
if reason != "": if reason != "":
logging.fatal(reason) logging.fatal(reason)
exit(1) exit(1)

17
ssh.py
View file

@ -1,6 +1,6 @@
import subprocess import subprocess
import click import click
from logger import * from logger import setup_logging, verbose_option
@click.command(name='ssh') @click.command(name='ssh')
@ -8,8 +8,13 @@ from logger import *
def cmd_ssh(verbose): def cmd_ssh(verbose):
setup_logging(verbose) setup_logging(verbose)
subprocess.run(['ssh', subprocess.run([
'-o', 'GlobalKnownHostsFile=/dev/null', 'ssh',
'-o', 'UserKnownHostsFile=/dev/null', '-o',
'-o', 'StrictHostKeyChecking=no', 'GlobalKnownHostsFile=/dev/null',
'kupfer@172.16.42.1']) '-o',
'UserKnownHostsFile=/dev/null',
'-o',
'StrictHostKeyChecking=no',
'kupfer@172.16.42.1',
])

69
wrapper.py
View file

@ -13,46 +13,73 @@ else:
version = version_file.read().replace('\n', '') version = version_file.read().replace('\n', '')
tag = f'registry.gitlab.com/kupfer/kupferbootstrap:{version}' tag = f'registry.gitlab.com/kupfer/kupferbootstrap:{version}'
if version == 'dev': if version == 'dev':
result = subprocess.run([ result = subprocess.run(
[
'docker', 'docker',
'build', 'build',
'.', '.',
'-t', tag '-t',
], cwd=script_path) tag,
],
cwd=script_path,
)
if result.returncode != 0: if result.returncode != 0:
print(f'Failed to build kupferbootstrap docker image') print(f'Failed to build kupferbootstrap docker image')
exit(1) exit(1)
else: else:
# Check if the image for the version already exists # Check if the image for the version already exists
result = subprocess.run(['docker', 'images', '-q', tag], result = subprocess.run(
capture_output=True) [
'docker',
'images',
'-q',
tag,
],
capture_output=True,
)
if result.stdout == b'': if result.stdout == b'':
print( print(f'Pulling kupferbootstrap docker image version \'{version}\'')
f'Pulling kupferbootstrap docker image version \'{version}\'') subprocess.run([
subprocess.run(['docker', 'pull', tag]) 'docker',
'pull',
tag,
])
def at_exit(): def at_exit():
subprocess.run(['docker', 'kill', 'kupferbootstrap'], subprocess.run(
[
'docker',
'kill',
'kupferbootstrap',
],
stdout=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,) stderr=subprocess.DEVNULL,
)
atexit.register(at_exit) atexit.register(at_exit)
# TODO: Remove the mount of /usr/share/i18n/locales. It's a trick so we don't need to generate the locales in the chroot. # TODO: Remove the mount of /usr/share/i18n/locales. It's a trick so we don't need to generate the locales in the chroot.
# Something like a prebuilt docker image as base or copying the files from it would be good. # Something like a prebuilt docker image as base or copying the files from it would be good.
subprocess.run(['docker', subprocess.run([
'docker',
'run', 'run',
'--name', 'kupferbootstrap', '--name',
'kupferbootstrap',
'--rm', '--rm',
'--interactive', '--interactive',
'--tty', '--tty',
'--privileged', '--privileged',
'-v', f'{os.getcwd()}:/src:z', '-v',
'-v', f'{os.path.join(appdirs.user_cache_dir("kupfer"),"chroot")}:/chroot:z', f'{os.getcwd()}:/src:z',
'-v', f'{os.path.join(appdirs.user_cache_dir("kupfer"),"pacman")}:/var/cache/pacman/pkg:z', '-v',
'-v', f'{os.path.join(appdirs.user_cache_dir("kupfer"),"jumpdrive")}:/var/cache/jumpdrive:z', f'{os.path.join(appdirs.user_cache_dir("kupfer"),"chroot")}:/chroot:z',
'-v', '/dev:/dev', '-v',
f'{os.path.join(appdirs.user_cache_dir("kupfer"),"pacman")}:/var/cache/pacman/pkg:z',
'-v',
f'{os.path.join(appdirs.user_cache_dir("kupfer"),"jumpdrive")}:/var/cache/jumpdrive:z',
'-v',
'/dev:/dev',
#'-v', '/mnt/kupfer:/mnt/kupfer:z', #'-v', '/mnt/kupfer:/mnt/kupfer:z',
'-v', '/usr/share/i18n/locales:/usr/share/i18n/locales:ro'] + '-v',
[tag, '/usr/share/i18n/locales:/usr/share/i18n/locales:ro'
'kupferbootstrap'] ] + [tag, 'kupferbootstrap'] + sys.argv[1:])
+ sys.argv[1:])