From f95d16c996b3ba8b9ba96b0a9d822b8fd8fd4c3e Mon Sep 17 00:00:00 2001 From: InsanePrawn Date: Mon, 4 Oct 2021 13:33:15 +0200 Subject: [PATCH] fix crosscompile -> add makepkg_cross_{arch}.conf generation TODO: use everywhere --- constants.py | 18 +++++ generator.py | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++ packages.py | 20 ++++-- 3 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 generator.py diff --git a/constants.py b/constants.py index 27d1f7f..c491c8e 100644 --- a/constants.py +++ b/constants.py @@ -78,6 +78,11 @@ KUPFER_HTTPS = 'https://gitlab.com/kupfer/packages/prebuilts/-/raw/main/$repo' DistroArch = TargetArch = Arch +COMPILE_ARCHES: dict[Arch, str] = { + 'x86_64': 'amd64', + 'aarch64': 'arm64', +} + GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { 'x86_64': { 'x86_64': 'x86_64-pc-linux-gnu', @@ -88,4 +93,17 @@ GCC_HOSTSPECS: dict[DistroArch, dict[TargetArch, str]] = { } } +CFLAGS_GENERAL = ['-O2', '-pipe', '-fstack-protector-strong'] +CFLAGS_ARCHES: dict[Arch, list[str]] = { + 'x86_64': ['-march=x86-64', '-mtune=generic'], + 'aarch64': [ + '-march=armv8-a', + '-fexceptions', + '-Wp,-D_FORTIFY_SOURCE=2', + '-Wformat', + '-Werror=format-security', + '-fstack-clash-protection', + ] +} + CROSSDIRECT_PKGS = ['crossdirect', 'qemu-user-static-bin', 'binfmt-qemu-static'] diff --git a/generator.py b/generator.py new file mode 100644 index 0000000..6727df3 --- /dev/null +++ b/generator.py @@ -0,0 +1,189 @@ +from constants import GCC_HOSTSPECS, CFLAGS_GENERAL, CFLAGS_ARCHES, COMPILE_ARCHES +from config import config + + +def generate_makepkg_conf(arch: str, cross: bool = False, chroot: str = None) -> str: + """ + Generate a makepkg.conf. For use with crosscompiling, specify `cross=True` and pass as `chroot` + the relative path inside the native chroot where the foreign chroot will be mounted. + """ + hostspec = GCC_HOSTSPECS[config.runtime['arch']][arch] + cflags = CFLAGS_ARCHES[arch] + CFLAGS_GENERAL + if cross and not chroot: + raise Exception('Cross-compile makepkg conf requested but no chroot path given: "{chroot}"') + conf = f''' +#!/hint/bash +# +# /etc/makepkg.conf +# + +######################################################################### +# SOURCE ACQUISITION +######################################################################### +# +#-- The download utilities that makepkg should use to acquire sources +# Format: 'protocol::agent' +DLAGENTS=('file::/usr/bin/curl -qgC - -o %o %u' + 'ftp::/usr/bin/curl -qgfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' + 'http::/usr/bin/curl -qgb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'https::/usr/bin/curl -qgb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'rsync::/usr/bin/rsync --no-motd -z %u %o' + 'scp::/usr/bin/scp -C %u %o') + +# Other common tools: +# /usr/bin/snarf +# /usr/bin/lftpget -c +# /usr/bin/wget + +#-- The package required by makepkg to download VCS sources +# Format: 'protocol::package' +VCSCLIENTS=('bzr::bzr' + 'fossil::fossil' + 'git::git' + 'hg::mercurial' + 'svn::subversion') + +######################################################################### +# ARCHITECTURE, COMPILE FLAGS +######################################################################### +# +CARCH="{arch}" +CHOST="{hostspec}" + +#-- Compiler and Linker Flags +# -march (or -mcpu) builds exclusively for an architecture +# -mtune optimizes for an architecture, but builds for whole processor family +CPPFLAGS="" +CFLAGS="{' '.join(cflags)}" +CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS" +LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now" +#RUSTFLAGS="-C opt-level=2" +#-- Make Flags: change this for DistCC/SMP systems +#MAKEFLAGS="-j2" +#-- Debugging flags +DEBUG_CFLAGS="-g -fvar-tracking-assignments" +DEBUG_CXXFLAGS="-g -fvar-tracking-assignments" +#DEBUG_RUSTFLAGS="-C debuginfo=2" + +######################################################################### +# BUILD ENVIRONMENT +######################################################################### +# +# Makepkg defaults: BUILDENV=(!distcc !color !ccache !check !sign) +# A negated environment option will do the opposite of the comments below. +# +#-- distcc: Use the Distributed C/C++/ObjC compiler +#-- color: Colorize output messages +#-- ccache: Use ccache to cache compilation +#-- check: Run the check() function if present in the PKGBUILD +#-- sign: Generate PGP signature file +# +BUILDENV=(!distcc color !ccache !check !sign) +# +#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, +#-- specify a space-delimited list of hosts running in the DistCC cluster. +#DISTCC_HOSTS="" +# +#-- Specify a directory for package building. +#BUILDDIR=/tmp/makepkg + +######################################################################### +# GLOBAL PACKAGE OPTIONS +# These are default values for the options=() settings +######################################################################### +# +# Makepkg defaults: OPTIONS=(!strip docs libtool staticlibs emptydirs !zipman !purge !debug !lto) +# A negated option will do the opposite of the comments below. +# +#-- strip: Strip symbols from binaries/libraries +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- staticlibs: Leave static library (.a) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +#-- debug: Add debugging flags as specified in DEBUG_* variables +#-- lto: Add compile flags for building with link time optimization +# +OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug !lto) + +#-- File integrity checks to use. Valid: md5, sha1, sha224, sha256, sha384, sha512, b2 +INTEGRITY_CHECK=(sha256) +#-- Options to be used when stripping binaries. See `man strip' for details. +STRIP_BINARIES="--strip-all" +#-- Options to be used when stripping shared libraries. See `man strip' for details. +STRIP_SHARED="--strip-unneeded" +#-- Options to be used when stripping static libraries. See `man strip' for details. +STRIP_STATIC="--strip-debug" +#-- Manual (man and info) directories to compress (if zipman is specified) +MAN_DIRS=({'{usr{,/local}{,/share},opt/*}/{man,info}'}) +#-- Doc directories to remove (if !docs is specified) +DOC_DIRS=({'usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}'}) +#-- Files to be removed from all packages (if purge is specified) +PURGE_TARGETS=({'usr/{,share}/info/dir .packlist *.pod'}) +#-- Directory to store source code in for debug packages +DBGSRCDIR="/usr/src/debug" + +######################################################################### +# PACKAGE OUTPUT +######################################################################### +# +# Default: put built package and cached source in build directory +# +#-- Destination: specify a fixed directory where all packages will be placed +#PKGDEST=/home/packages +#-- Source cache: specify a fixed directory where source files will be cached +#SRCDEST=/home/sources +#-- Source packages: specify a fixed directory where all src packages will be placed +#SRCPKGDEST=/home/srcpackages +#-- Log files: specify a fixed directory where all log files will be placed +#LOGDEST=/home/makepkglogs +#-- Packager: name/email of the person or organization building packages +#PACKAGER="John Doe " +#-- Specify a key to use for package signing +#GPGKEY="" + +######################################################################### +# COMPRESSION DEFAULTS +######################################################################### +# +COMPRESSGZ=(gzip -c -f -n) +COMPRESSBZ2=(bzip2 -c -f) +COMPRESSXZ=(xz -T0 -c -z -) +COMPRESSZST=(zstd -c -z -q -) +COMPRESSLRZ=(lrzip -q) +COMPRESSLZO=(lzop -q) +COMPRESSZ=(compress -c -f) +COMPRESSLZ4=(lz4 -q) +COMPRESSLZ=(lzip -c -f) + +######################################################################### +# EXTENSION DEFAULTS +######################################################################### +# +PKGEXT='.pkg.tar.xz' +SRCEXT='.src.tar.gz' + +######################################################################### +# OTHER +######################################################################### +# +#-- Command used to run pacman as root, instead of trying sudo and su +#PACMAN_AUTH=() +''' + if cross: + chroot = chroot.lstrip('/') + includes = f'-I/usr/{hostspec}/usr/include -I/{chroot}/usr/include' + libs = f'-L/usr/{hostspec}/lib -L/{chroot}/usr/lib' + conf += f''' + +export ARCH="{COMPILE_ARCHES[arch]}" +export CROSS_COMPILE="{hostspec}-" +export CC="{hostspec}-gcc {includes} {libs}" +export CXX="{hostspec}-g++ {includes} {libs}" +export CFLAGS="$CFLAGS {includes}" +export CXXFLAGS="$CXXFLAGS {includes}" +export LDFLAGS="$LDFLAGS,-L/usr/{hostspec}/lib,-L/{chroot}/usr/lib,-rpath-link,/usr/{hostspec}/lib,-rpath-link,/{chroot}/usr/lib" +''' + + return conf diff --git a/packages.py b/packages.py index ba2f0da..06f4980 100644 --- a/packages.py +++ b/packages.py @@ -4,14 +4,16 @@ import multiprocessing import os import shutil import subprocess +from copy import deepcopy +from joblib import Parallel, delayed + from constants import REPOSITORIES, CROSSDIRECT_PKGS, GCC_HOSTSPECS from config import config from chroot import create_chroot, run_chroot_cmd, try_install_packages, mount_crossdirect -from joblib import Parallel, delayed from distro import get_kupfer_local from wrapper import enforce_wrap, check_programs_wrap from utils import mount, umount -from copy import deepcopy +from generator import generate_makepkg_conf makepkg_env = os.environ.copy() | { 'LANG': 'C', @@ -355,6 +357,7 @@ def build_package( makepkg_compile_opts = [ '--holdver', ] + makepkg_conf_path = 'etc/makepkg.conf' repo_dir = repo_dir if repo_dir else config.get_path('pkgbuilds') foreign_arch = config.runtime['arch'] != arch target_chroot = setup_build_chroot(arch=arch, extra_packages=package.depends) @@ -375,13 +378,19 @@ def build_package( logging.info(f'Cross-compiling {package.path}') build_root = native_chroot makepkg_compile_opts += ['--nodeps'] - env = makepkg_cross_env + #env = makepkg_cross_env + env = makepkg_env if enable_ccache: env['PATH'] = f"/usr/lib/ccache:{env['PATH']}" logging.info('Setting up dependencies for cross-compilation') try_install_packages(package.depends + ['crossdirect', f"{GCC_HOSTSPECS[config.runtime['arch']][arch]}-gcc"], native_chroot) # mount foreign arch chroot inside native chroot - chroot_mount_path = os.path.join(native_chroot, 'chroot', os.path.basename(target_chroot)) + chroot_relative = os.path.join('chroot', os.path.basename(target_chroot)) + chroot_mount_path = os.path.join(native_chroot, chroot_relative) + makepkg_cross_conf = generate_makepkg_conf(arch, cross=True, chroot=chroot_relative) + makepkg_conf_path = os.path.join('etc', f'makepkg_cross_{arch}.conf') + with open(os.path.join(native_chroot, makepkg_conf_path), 'w') as f: + f.write(makepkg_cross_conf) os.makedirs(chroot_mount_path) mount(target_chroot, chroot_mount_path) else: @@ -407,7 +416,8 @@ def build_package( if result.returncode != 0: raise Exception(f'Failed to bind mount pkgdirs to {build_root}/src') - build_cmd = f'cd /src/{package.path} && echo $PATH && makepkg --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' + makepkg_conf_absolute = os.path.join('/', makepkg_conf_path) + build_cmd = f'cd /src/{package.path} && makepkg --config {makepkg_conf_absolute} --needed --noconfirm --ignorearch {" ".join(makepkg_compile_opts)}' result = run_chroot_cmd(build_cmd, chroot_path=build_root, inner_env=env) umount_result = umount(src_dir) if umount_result != 0: