parchlinux-mobile/parchbootstrap
1
0
Fork 0
forked from parchlinux-mobile/kupferbootstrap
Code Issues Pull requests Activity

fixed dep detection. TODO: add /prebuilts/$repo to host/native pacman.conf

Browse source 
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>
This commit is contained in:
InsanePrawn 2021-09-17 06:41:28 +02:00
parent ba57162f3b
commit 044b721edd
1 changed files with 61 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

130
packages.py
View file

@ -44,7 +44,7 @@ class Package:
def __init__(self, path: str, dir: str = None) -> None: def __init__(self, path: str, dir: str = None) -> None:
self.path = path self.path = path
dir = dir if dir else config['paths']['pkgbuilds'] dir = dir if dir else config.file['paths']['pkgbuilds']
self._loadinfo(dir) self._loadinfo(dir)
def _loadinfo(self, dir): def _loadinfo(self, dir):
@ -171,7 +171,7 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
visited_names = set[str]() visited_names = set[str]()
dep_levels: list[set[Package]] = [set(), set()] dep_levels: list[set[Package]] = [set(), set()]
def visited(package: Package, visited=visited, visited_names=visited_names): def visit(package: Package, visited=visited, visited_names=visited_names):
visited.add(package) visited.add(package)
visited_names.update(package.names) visited_names.update(package.names)
@ -180,20 +180,32 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
for i, level in enumerate(levels): for i, level in enumerate(levels):
result[level] = i result[level] = i
# init level 0 def get_dependencies(package: Package, package_repo: dict[str, Package] = package_repo) -> list[Package]:
for package in to_build:
visited(package)
dep_levels[0].add(package)
# add dependencies of our requested builds to level 0
for dep_name in package.depends: for dep_name in package.depends:
if dep_name in visited_names: if dep_name in visited_names:
continue continue
elif dep_name in package_repo: elif dep_name in package_repo:
dep_pkg = package_repo[dep_name] dep_pkg = package_repo[dep_name]
logging.debug(f"Adding {package.name}'s dependency {dep_name} to level 0") visit(dep_pkg)
dep_levels[0].add(dep_pkg) yield dep_pkg
visited(dep_pkg)
def get_recursive_dependencies(package: Package, package_repo: dict[str, Package] = package_repo) -> list[Package]:
for pkg in get_dependencies(package, package_repo):
yield pkg
for sub_pkg in get_recursive_dependencies(pkg, package_repo):
yield sub_pkg
logging.debug('Generating dependency chain:') logging.debug('Generating dependency chain:')
# init level 0
for package in to_build:
visit(package)
dep_levels[0].add(package)
logging.debug(f'Adding requested package {package.name}')
# add dependencies of our requested builds to level 0
for dep_pkg in get_recursive_dependencies(package):
logging.debug(f"Adding {package.name}'s dependency {dep_pkg.name} to level 0")
dep_levels[0].add(dep_pkg)
visit(dep_pkg)
""" """
Starting with `level` = 0, iterate over the packages in `dep_levels[level]`: Starting with `level` = 0, iterate over the packages in `dep_levels[level]`:
1. Moving packages that are dependencies of other packages up to `level`+1 1. Moving packages that are dependencies of other packages up to `level`+1
@ -206,17 +218,19 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
repeat_count = 0 repeat_count = 0
_last_level: set[Package] = None _last_level: set[Package] = None
while dep_levels[level]: while dep_levels[level]:
level_copy = dep_levels[level].copy()
modified = False
logging.debug(f'Scanning dependency level {level}') logging.debug(f'Scanning dependency level {level}')
if level > 100: if level > 100:
raise Exception('Dependency chain reached 100 levels depth, this is probably a bug. Aborting!') raise Exception('Dependency chain reached 100 levels depth, this is probably a bug. Aborting!')
for pkg in dep_levels[level].copy(): for pkg in level_copy:
pkg_done = False pkg_done = False
if pkg not in dep_levels[level]: if pkg not in dep_levels[level]:
# pkg has been moved, move on # pkg has been moved, move on
continue continue
# move pkg to level+1 if something else depends on it # move pkg to level+1 if something else depends on it
for other_pkg in dep_levels[level].copy(): for other_pkg in level_copy:
if pkg == other_pkg: if pkg == other_pkg:
continue continue
if pkg_done: if pkg_done:
@ -228,6 +242,7 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
dep_levels[level].remove(pkg) dep_levels[level].remove(pkg)
dep_levels[level + 1].add(pkg) dep_levels[level + 1].add(pkg)
logging.debug(f'Moving {pkg.name} to level {level+1} because {other_pkg.name} depends on it as {dep_name}') logging.debug(f'Moving {pkg.name} to level {level+1} because {other_pkg.name} depends on it as {dep_name}')
modified = True
pkg_done = True pkg_done = True
break break
for dep_name in pkg.depends: for dep_name in pkg.depends:
@ -235,9 +250,10 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
continue continue
elif dep_name in package_repo: elif dep_name in package_repo:
dep_pkg = package_repo[dep_name] dep_pkg = package_repo[dep_name]
logging.debug(f"Adding {pkg.name}'s dependency {dep_name} to level {level+1}") logging.debug(f"Adding {pkg.name}'s dependency {dep_name} to level {level}")
dep_levels[level + 1].add(dep_pkg) dep_levels[level].add(dep_pkg)
visited(dep_pkg) visit(dep_pkg)
modified = True
if _last_level == dep_levels[level]: if _last_level == dep_levels[level]:
repeat_count += 1 repeat_count += 1
@ -245,43 +261,14 @@ def generate_dependency_chain(package_repo: dict[str, Package], to_build: list[P
repeat_count = 0 repeat_count = 0
if repeat_count > 10: if repeat_count > 10:
raise Exception(f'Probable dependency cycle detected: Level has been passed on unmodifed multiple times: #{level}: {_last_level}') raise Exception(f'Probable dependency cycle detected: Level has been passed on unmodifed multiple times: #{level}: {_last_level}')
_last_level = dep_levels[level] _last_level = dep_levels[level].copy()
level += 1 if not modified: # if the level was modified, make another pass.
dep_levels.append(set[Package]()) level += 1
dep_levels.append(set[Package]())
# reverse level list into buildorder (deps first!), prune empty levels # reverse level list into buildorder (deps first!), prune empty levels
return list([lvl for lvl in dep_levels[::-1] if lvl]) return list([lvl for lvl in dep_levels[::-1] if lvl])
def generate_package_order(packages: list[Package]) -> list[Package]:
unsorted = packages.copy()
sorted = []
"""
It goes through all unsorted packages and checks if the dependencies have already been sorted.
If that is true, the package itself is added to the sorted packages
"""
while len(unsorted) > 0:
changed = False
for package in unsorted.copy():
if len(package.local_depends) == 0:
sorted.append(package)
unsorted.remove(package)
changed = True
for package in sorted:
for name in package.names:
for p in unsorted:
for dep in p.local_depends.copy():
if name == dep:
p.local_depends.remove(name)
changed = True
if not changed:
print('emergency break:', 'sorted:', repr(sorted), 'unsorted:', repr(unsorted))
sorted += unsorted
print('merged:', repr(sorted))
break
return sorted
def check_package_version_built(package: Package) -> bool: def check_package_version_built(package: Package) -> bool:
built = True built = True
@ -302,7 +289,7 @@ def check_package_version_built(package: Package) -> bool:
for line in result.stdout.decode('utf-8').split('\n'): for line in result.stdout.decode('utf-8').split('\n'):
if line != "": if line != "":
file = os.path.basename(line) file = os.path.basename(line)
if not os.path.exists(os.path.join('prebuilts', package.repo, file)): if not os.path.exists(os.path.join(config.file['paths']['packages'], package.repo, file)):
built = False built = False
return built return built
@ -375,7 +362,7 @@ def setup_dependencies_and_sources(package: Package, chroot: str, repo_dir: str
[os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + [ [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + [
'--nobuild', '--nobuild',
'--holdver', '--holdver',
'--syncdeps', '--nodeps',
], ],
env=makepkg_cross_env | {'PACMAN_CHROOT': chroot}, env=makepkg_cross_env | {'PACMAN_CHROOT': chroot},
cwd=os.path.join(repo_dir, package.path), cwd=os.path.join(repo_dir, package.path),
@ -389,7 +376,7 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable
'--noextract', '--noextract',
'--skipinteg', '--skipinteg',
'--holdver', '--holdver',
'--nodeps', '--syncdeps',
] ]
repo_dir = repo_dir if repo_dir else config.file['paths']['pkgbuilds'] repo_dir = repo_dir if repo_dir else config.file['paths']['pkgbuilds']
chroot = setup_build_chroot(arch=arch) chroot = setup_build_chroot(arch=arch)
@ -423,7 +410,7 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable
result = subprocess.run( result = subprocess.run(
[os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_compile_opts, [os.path.join(chroot, 'usr/bin/makepkg')] + makepkg_cmd[1:] + makepkg_compile_opts,
env=makepkg_cross_env | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'}, env=makepkg_cross_env | {'QEMU_LD_PREFIX': '/usr/aarch64-linux-gnu'},
cwd=os.path.join(dir, package.path), cwd=os.path.join(repo_dir, package.path),
) )
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed to cross-compile package {package.path}') logging.fatal(f'Failed to cross-compile package {package.path}')
@ -461,16 +448,16 @@ def build_package(package: Package, repo_dir: str = None, arch='aarch64', enable
def add_package_to_repo(package: Package): def add_package_to_repo(package: Package):
logging.info(f'Adding {package.path} to repo') logging.info(f'Adding {package.path} to repo')
dir = os.path.join('prebuilts', package.repo) binary_dir = os.path.join(config.file['paths']['packages'], package.repo)
if not os.path.exists(dir): pkgbuild_dir = os.path.join(config.file['paths']['pkgbuilds'], package.path)
os.mkdir(dir) os.makedirs(binary_dir, exist_ok=True)
for file in os.listdir(package.path): for file in os.listdir(pkgbuild_dir):
# Forced extension by makepkg.conf # Forced extension by makepkg.conf
if file.endswith('.pkg.tar.xz'): if file.endswith('.pkg.tar.xz'):
shutil.move( shutil.move(
os.path.join(package.path, file), os.path.join(pkgbuild_dir, file),
os.path.join(dir, file), os.path.join(binary_dir, file),
) )
result = subprocess.run([ result = subprocess.run([
'repo-add', 'repo-add',
@ -478,22 +465,24 @@ def add_package_to_repo(package: Package):
'--new', '--new',
'--prevent-downgrade', '--prevent-downgrade',
os.path.join( os.path.join(
'prebuilts', binary_dir,
package.repo,
f'{package.repo}.db.tar.xz', f'{package.repo}.db.tar.xz',
), ),
os.path.join(dir, file), os.path.join(binary_dir, file),
]) ])
if result.returncode != 0: if result.returncode != 0:
logging.fatal(f'Failed add package {package.path} to repo') logging.fatal(f'Failed add package {package.path} to repo')
exit(1) exit(1)
for repo in REPOSITORIES: for repo in REPOSITORIES:
for ext in ['db', 'files']: for ext in ['db', 'files']:
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz')): if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz')):
os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}')) os.unlink(os.path.join(binary_dir, f'{repo}.{ext}'))
shutil.copyfile(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz'), os.path.join('prebuilts', repo, f'{repo}.{ext}')) shutil.copyfile(
if os.path.exists(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old')): os.path.join(binary_dir, f'{repo}.{ext}.tar.xz'),
os.unlink(os.path.join('prebuilts', repo, f'{repo}.{ext}.tar.xz.old')) os.path.join(binary_dir, f'{repo}.{ext}'),
)
if os.path.exists(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old')):
os.unlink(os.path.join(binary_dir, f'{repo}.{ext}.tar.xz.old'))
@click.group(name='packages') @click.group(name='packages')
@ -519,19 +508,19 @@ def cmd_build(paths, arch='aarch64'):
for packages in package_levels: for packages in package_levels:
level = set[Package]() level = set[Package]()
for package in packages: for package in packages:
if not check_package_version_built(package) or package.depends in build_names: if (not check_package_version_built(package)) or set.intersection(set(package.depends), set(build_names)):
level.add(package) level.add(package)
build_names.update(package.names) build_names.update(package.names)
if level: if level:
build_levels.append(level) build_levels.append(level)
logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.path for p in level]))) logging.debug(f'Adding to level {i}:' + '\n' + ('\n'.join([p.name for p in level])))
i += 1 i += 1
if not build_levels: if not build_levels:
logging.info('Everything built already') logging.info('Everything built already')
return return
for level, need_build in enumerate(build_levels): for level, need_build in enumerate(build_levels):
logging.info(f"(Level {level}) Building {', '.join([x.path for x in need_build])}") logging.info(f"(Level {level}) Building {', '.join([x.name for x in need_build])}")
crosscompile = config.file['build']['crosscompile'] crosscompile = config.file['build']['crosscompile']
for package in need_build: for package in need_build:
build_package(package, arch=arch, enable_crosscompile=crosscompile) build_package(package, arch=arch, enable_crosscompile=crosscompile)
@ -694,3 +683,6 @@ def cmd_check(paths):
cmd_packages.add_command(cmd_build) cmd_packages.add_command(cmd_build)
cmd_packages.add_command(cmd_clean) cmd_packages.add_command(cmd_clean)
cmd_packages.add_command(cmd_check) cmd_packages.add_command(cmd_check)
cmd_packages.add_command(cmd_build)
cmd_packages.add_command(cmd_clean)
cmd_packages.add_command(cmd_check)