diff options
author | Luke Shumaker <lukeshu@parabola.nu> | 2018-06-22 13:22:44 -0400 |
---|---|---|
committer | Luke Shumaker <lukeshu@parabola.nu> | 2018-10-07 19:36:55 -0400 |
commit | c3a616f64fb29804695440811489e547e1f56506 (patch) | |
tree | fca57706d58782ffff5036b2cb87829bb7527241 | |
parent | 310f65479d6d139045a4aea7bc1d5fea3ccd12bb (diff) |
Add from Parabola: db-check-*
If you're seeing this in `git blame`, it chose to follow the wrong
ancestor of a merge commit.
-rw-r--r-- | README.md | 11 | ||||
-rwxr-xr-x | db-check-nonfree | 47 | ||||
-rwxr-xr-x | db-check-package-libraries | 203 | ||||
-rwxr-xr-x | db-check-repo-sanity | 57 | ||||
-rwxr-xr-x | db-check-unsigned-packages | 38 | ||||
-rwxr-xr-x | db-check-unsigned-packages.py | 96 |
6 files changed, 452 insertions, 0 deletions
@@ -14,6 +14,11 @@ The executables that you (might) care about are: │ ├── integrity-check │ ├── make_repo_torrents [Parabola only] │ └── sourceballs + ├── db-check-nonfree [Parabola only] + ├── db-check-package-libraries [Parabola only] + ├── db-check-repo-sanity [Parabola only] + ├── db-check-unsigned-packages [Parabola only] + ├── db-check-unsigned-packages.py [Parabola only] ├── db-import-archlinuxarm-src [Parabola only] ├── db-import-any [Parabola only] ├── db-import-pkg [Parabola only] @@ -45,6 +50,12 @@ have - `cron-jobs/integrity-check` +Instead of enhancing `integrity-check`, Parabola developers have +decided to write multiple stand-alone tools that should probably be +merged into `integrity-check` + + - `db-check-*` + When we remove a package from a repository, it stays in the package "pool". We would like to be able to eventually remove packages from the pool, to reclaim the disk space: diff --git a/db-check-nonfree b/db-check-nonfree new file mode 100755 index 0000000..7cbeb8f --- /dev/null +++ b/db-check-nonfree @@ -0,0 +1,47 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -ge 1 ]; then + error "Calling %s with a specific repository is not supported" "${0##*/}" + exit 1 +fi + +# TODO: this might lock too much (architectures) +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 + done +done + +msg "Check nonfree in repo:" +libreblacklist update +nonfree=($(libreblacklist cat | libreblacklist get-pkg | sort -u)) +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + msg2 "%s %s" "$repo" "$pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Nonfree: %s" "${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" + fi + done +done + +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" + done +done diff --git a/db-check-package-libraries b/db-check-package-libraries new file mode 100755 index 0000000..72b7db9 --- /dev/null +++ b/db-check-package-libraries @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski <mtjm@mtjm.eu> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + print("adding package:", package) + with tempfile.TemporaryDirectory(None, "db-check-package-libraries."+os.path.basename(package)+".") as temp: + subprocess.Popen(("bsdtar", "xf", package, "-C", temp)).communicate() + subprocess.Popen(('find', temp, + '-type', 'd', + '(', '-not', '-readable', '-o', '-not', '-executable', ')', + '-exec', 'chmod', '755', '--', '{}', ';')).communicate() + subprocess.Popen(('find', temp, + '-type', 'f', + '-not', '-readable', + '-exec', 'chmod', '644', '--', '{}', ';')).communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + if os.path.islink(path) or not os.path.isfile(path): + continue + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="db-check-package-libraries", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/db-check-repo-sanity b/db-check-repo-sanity new file mode 100755 index 0000000..239f042 --- /dev/null +++ b/db-check-repo-sanity @@ -0,0 +1,57 @@ +#!/bin/bash +# Solves issue165... on the old roundup install. From the database +# backups, the title was "Older/deprecated packages never leave the +# repo", I don't know how the body of the issue is stored in the DB, +# but the title says enough, I think. + +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" + +# Traverse all repos +for _repo in "${PKGREPOS[@]}"; do + msg "Cleaning up [%s]" "${_repo}" + + # Find all pkgnames on this repo's abs + on_abs=($( + find "${SVNREPO}/${_repo}" -name PKGBUILD | \ + while read pkgbuild; do + source "${pkgbuild}" >/dev/null 2>&1 + # cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + + # also cleanup package functions + for _pkg in "${pkgname[@]}"; do + unset "package_${pkg}" >/dev/null 2>&1 + done + + # this fills the on_abs array + echo "${pkgname[@]}" + done + )) + + # quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[%s]'s ABS tree is empty, skipping" "${_repo}" + break + fi + + # Find all pkgnames on repos + on_repo=($( + find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ + -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + + # Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) + + # Remove them from databases, ftpdir-cleanup will take care of the rest + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ + -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 + + msg2 "Removed the following packages:" + plain '%s' "${remove[@]}" +done diff --git a/db-check-unsigned-packages b/db-check-unsigned-packages new file mode 100755 index 0000000..0fc053b --- /dev/null +++ b/db-check-unsigned-packages @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski <mtjm@mtjm.eu> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 1 ]; then + msg "usage: %s <architecture>" "${0##*/}" + exit 1 +fi + +arch=$1 +shift + +for repo in "${PKGREPOS[@]}" +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-check-unsigned-packages.py" "$repo" "$@" < "$db" +done diff --git a/db-check-unsigned-packages.py b/db-check-unsigned-packages.py new file mode 100755 index 0000000..625c0e8 --- /dev/null +++ b/db-check-unsigned-packages.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski <mtjm@mtjm.eu> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. +""" + + +import base64 +import subprocess +import sys +import tarfile + + +def main(): + """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + key = None + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + key = b"" + if check_keys: + continue + else: + break + if line == b"%ARCH%\n": + is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) + if skip: + continue + print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 + + +if __name__ == "__main__": + main() |