From: Martin Pitt Date: Thu, 2 May 2024 05:58:57 +0000 (+0200) Subject: consors-report.py: updates X-Git-Url: https://piware.de/gitweb/?p=bin.git;a=commitdiff_plain;h=HEAD;hp=b7614d3eb98e0a9421f2b5bfc8fa499b35792b34 consors-report.py: updates --- diff --git a/backup b/backup index 7a42486..f1bef09 100755 --- a/backup +++ b/backup @@ -1,8 +1,14 @@ #!/bin/sh set -eu cd $HOME -LOG=.cache/duplicity/log +LOG=.cache/backup/log PATH=$PATH:/sbin:/usr/sbin +RESTIC="restic --password-file $HOME/.config/backup-passphrase --repo sftp:piware.de:backup/restic" + +fail() { + notify-send -i /usr/share/icons/Adwaita/48x48/status/network-error-symbolic.symbolic.png -u critical -t 180000 "${1:-BACKUP FAILED!}" + exit 1 +} # do backup every day if [ -e "$LOG" ] && [ $(( `date +%s` - `stat -c %Y $LOG` )) -lt 86300 ]; then @@ -27,6 +33,17 @@ fi notify-send "Backup started" mkdir -p $(dirname $LOG) -env PASSPHRASE="$(cat ~/.backup-passphrase)" duplicity --allow-source-mismatch --full-if-older-than 1M --exclude-filelist .duplicity-ignore . rsync://piware.de/backup/laptop >> $LOG || { notify-send "BACKUP FAILED!"; exit 1; } + +$RESTIC backup --exclude-file=$HOME/.config/backup-ignore $HOME >> $LOG 2>&1 || fail +# TODO: forget --prune policy: https://restic.readthedocs.io/en/stable/060_forget.html notify-send "Backup finished successfully" -duplicity remove-all-but-n-full 6 --force rsync://piware.de/backup/laptop + +scp .config/backup-passphrase piware.de:.cache/ +ssh piware.de chmod u+w .cache/backup-passphrase +trap "ssh piware.de shred -u .cache/backup-passphrase" EXIT INT QUIT PIPE + +ssh piware.de restic --password-file .cache/backup-passphrase --repo backup/restic forget --prune --keep-within-hourly 24h --keep-within-daily 7d --keep-within-weekly 30d --keep-within-monthly 12m +notify-send "Backup pruned successfully" + +ssh piware.de restic --password-file .cache/backup-passphrase --repo backup/restic check || fail "BACKUP CHECK FAILED!" +notify-send "Backup checked successfully" diff --git a/build-cockpit-toolbox b/build-cockpit-toolbox index 3b0ae86..39844bd 100755 --- a/build-cockpit-toolbox +++ b/build-cockpit-toolbox @@ -4,8 +4,8 @@ NAME=${NAME:-cockpit} TAG=${TAG:-latest} toolbox rm --force $NAME || true -podman pull quay.io/cockpit/tasks:${TAG} -yes | toolbox create --image quay.io/cockpit/tasks:${TAG} -c $NAME +podman pull ghcr.io/cockpit-project/tasks:${TAG} +yes | toolbox create --image ghcr.io/cockpit-project/tasks:${TAG} -c $NAME # install some extra development and desktop tools toolbox run -c "$NAME" sh -exc ' @@ -16,42 +16,17 @@ sudo hostname -F /etc/hostname # enable manpages sudo sed -i s/nodocs// /etc/dnf/dnf.conf -sudo dnf install -y ansible man-db man-pages moreutils fd-find ripgrep gh \ +sudo dnf install -y ansible bash-completion man-db man-pages moreutils fd-find ripgrep gh git-delta neovim \ python3-boto python3-boto3 python3-openstacksdk libnotify \ - pandoc texlive-ec texlive-pdfjam texlive-beamer \ - /usr/bin/scanimage /usr/bin/pngquant /usr/bin/convert \ - kpcli \ + simple-scan /usr/bin/scanimage /usr/bin/pngquant /usr/bin/convert pdfmerge \ + cargo rustfmt clippy \ calibre qt5-qtwayland \ - chromium dbus-daemon flatpak-builder flatpak-spawn - -# HACK: tmt-provision-virtual has way too heavy dependencies: https://bugzilla.redhat.com/show_bug.cgi?id=2093717 -cat < /tmp/spec -Name: dummy-deps -Summary: dummy dependencies -Version: 999 -Release: 1%{?dist} -# https://bugzilla.redhat.com/show_bug.cgi?id=2093717 -Provides: python3-libguestfs -Provides: libguestfs-tools-c -# also for tmt-provision-virtual -Provides: libvirt -License: BSD - -%description -Dummy dependencies to avoid bloat. - -%files -EOF -rpmbuild --define "_topdir /tmp/rpmbuild-dummy" -bb /tmp/spec -sudo rpm -i --verbose /tmp/rpmbuild-dummy/RPMS/x86_64/dummy-deps-999-1.fc*.x86_64.rpm -rm -rf /tmp/rpmbuild-dummy /tmp/spec - -sudo dnf install -y --setopt=install_weak_deps=False tmt tmt-provision-virtual - -# run podman in toolbox -printf "flatpak-spawn --host podman \42\$@\42\n" | sudo tee /usr/local/bin/podman >/dev/null -sudo chmod a+x /usr/local/bin/podman - -# debugging cockpit crashes -sudo dnf debuginfo-install -y glib2 glibc libssh gnutls + tmt flatpak-builder flatpak-spawn \ + fedpkg centpkg + +# commands to forward to the host +for cmd in podman toolbox nmcli eog evince; do + printf "#!/bin/sh\nexec flatpak-spawn --host $cmd \42\$@\42\n" | sudo tee /usr/local/bin/$cmd >/dev/null + sudo chmod a+x /usr/local/bin/$cmd +done ' diff --git a/build-debian-toolbox b/build-debian-toolbox index 13ee84d..c1d139a 100755 --- a/build-debian-toolbox +++ b/build-debian-toolbox @@ -17,11 +17,19 @@ echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/unsafe-io # otherwise installing systemd fails umount /var/log/journal +# enable sources +if [ -e /etc/apt/sources.list.d/debian.sources ]; then + sed -i "/^Types:/ s/deb$/deb deb-src/" /etc/apt/sources.list.d/debian.sources +fi + apt-get update apt-get install -y libnss-myhostname sudo eatmydata libcap2-bin # allow sudo with empty password sed -i "s/nullok_secure/nullok/" /etc/pam.d/common-auth + +# unbreak slow host name resolution +sed -i "/^hosts:/ s/files dns myhostname/files myhostname dns/" /etc/nsswitch.conf ' toolbox run --container $RELEASE sh -exc ' @@ -30,13 +38,19 @@ toolbox run --container $RELEASE sh -exc ' echo "${ID}-${VERSION_ID:-sid}" | sudo tee /etc/hostname sudo hostname -F /etc/hostname -sudo eatmydata apt-get -y dist-upgrade +sudo DEBIAN_FRONTEND=noninteractive eatmydata apt-get -y dist-upgrade # development tools -sudo eatmydata apt-get install -y --no-install-recommends build-essential git-buildpackage libwww-perl less vim lintian debhelper manpages-dev git dput pristine-tar bash-completion wget gnupg ubuntu-dev-tools python3-debian fakeroot libdistro-info-perl openssh-client +sudo eatmydata apt-get install -y --no-install-recommends build-essential git-buildpackage libwww-perl less vim lintian debhelper manpages-dev git dput pristine-tar bash-completion wget gnupg ubuntu-dev-tools python3-debian fakeroot libdistro-info-perl openssh-client flatpak-xdg-utils # autopkgtest sudo eatmydata apt-get install -y --no-install-recommends autopkgtest qemu-system-x86 qemu-utils genisoimage + +# commands to forward to the host +for cmd in podman toolbox; do + printf "#!/bin/sh\n/usr/libexec/flatpak-xdg-utils/flatpak-spawn --host $cmd \42\$@\42\n" | sudo tee /usr/local/bin/$cmd >/dev/null + sudo chmod a+x /usr/local/bin/$cmd +done ' toolbox enter --container $RELEASE diff --git a/ci-clear-npm-cache b/ci-clear-npm-cache deleted file mode 100755 index e73e4d9..0000000 --- a/ci-clear-npm-cache +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -set -eux -cd ~/upstream/cockpituous/ansible -for cloud in openstack_tasks e2e; do - ansible -f15 -i inventory -m shell -a 'podman exec cockpit-tasks-1 rm -rf /work/.npm/_cacache/' $cloud -done diff --git a/consors-report.py b/consors-report.py new file mode 100755 index 0000000..84e21b7 --- /dev/null +++ b/consors-report.py @@ -0,0 +1,106 @@ +#!/usr/bin/python3 + +import argparse +import collections +import csv +import itertools +import re +from collections import namedtuple +from pathlib import Path +from typing import Iterable + +Entry = namedtuple('Entry', ['category', 'date', 'valuta', 'who', 'iban', 'bic', 'type', 'desc', 'value']) + +CATEGORIES = { + 'Gehalt/Steuern': re.compile('Gehalt/Rente|RED HAT|Finanzamt|FK Guenzburg', re.I), + 'Wohnung': re.compile('Telekom|hands4home|Goetzfried|Green Planet Energy|Beitragsservice.*Rundfunk', re.I), + 'Medizin': re.compile(r'Apotheke|MVZ|\bmed\b|ZAB Abrechnung|Caika|HNOeins|PVS|Dr\..*Sellier|' + r'BFS Health|Streifeneder|Labor|Physio|(Drescher.*Lung)|Osteopath|' + '(Dr.*Borchers)|(Debeka.*Überweisung)|(DKV.*Überweisung)|Beihilfe|Klinik', re.I), + 'Versicherung': re.compile('((debeka|DKV|Hallesche|Versicherung|Alte Leipziger|ConceptIF|' + 'Baloise).*Lastschrift)|Hallesche.*Bonu', re.I), + 'Transport': re.compile('DB Vertrieb|Deutsche Bahn|Nextbike|Carsharing|Radstation', re.I), + 'Lebensmittel': re.compile('BIOS|Bäcker|Baecker|Ruta|Rewe', re.I), + 'Eigentumswohnungen': re.compile('Rechnung Darl.-Leistung|Semmelweis', re.I), + 'Hobby/Sport': re.compile('Holstein|Mrs. Sporty|Kieser|DJK|Dimaso', re.I), + 'Sparen': re.compile('Bausparkasse Mainz AG|FIL Fondsbank|MIG (Fonds|GmbH)|Netfonds AG', re.I), + 'Spenden': re.compile('Spende|Signal Foundation|nebenan.de|(paypal.*Sabine.Hossenf)|' + 'Patreon|Umwelthilfe|Foerderer|Malteser|(Aktion Tier.*Mensch)|' + 'campact|Amnesty', re.I), +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description='Read Consorsbank CSV and generate report') + parser.add_argument('-d', '--date', metavar='REGEX', default='.', help='date filter regex') + parser.add_argument('-s', '--summary', action='store_true', help='only show category sums') + parser.add_argument('csv', type=Path, nargs='+', help='CSV files to parse') + return parser.parse_args() + + +def get_category(item: str) -> str: + for category, pattern in CATEGORIES.items(): + if pattern.search(item): + return category + return 'Sonstiges' + + +def parse_entry(raw_fields: Iterable[str]) -> Entry: + fields = [f.strip() for f in raw_fields] + # format change in May 2024, adds a 9th field "Währung"; ignore + fields = fields[:8] + # last field is the value, parse as float + value = float(fields.pop().replace('.', '').replace(',', '.')) + # match on who, IBAN, type, or desc + category = get_category(fields[2] + fields[3] + fields[5] + fields[6]) + return Entry._make([category, *fields, value]) + + +def parse_csv(path: Path, date_filter: str) -> Iterable[Entry]: + filter_re = re.compile(date_filter, re.I) + + def filter_entry(entry: Entry): + # ignore from/to Tagesgeldkonto + if entry.iban == 'DE13760300800853589101': + return None + return filter_re.search(entry.date) + + with path.open() as f: + reader = csv.reader(f, delimiter=';') + next(reader) # skip header + # first line is the column headers, chop it off + entries = map(parse_entry, reader) + # do the actual iteration here, as the files get closed afterwards + return list(filter(filter_entry, entries)) + + +def main(): + args = parse_args() + + entries = itertools.chain.from_iterable(map(lambda p: parse_csv(p, args.date), args.csv)) + by_category = collections.defaultdict(lambda: []) + category_sum = collections.defaultdict(lambda: 0.0) + balance = 0.0 + for e in entries: + by_category[e.category].append(e) + category_sum[e.category] += e.value + balance += e.value + + if args.summary: + for cat in sorted(by_category): + print(f'| {category_sum[cat]:>8.2f} | {cat:20} |') + print('|----------|----------------------|') + print(f'| {balance:>8.2f} | {"Saldo":20} |') + + else: + for cat in sorted(by_category): + print(f'## {cat}: {category_sum[cat]:.2f}\n') + for entry in by_category[cat]: + print(f'| {entry.date:7} | {entry.value:>7.2f} | _{entry.type}_ - {entry.who} - {entry.desc} |') + print() + + print(f'## Saldo: {balance:.2f}') + + +if __name__ == '__main__': + main() diff --git a/debian-backport-cockpit b/debian-backport-cockpit index 109f848..a0284ed 100755 --- a/debian-backport-cockpit +++ b/debian-backport-cockpit @@ -20,14 +20,14 @@ upload() { rm -r ${PACKAGE}-* } +rm -rf /tmp/backport mkdir /tmp/backport cd /tmp/backport apt-get source $PACKAGE cd ${PACKAGE}-* -# bullseye -series_ver bullseye-backports -[ ! -x debian/adjust-for-release ] || debian/adjust-for-release bullseye -dch --local ~bpo11+ --distribution bullseye-backports --force-distribution "No-change backport to Debian Bullseye" -dpkg-buildpackage -S -sd -nc -v$ver +# bookworm +series_ver bookworm-backports +dch --local ~bpo12+ --distribution bookworm-backports --force-distribution "No-change backport to Debian Bookworm" +debuild -S -sd -nc -v$ver upload diff --git a/gomuks b/gomuks new file mode 100755 index 0000000..ebe585d --- /dev/null +++ b/gomuks @@ -0,0 +1,3 @@ +#!/bin/sh +printf '\033]2;gomuks\a' +TERM=xterm ~/.local/bin/gomuks diff --git a/install-rhel-tools b/install-rhel-tools index 7953c72..741e0cf 100755 --- a/install-rhel-tools +++ b/install-rhel-tools @@ -2,23 +2,14 @@ # Install RHEL development tools into a toolbox set -eux -sudo curl -o /etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt https://password.corp.redhat.com/RH-IT-Root-CA.crt +sudo curl -o /etc/pki/ca-trust/source/anchors/2015-RH-IT-Root-CA.crt https://certs.corp.redhat.com/certs/2015-IT-Root-CA.pem +sudo curl -o /etc/pki/ca-trust/source/anchors/2022-RH-IT-Root-CA.crt https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem + sudo update-ca-trust (cd /etc/yum.repos.d; sudo curl -O --location https://download.devel.redhat.com/rel-eng/RCMTOOLS/rcm-tools-fedora.repo) -sudo dnf install -y rhpkg - -# CentOS stream -sudo dnf copr enable -y james/centpkg -sudo dnf install -y centpkg streamkoji +sudo dnf install -y rhel-packager # coverity sudo dnf copr enable -y copr.devel.redhat.com/kdudka/covscan sudo dnf install -y covscan-client - -# 1minutetip: https://wiki.test.redhat.com/BaseOs/Projects/1minuteTIP -if mountpoint /mnt; then sudo umount /mnt; fi # HACK: rhts-test-env-5.0-2.fc32eng.noarch wants to fiddle with it -. /etc/os-release -sudo wget -O /etc/yum.repos.d/qa-tools.repo https://copr.devel.redhat.com/coprs/lpol/qa-tools/repo/fedora-36/lpol-qa-tools-fedora-${VERSION_ID}.repo -sudo wget -O /etc/yum.repos.d/beaker-client.repo http://download.lab.bos.redhat.com/beakerrepos/beaker-client-Fedora.repo -sudo dnf install -y qa-tools-workstation-1minutetip tmt-redhat-provision-minute diff --git a/install-tex b/install-tex new file mode 100755 index 0000000..62d2a6f --- /dev/null +++ b/install-tex @@ -0,0 +1,6 @@ +#!/bin/sh +if command -v apt; then + sudo apt install -y texlive-lang-german texlive-latex-base texlive-latex-extra +else + sudo dnf install texlive-german texlive-german texlive-a4wide texlive-dinbrief texlive-latex texlive-hyphen-german pandoc texlive-ec texlive-pdfjam texlive-beamer +fi diff --git a/process-photos b/process-photos new file mode 100755 index 0000000..219edf3 --- /dev/null +++ b/process-photos @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +for f in *.JPG; do + mv "$f" "${f%JPG}jpg" +done + +for f in *.jpg; do + echo "$f" + chmod 644 "$f" + info="$(exiv2 -Pnv "$f")" + # copy tags to IPTC description (sigal) + tags=$(echo "$info" | sed -n 's/^subject *//p' | tr '[:upper:]' '[:lower:]') + if [ -n "$tags" ]; then + exiv2 -M"add Iptc.Application2.Caption String $tags" mo "$f" + fi + + # rename to EXIF date, if available + if echo "$info" | grep -q '^DateTime'; then + exiv2 mv "$f" + fi +done diff --git a/scandoc b/scandoc index f62373c..ebbce60 100755 --- a/scandoc +++ b/scandoc @@ -22,6 +22,11 @@ while true; do [ "$x" = q ] && break || true done -pdfjam --outfile "$name" --fitpaper true --rotateoversize false page*.pdf +# pdfmerge does not get along with just one input file +if [ $PAGE -eq 1 ]; then + mv page*.pdf "$name" +else + pdfmerge page*.pdf "$name" +fi rm -rf "$D" diff --git a/ubuntu-backport-cockpit b/ubuntu-backport-cockpit index 8d46bf8..c9e90e7 100755 --- a/ubuntu-backport-cockpit +++ b/ubuntu-backport-cockpit @@ -25,15 +25,18 @@ upload() { rm -r ${PACKAGE}-* } +rm -rf /tmp/backport mkdir /tmp/backport cd /tmp/backport apt-get source $PACKAGE cd ${PACKAGE}-* case "$TARGET" in + noble) VER=24.04 ;; + mantic) VER=23.10 ;; + lunar) VER=23.04 ;; + kinetic) VER=22.10 ;; jammy) VER=22.04 ;; - impish) VER=21.10 ;; - focal) VER=20.04 ;; *) echo "Unknown target $TARGET" >&2; exit 1 ;; esac