New superseeder

bittorrent-superseeder/Dockerfile

@@ -0,0 +1,26 @@
+FROM debian:buster-slim
+
+MAINTAINER Emmanuel Engelhart <[email protected]>
+
+ENV QBITTORRENT_PORT=8080
+
+WORKDIR /tmp
+
+# All-in-One RUN for a very small image size (< 5 MB)
+RUN apt update 2> /dev/null && \
+    apt install -y --no-install-recommends apt-transport-https curl gnupg ca-certificates cron gron xml2 && \
+
+    curl -fsSL 'https://dl.cloudsmith.io/public/qbittorrent-cli/qbittorrent-cli/gpg.F8756541ADDA2B7D.key' | apt-key add - && \
+    curl "https://repos.fedarovich.com/debian/buster/qbittorrent-cli.list" --output /etc/apt/sources.list.d/qbittorrent-cli.list && \
+    apt update && \
+    apt install -y --no-install-recommends qbittorrent-cli && \
+
+    apt remove -y --purge gnupg && \
+    apt clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install start script
+COPY bin/ /usr/local/bin/
+RUN chmod -R 0500 /usr/local/bin/*
+
+ENTRYPOINT start.sh "$QBITTORRENT_PORT"

bittorrent-superseeder/README.md

@@ -0,0 +1,68 @@
+## openZIM BitTorrent Super-seeder
+
+The **openZIM BitTorrent Super-seeder** is a Docker image to easily
+launch a BitTorrent super-seeder for all or part of the ZIM files
+published at https://library.kiwix.org/zim.
+
+It will share theses files which means concretly automatically:
+* Download new ones
+* Delete older one
+
+### Principle
+
+This Docker image works as a companion to the
+[linuxserver/qbittorrent](https://hub.docker.com/r/linuxserver/qbittorrent)
+Docker image. It is responsible the ZIM files served by
+[qBittorrent](https://www.qbittorrent.org/) are kept in sync with
+upstream https://library.kiwix.org/zim.
+
+### Configure
+
+The instance can be configurd via environnment variables:
+* `DOWNLOAD_DIRECTORY_PATH`: directory path you want to have the ZIM files stored
+* `DATA_PORT`: TCP & UDP ports for the data exchanges
+* `ADMIN_PORT`: TCP admin port for qBittorrent
+* `DOWNLOAD_DIRECTORY_MODE=ro` if the ZIM files are already downloaded and you don't want any data to be downloaded from qBittorrent.
+
+You can gather this conf in an environment file for example and run for example:
+```bash
+env $(env .env) docker-compose ...
+```
+
+### Run
+
+The easiest solution to run the solution is to use
+[docker-compose](https://docs.docker.com/compose/). Doing so, you will
+be able with a simple command to start/stop the two Docker containers
+(qBittorrent and the super-seeder companion).
+
+To launch the service:
+```bash
+docker-compose up -d
+```
+
+To stop it:
+```bash
+docker-compose stop
+```
+
+### Advanced
+
+Here are a bit of documentation for developers or users wanting to go
+a bit more in depth with the solution.
+
+#### qBittorrent
+
+Refers to https://hub.docker.com/r/linuxserver/qbittorrent
+
+#### Companion
+
+To launch the companion:
+```bash
+docker run --network="host" --name=qbittorrent-zim-superseeder openzim/qbittorrent-zim-superseeder
+```
+
+To build the companion from the source code:
+```bash
+docker build -t openzim/qbittorrent-zim-superseeder .
+```

bittorrent-superseeder/bin/start.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+QBITTORRENT_PORT=$1
+
+echo "Update crontab for ZIM synchronisation"
+printf "
+@reboot root /usr/local/bin/sync_superseeder.sh ${QBITTORRENT_PORT} >> /var/log/sync_superseeder.log
+0 * * * * root /usr/local/bin/sync_superseeder.sh ${QBITTORRENT_PORT} >> /var/log/sync_superseeder.log
+" > /etc/crontab
+
+service cron start
+
+echo "Listening to cron logs..."
+sleep 2
+tail -f /var/log/sync_superseeder.log

bittorrent-superseeder/bin/sync_superseeder.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+set -e
+
+QBT_PORT=$1
+FEED_URL="https://library.kiwix.org/catalog/search?count=20"
+CURL="curl -s"
+QBT="qbt"
+QBT_CREDENTIALS="--username admin --password adminadmin --url http://qbittorrent:${QBT_PORT}"
+
+ONLINE_ZIM_URLS="/dev/shm/online_zim_urls.tsv"
+ONLINE_ZIM_PATHS="/dev/shm/online_zim_paths.tsv"
+
+LOCAL_ZIMS="/dev/shm/local_zims.tsv"
+LOCAL_ZIM_PATHS="/dev/shm/local_zim_paths.tsv"
+LOCAL_ONLY_ZIM_PATHS="/dev/shm/local_only_zim_paths.tsv"
+
+DOWNLOAD_ZIM_PATHS="/dev/shm/download_zim_paths.tsv"
+DOWNLOAD_ZIM_URLS="/dev/shm/download_zim_urls.tsv"
+
+DOWNLOAD_PATH="/downloads"
+DOWNLOAD_PATH_4_SED=$(printf '%s\n' "${DOWNLOAD_PATH}" | sed -e 's/[\/&]/\\&/g')
+
+# Clean temporary files
+for FILE in ${ONLINE_ZIM_URLS} ${ONLINE_ZIM_PATHS} ${LOCAL_ZIMS} ${LOCAL_ZIM_PATHS} ${LOCAL_ONLY_ZIM_PATHS} ${DOWNLOAD_ZIM_PATHS} ${DOWNLOAD_ZIM_URLS}
+do
+  rm -f ${FILE}
+  touch ${FILE}
+done
+
+# Retrieve online ZIMs
+${CURL} ${FEED_URL} | xml2 | grep '^/feed/entry/link/@href=.*\.zim.*$' | sed 's/\.meta4$//' | sed 's/\/feed\/entry\/link\/@href=//' | sort > ${ONLINE_ZIM_URLS}
+cat ${ONLINE_ZIM_URLS} | sed 's/^http.*:\/\/[^/]*//' | sort > ${ONLINE_ZIM_PATHS}
+
+# Retrieve local ZIMs
+${QBT} torrent list --format=json ${QBT_CREDENTIALS} | gron | sed 's/^json\[[[:digit:]]\+\]\.//' | grep -P '^(save_path|name|infohash_v1)' | sed 's/^.* = ["]*//' | sed 's/["]*;$//' | sed 'N;N;s/\n/\t/g' | awk '{ printf ("%s\t%s%s\n", $1, $3, $2) }' > ${LOCAL_ZIMS}
+cat ${LOCAL_ZIMS} | cut -f2 | sed "s/^${DOWNLOAD_PATH_4_SED}//" | sort > ${LOCAL_ZIM_PATHS}
+
+# Compute ZIMs to sync
+comm -13 ${LOCAL_ZIM_PATHS} ${ONLINE_ZIM_PATHS} > ${DOWNLOAD_ZIM_PATHS}
+for ZIM in `cat ${DOWNLOAD_ZIM_PATHS}`
+do
+  ZIM_URL=`grep -P "http[s]*://[^/]+${ZIM}$" ${ONLINE_ZIM_URLS}`
+
+  if [ ! -z "${ZIM_URL}" ]
+  then
+    echo "${ZIM_URL}.torrent" >> ${DOWNLOAD_ZIM_URLS}
+  fi
+done
+
+# Download ZIMs
+for URL in `cat ${DOWNLOAD_ZIM_URLS}`
+do
+  echo "Downloading ${URL}..."
+  ZIM_PATH=`echo ${URL} | sed 's/^http.*:\/\/[^/]*//' | sed 's/[^/]*$//'`
+  ${QBT} torrent add url ${URL} --folder="${DOWNLOAD_PATH}${ZIM_PATH}" ${QBT_CREDENTIALS}
+done
+
+# Compute ZIMs to delete
+comm -23 ${LOCAL_ZIM_PATHS} ${ONLINE_ZIM_PATHS} > ${LOCAL_ONLY_ZIM_PATHS}
+
+# Delete old ZIMs
+for ZIM in `cat ${LOCAL_ONLY_ZIM_PATHS}`
+do
+    echo "Deleting ${ZIM}..."
+    ZIM_HASH=`grep -P ${ZIM} ${LOCAL_ZIMS} | cut -f1`
+    ${QBT} torrent delete ${ZIM_HASH} ${PURGE_ZIM_FILES} ${QBT_CREDENTIALS}
+done

bittorrent-superseeder/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.5'
+services:
+
+  qbittorrent:
+    image: linuxserver/qbittorrent:4.4.1
+    container_name: qbittorrent
+    volumes:
+      - "${DOWNLOAD_DIRECTORY_PATH:?Specify a DOWNLOAD_DIRECTORY_PATH environment value}:/downloads:${DOWNLOAD_DIRECTORY_MODE:-rw}"
+    ports:
+      - "${DATA_PORT:-6881}:6881/udp"
+      - "${DATA_PORT:-6881}:6881/tcp"
+      - "${ADMIN_PORT:-8080}:${ADMIN_PORT:-8080}/tcp"
+    environment:
+      - WEBUI_PORT=${ADMIN_PORT:-8080}
+    restart: always
+
+  qbittorrent-zim-superseeder:
+    image: openzim/qbittorrent-zim-superseeder
+    container_name: qbittorrent-zim-superseeder
+    depends_on:
+      - "qbittorrent"
+    environment:
+      - QBITTORRENT_PORT=${ADMIN_PORT:-8080}
+    restart: always
+
+networks:
+  default:
+    name: qbittorrent-zim-superseeder