Include mirrorbits

Exposes a mirrorbits instance under mirror.msys2.org and includes
some mirrors. This is still an experiment at this point, so not
sure if everything will work as needed.

Author: lazka

.gitignore

@@ -1 +1,2 @@
 /github_token.env
+/geopip.env

docker-compose.yml

@@ -164,10 +164,51 @@ services:
       - "traefik.tcp.routers.rsync.entrypoints=rsync"
     restart: always
 
+  redis:
+    image: redis:6-alpine
+    restart: unless-stopped
+    hostname: redis
+
+  geoipupdate:
+    image: maxmindinc/geoipupdate
+    restart: unless-stopped
+    env_file:
+        geopip.env
+    environment:
+      - GEOIPUPDATE_EDITION_IDS=GeoLite2-City GeoLite2-ASN
+      - GEOIPUPDATE_FREQUENCY=24
+    volumes:
+      - geoip-data:/usr/share/GeoIP
+
+  mirrorbits:
+    build: ./mirrorbits
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - geoipupdate
+    volumes:
+      - geoip-data:/usr/share/GeoIP:ro
+      - /srv/msys2repo:/repo-volume:ro
+      - ./mirrorbits/mirrorbits.conf:/etc/mirrorbits.conf:ro
+      - ./mirrorbits/add_mirrors.sh:/add_mirrors.sh:ro
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.middlewares.mirrorbits-scheme-redir.redirectscheme.scheme=https"
+      # http
+      - "traefik.http.routers.mirrorbits.rule=Host(`mirror.msys2.org`)"
+      - "traefik.http.routers.mirrorbits.entrypoints=web"
+      - "traefik.http.routers.mirrorbits.middlewares=mirrorbits-scheme-redir"
+      # https
+      - "traefik.http.routers.mirrorbits-secure.rule=Host(`mirror.msys2.org`)"
+      - "traefik.http.routers.mirrorbits-secure.tls=true"
+      - "traefik.http.routers.mirrorbits-secure.tls.certresolver=le"
+      - "traefik.http.routers.mirrorbits-secure.entrypoints=web-secure"
+
 volumes:
   letsencrypt:
   staging-gnupg:
   staging-repos:
+  geoip-data:
 
 networks:
   default:

geopip.env.example

@@ -0,0 +1,2 @@
+GEOIPUPDATE_ACCOUNT_ID=
+GEOIPUPDATE_LICENSE_KEY=

mirrorbits/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:focal as build
+
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    pkg-config zlib1g-dev protobuf-compiler libprotoc-dev git make golang && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/etix/mirrorbits.git && \
+    cd mirrorbits && \
+    git checkout eeea0e026d0ca99c7009a91a54369f7c29e87049
+RUN cd mirrorbits && \
+    mkdir -p dist/usr/bin && \
+    make install PREFIX=/usr DESTDIR=dist
+
+FROM ubuntu:focal
+
+COPY --from=build /mirrorbits/dist /
+
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    rsync redis-tools ca-certificates tzdata && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY init.sh /init.sh
+CMD bash /init.sh
+EXPOSE 80

mirrorbits/add_mirrors.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+mirrorbits add -http=https://mirror.yandex.ru/mirrors/msys2/ -rsync=rsync://mirror.yandex.ru/mirrors/msys2/ yandex.ru
+mirrorbits enable yandex.ru
+
+mirrorbits add -http=https://mirror.selfnet.de/msys2/ -rsync=rsync://mirror.selfnet.de/msys2/ selfnet.de
+mirrorbits enable selfnet.de
+
+mirrorbits add -country-only -http=https://mirrors.tuna.tsinghua.edu.cn/msys2/ -rsync=rsync://mirror.selfnet.de/msys2/ tsinghua.edu.cn
+mirrorbits enable tsinghua.edu.cn
+
+mirrorbits add -http=https://ftp.acc.umu.se/mirror/msys2.org/ -rsync=rsync://ftp.acc.umu.se/mirror/msys2.org/ umu.se
+mirrorbits enable umu.se
+
+mirrorbits add -http=https://quantum-mirror.hu/mirrors/pub/msys2/ -rsync=rsync://quantum-mirror.hu/msys2/ quantum-mirror.hu
+mirrorbits enable quantum-mirror.hu
+
+mirrorbits add -http=https://mirrors.dotsrc.org/msys2/ -rsync=rsync://mirrors.dotsrc.org/msys2/ dotsrc.org
+mirrorbits enable dotsrc.org
+
+mirrorbits add -http=https://mirror.ufro.cl/msys2/ -rsync=rsync://mirror.ufro.cl/msys2/ ufro.cl
+mirrorbits enable ufro.cl

mirrorbits/init.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+while [ ! -f /usr/share/GeoIP/GeoLite2-ASN.mmdb ] || [ ! -f /usr/share/GeoIP/GeoLite2-ASN.mmdb ]
+do
+    echo "waiting for GeoIP data to be loaded"
+    sleep 2
+done
+
+# Throw out the redis db, mirrorbits gets confused easily when adding/removing mirrors
+redis-cli -h redis FLUSHALL
+
+mirrorbits daemon --debug &
+pid=$!
+
+while ! mirrorbits list
+do
+  echo "waiting for server"
+  sleep 3
+done
+
+# remove all mirrors
+mirrorbits list -state=false | tail -n +2 | xargs --no-run-if-empty -n1 mirrorbits remove -f
+
+# add all mirrors
+if [ -f "add_mirrors.sh" ]; then
+    source add_mirrors.sh
+fi
+
+wait $pid

mirrorbits/mirrorbits.conf

@@ -0,0 +1,23 @@
+Repository: /repo-volume
+Templates: /usr/share/mirrorbits/
+OutputMode: auto
+ListenAddress: :80
+Gzip: false
+RedisAddress: redis:6379
+GeoipDatabasePath: /usr/share/GeoIP/
+ConcurrentSync: 5
+ScanInterval: 30
+CheckInterval: 5
+RepositoryScanInterval: 5
+Hashes:
+    SHA1: Off
+    SHA256: Off
+    MD5: Off
+DisallowRedirects: false
+WeightDistributionRange: 4
+DisableOnMissingFile: false
+MaxLinkHeaders: 10
+Fallbacks:
+    - URL: https://repo.msys2.org/
+      CountryCode: cz
+      ContinentCode: eu