Upgrade monitor

dev/docker-compose.yml

@@ -33,7 +33,7 @@ services:
       JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
       POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@postgresdb:5432/zimfarm
       ALEMBIC_UPGRADE_HEAD_ON_START: "true"
-      # upload logs to receiver but logs, zimcheck results and artificats to s3
+      # upload zims to receiver but logs, zimcheck results and artificats to s3
       ZIM_UPLOAD_URI: sftp://uploader@receiver:22/zim/
       LOGS_UPLOAD_URI: s3+http://minio:9000/?keyId=minio_key&secretAccessKey=minio_secret&bucketName=zimfarm-logs
       ZIMCHECK_RESULTS_UPLOAD_URI: s3+http://minio:9000/?keyId=minio_key&secretAccessKey=minio_secret&bucketName=zimfarm-zimchecks
@@ -142,6 +142,48 @@ services:
     profiles:
       - worker
 
+  monitor:
+    build: ./netdata
+    container_name: zf_monitor
+    restart: unless-stopped
+    ports:
+      - 127.0.0.1:8004:19999
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - apparmor:unconfined
+    environment:
+      - ZIMFARM_API_URL=http://backend:80/v2
+      - ZIMFARM_USERNAME=admin
+      - ZIMFARM_PASSWORD=admin
+      - NETDATA_HOSTNAME=monitor
+    volumes:
+      - ./netdataconfig/cloud.d:/var/lib/netdata/cloud.d
+      - netdatalib:/var/lib/netdata
+      - netdatacache:/var/cache/netdata
+      - ./netdata:/host/root:ro,rslave
+      - /etc/passwd:/host/etc/passwd:ro
+      - /etc/group:/host/etc/group:ro
+      - /etc/localtime:/etc/localtime:ro
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /etc/os-release:/host/etc/os-release:ro
+      - /var/log:/host/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      backend:
+        condition: service_healthy
+    profiles:
+      - worker
+
+  task_monitor:
+    build:
+      context: ../monitor
+    image: zf_task_monitor
+    container_name: zf_task_monitor
+    entrypoint: ["/bin/sh", "-c"]
+    command: "exit"
+
   watcher:
     build: ../watcher
     container_name: zf_watcher
@@ -166,12 +208,15 @@ services:
         condition: service_healthy
       task_worker:
         condition: service_started
+
     command: worker-manager --webapi-uri 'http://backend:80/v2' --username test_worker --name test_worker
     environment:
       - DEBUG=true
       - TASK_WORKER_IMAGE=zf_task_worker
       - ENVIRONMENT=development
       - POLL_INTERVAL=10
+      - MONITORING_DEST=monitor:19999
+      - MONITOR_IMAGE=zf_task_monitor
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./test_worker-identity/id_ed25519:/etc/ssh/keys/zimfarm
@@ -233,3 +278,5 @@ services:
 volumes:
   pg_data_zimfarm:
   minio_data_zimfarm:
+  netdatalib:
+  netdatacache:

dev/frontend-ui-dev/config.json

@@ -1,3 +1,4 @@
 {
-    "ZIMFARM_WEBAPI": "http://localhost:8000/v2"
+  "ZIMFARM_WEBAPI": "http://localhost:8000/v2",
+  "MONITORING_URL": "http://localhost:8004/v3"
 }

dev/netdata/Dockerfile

@@ -0,0 +1,34 @@
+FROM ghcr.io/netdata/netdata:v2.8.0
+LABEL org.opencontainers.image.source=https://github.com/kiwix/container-images
+
+RUN apt-get update && apt-get install -y wget cron procps psmisc && rm -rf /var/lib/apt/lists/*
+
+ENV DO_NOT_TRACK=1
+# CIDRs to allow regarless of whether in zimfarm API. Separate with spaces
+ENV STREAM_WHITELIST=""
+ENV NETDATA_HOSTNAME=monitoring.openzim.org
+ENV STREAM_KEY=7DC0DDB8-8EAA-4391-AF5D-BE017EECF0EE
+
+COPY netdata.conf /etc/netdata/netdata.conf
+COPY update-stream-whitelist.sh /usr/local/bin/update-stream-whitelist.sh
+COPY regen-stream-conf.py /usr/local/bin/regen-stream-conf
+COPY entrypoint.sh /usr/local/bin/entrypoint
+
+# disable all external modules and setup cron
+RUN printf "enabled: no\n" | tee /etc/netdata/python.d.conf /etc/netdata/go.d.conf /etc/netdata/node.d.conf \
+    && printf "#!/bin/bash\nrm -rf /var/cache/netdata/* && kill 1\n" > /usr/local/bin/reset-netdata \
+    && chmod a+x /usr/local/bin/reset-netdata \
+    && chmod a+x /usr/local/bin/regen-stream-conf \
+    && chmod a+x /usr/local/bin/update-stream-whitelist.sh \
+    && touch /etc/netdata/.opt-out-from-anonymous-statistics \
+    && echo "DOCKER_USR=netdata" > /etc/cron.d/update-stream-whitelist \
+    && echo "NETDATA_LISTENER_PORT=19999" >> /etc/cron.d/update-stream-whitelist \
+    && echo "*/15 * * * * root /usr/local/bin/update-stream-whitelist.sh >> /var/log/cron.log 2>&1" >> /etc/cron.d/update-stream-whitelist \
+    && chmod 0644 /etc/cron.d/update-stream-whitelist \
+    && touch /etc/netdata/stream.conf \
+    && chmod a+x /usr/local/bin/entrypoint \
+    && wget https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \
+    && mv dumb-init_1.2.5_x86_64 /usr/bin/dumb-init \
+    && chmod +x /usr/bin/dumb-init
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "entrypoint"]

dev/netdata/cloud.conf

@@ -0,0 +1,2 @@
+[global]
+  enabled = no

dev/netdata/entrypoint.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# setup custom hostname for node in netdata
+if [ ! -z "${NETDATA_HOSTNAME}" ]
+then
+    printf "\n hostname = ${NETDATA_HOSTNAME}\n" >> /etc/netdata/netdata.conf
+fi
+
+# Define the netdata user
+DOCKER_USR="${DOCKER_USR:-netdata}"
+
+# copy of netdata's entrypoint
+BALENA_PGID=$(stat -c %g /var/run/balena.sock 2>/dev/null || true)
+DOCKER_PGID=$(stat -c %g /var/run/docker.sock 2>/dev/null || true)
+
+re='^[0-9]+$'
+if [[ $BALENA_PGID =~ $re ]]; then
+  echo "Netdata detected balena-engine.sock"
+  DOCKER_HOST='unix:///var/run/balena-engine.sock'
+  PGID="$BALENA_PGID"
+elif [[ $DOCKER_PGID =~ $re ]]; then
+  echo "Netdata detected docker.sock"
+  DOCKER_HOST="unix:///var/run/docker.sock"
+  PGID="$DOCKER_PGID"
+fi
+export PGID
+export DOCKER_HOST
+
+if [ -n "${PGID}" ]; then
+  echo "Creating docker group ${PGID}"
+  addgroup --gid "${PGID}" "docker" 2>/dev/null || echo >&2 "Could not add group docker with ID ${PGID}, its already there probably"
+  echo "Assign netdata user to docker group ${PGID}"
+  usermod -a -G "docker" "${DOCKER_USR}" 2>/dev/null || echo >&2 "Could not add netdata user to group docker with ID ${PGID}"
+fi
+
+if mountpoint -q /etc/netdata && [ -z "$(ls -A /etc/netdata)" ]; then
+  echo "Copying stock configuration to /etc/netdata"
+  cp -a /etc/netdata.stock/. /etc/netdata
+fi
+
+# Run update-stream-whitelist on startup
+echo "Running initial stream whitelist update"
+/usr/local/bin/update-stream-whitelist.sh &
+
+# Start cron in the background
+echo "Starting cron daemon"
+cron
+
+# Start netdata in the foreground
+echo "Starting netdata"
+exec /usr/sbin/netdata -u "${DOCKER_USR}" -D -s /host -p "${NETDATA_LISTENER_PORT}"

dev/netdata/netdata.conf

@@ -0,0 +1,54 @@
+# netdata can generate its own config which is available at 'http://<netdata_ip>/netdata.conf'
+# You can download it with command like: 'wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf'
+
+# This is configured as a streaming parent - stores only data from child agents, no local collection
+
+[web]
+ bind to = *
+
+[ml]
+ enabled = no
+
+[health]
+ enabled = no
+
+[plugins]
+ # Disable ALL local data collection plugins
+ cgroups = no
+ tc = no
+ enable running new plugins = no
+ check for new plugins every = 72000
+ python.d = no
+ charts.d = no
+ node.d = no
+ go.d = no
+ apps = no
+ proc = no
+ idlejitter = no
+ diskspace = no
+ timex = no
+ ioping = no
+ fping = no
+ nfacct = no
+ cups = no
+ perf = no
+ slabinfo = no
+ debugfs = no
+ ebpf = no
+ systemdunits = no
+ statsd = no
+ netdata monitor = no
+
+[global]
+ # Use dbengine to store streamed data from children
+ memory mode = dbengine
+ # Don't allocate memory for local host metrics
+ page size cache = 32
+ dbengine disk space = 1024
+ # Allocate storage for child nodes
+ dbengine multihost disk space = 8192
+ update every = 1
+ cleanup obsolete charts after seconds = 2592000
+ cleanup orphan hosts after seconds = 2592000
+ delete obsolete charts files = no
+ delete orphan hosts files = no