Skip to content

Commit b492a97

Browse files
s4kes4ke
authored
feat: add experimental Docker Swarm support (#376)
1 parent 12a4434 commit b492a97

File tree

12 files changed

+420
-14
lines changed

12 files changed

+420
-14
lines changed

‎.github/workflows/publish_on_master.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,7 @@ jobs:
2626
tags: ${{ github.repository_owner }}/hcloud-csi-driver:latest
2727
cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache
2828
cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max
29+
- name: "make docker plugin"
30+
run: |
31+
cd deploy/docker-swarm/pkg
32+
make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=latest-swarm

‎.github/workflows/publish_on_tag.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ jobs:
4242
tags: ${{ github.repository_owner }}/hcloud-csi-driver:${{ steps.release_version.outputs.value}}
4343
cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache
4444
cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max
45+
- name: "make docker plugin"
46+
env:
47+
RELEASE_VERSION: ${{ steps.release_version.outputs.value}}
48+
run: |
49+
cd deploy/docker-swarm/pkg
50+
make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=$RELEASE_VERSION-swarm
4551
- name: Run GoReleaser
4652
uses: goreleaser/goreleaser-action@v2
4753
with:

‎cmd/aio/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
This contains an all in one binary (aio). This is required
2+
for orchestrators such as Docker Swarm which need all endpoints in a single
3+
API.

‎cmd/aio/main.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"strconv"
7+
"strings"
8+
9+
proto "github.com/container-storage-interface/spec/lib/go/csi"
10+
"github.com/go-kit/kit/log"
11+
"github.com/go-kit/kit/log/level"
12+
"github.com/hetznercloud/csi-driver/api"
13+
"github.com/hetznercloud/csi-driver/app"
14+
"github.com/hetznercloud/csi-driver/driver"
15+
"github.com/hetznercloud/csi-driver/volumes"
16+
"github.com/hetznercloud/hcloud-go/hcloud/metadata"
17+
)
18+
19+
var logger log.Logger
20+
21+
func main() {
22+
logger = app.CreateLogger()
23+
24+
m := app.CreateMetrics(logger)
25+
26+
hcloudClient, err := app.CreateHcloudClient(m.Registry(), logger)
27+
if err != nil {
28+
level.Error(logger).Log(
29+
"msg", "failed to initialize hcloud client",
30+
"err", err,
31+
)
32+
os.Exit(1)
33+
}
34+
35+
metadataClient := metadata.NewClient(metadata.WithInstrumentation(m.Registry()))
36+
37+
// node
38+
serverID, err := metadataClient.InstanceID()
39+
if err != nil {
40+
level.Error(logger).Log("msg", "failed to fetch server ID from metadata service", "err", err)
41+
os.Exit(1)
42+
}
43+
44+
serverAZ, err := metadataClient.AvailabilityZone()
45+
if err != nil {
46+
level.Error(logger).Log("msg", "failed to fetch server availability-zone from metadata service", "err", err)
47+
os.Exit(1)
48+
}
49+
parts := strings.Split(serverAZ, "-")
50+
if len(parts) != 2 {
51+
level.Error(logger).Log("msg", fmt.Sprintf("unexpected server availability zone: %s", serverAZ), "err", err)
52+
os.Exit(1)
53+
}
54+
serverLocation := parts[0]
55+
56+
level.Info(logger).Log("msg", "Fetched data from metadata service", "id", serverID, "location", serverLocation)
57+
58+
volumeMountService := volumes.NewLinuxMountService(
59+
log.With(logger, "component", "linux-mount-service"),
60+
)
61+
volumeResizeService := volumes.NewLinuxResizeService(
62+
log.With(logger, "component", "linux-resize-service"),
63+
)
64+
volumeStatsService := volumes.NewLinuxStatsService(
65+
log.With(logger, "component", "linux-stats-service"),
66+
)
67+
nodeService := driver.NewNodeService(
68+
log.With(logger, "component", "driver-node-service"),
69+
strconv.Itoa(serverID),
70+
serverLocation,
71+
volumeMountService,
72+
volumeResizeService,
73+
volumeStatsService,
74+
)
75+
76+
// controller
77+
volumeService := volumes.NewIdempotentService(
78+
log.With(logger, "component", "idempotent-volume-service"),
79+
api.NewVolumeService(
80+
log.With(logger, "component", "api-volume-service"),
81+
hcloudClient,
82+
),
83+
)
84+
controllerService := driver.NewControllerService(
85+
log.With(logger, "component", "driver-controller-service"),
86+
volumeService,
87+
serverLocation,
88+
)
89+
90+
// common
91+
identityService := driver.NewIdentityService(
92+
log.With(logger, "component", "driver-identity-service"),
93+
)
94+
95+
// common
96+
listener, err := app.CreateListener()
97+
if err != nil {
98+
level.Error(logger).Log(
99+
"msg", "failed to create listener",
100+
"err", err,
101+
)
102+
os.Exit(1)
103+
}
104+
105+
grpcServer := app.CreateGRPCServer(logger, m.UnaryServerInterceptor())
106+
107+
// controller
108+
proto.RegisterControllerServer(grpcServer, controllerService)
109+
// common
110+
proto.RegisterIdentityServer(grpcServer, identityService)
111+
// node
112+
proto.RegisterNodeServer(grpcServer, nodeService)
113+
114+
m.InitializeMetrics(grpcServer)
115+
116+
identityService.SetReady(true)
117+
118+
if err := grpcServer.Serve(listener); err != nil {
119+
level.Error(logger).Log(
120+
"msg", "grpc server failed",
121+
"err", err,
122+
)
123+
os.Exit(1)
124+
}
125+
}

‎deploy/docker-swarm/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
plugin

‎deploy/docker-swarm/README.md‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Docker Swarm Hetzner CSI plugin
2+
3+
Currently in Beta. Please consult the Docker Swarm documentation
4+
for cluster volumes (=CSI) support at https://github.com/moby/moby/blob/master/docs/cluster_volumes.md
5+
6+
The community is tracking the state of support for CSI in Docker Swarm over at https://github.com/olljanat/csi-plugins-for-docker-swarm
7+
8+
## How to install the plugin
9+
10+
Run the following steps on all nodes (especially master nodes).
11+
The simplest way to achieve this
12+
13+
1. Create a read+write API token in the [Hetzner Cloud Console](https://console.hetzner.cloud/).
14+
15+
2. Install the plugin
16+
17+
Note that docker plugins without a tag in the alias currently get `:latest` appended. To prevent this from happening, we will use
18+
the fake tag `:swarm` instead.
19+
20+
```bash
21+
docker plugin install --disable --alias hetznercloud/hcloud-csi-driver:swarm --grant-all-permissions hetznercloud/hcloud-csi-driver:<version>-swarm
22+
```
23+
24+
3. Set HCLOUD_TOKEN
25+
26+
```bash
27+
docker plugin set hetznercloud/hcloud-csi-driver:swarm HCLOUD_TOKEN=<your token>
28+
```
29+
30+
4. Enable plugin
31+
32+
```bash
33+
docker plugin enable hetznercloud/hcloud-csi-driver:swarm
34+
```
35+
36+
## How to create a volume
37+
38+
Example: Create a volume wih size 50G in Nuremberg:
39+
40+
```bash
41+
docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1
42+
```
43+
44+
We can now use this in a service:
45+
46+
```bash
47+
docker service create --name hcloud-debug-serv1 --mount type=cluster,src=hcloud-debug1,dst=/srv/www nginx:alpine
48+
```
49+
50+
Note that only scope `single` is supported as Hetzner Cloud volumes can only be attached to one node at a time
51+
52+
We can however share the volume on multiple containers on the same host:
53+
54+
```bash
55+
docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1
56+
```
57+
58+
After creation we can now use this volume with `--sharing all` in more than one replica:
59+
60+
```bash
61+
docker service create --name hcloud-debug-serv2 --mount type=cluster,src=hcloud-debug2,dst=/srv/www nginx:alpine
62+
docker service scale hcloud-debug-serv2=2
63+
```
64+
65+
## How to resize a docker swarm Hetzner CSI volume
66+
67+
Currently, the Docker Swarm CSI support does not come with support for volume resizing. See [this ticket](https://github.com/moby/moby/issues/44985) for the current state on the Docker side.
68+
The following explains a step by step guide on how to do this manually instead.
69+
70+
Please test the following on a Swarm with the same version as your target cluster
71+
as this strongly depends on the logic of `docker volume rm -f` not deleting the cloud volume.
72+
73+
### Steps
74+
75+
1. Drain Volume
76+
77+
```
78+
docker volume update <volume-name> --availability drain
79+
```
80+
81+
This way, we ensure that all services stop using the volume.
82+
83+
2. Force remove volume on cluster
84+
85+
```
86+
docker volume rm -f <volume-name>
87+
```
88+
89+
4. Resize Volume in Hetzner UI
90+
5. Attach Volume to temporary server manually
91+
6. Run resize2fs manually
92+
7. Detach Volume from temporary server manually
93+
8. Recreate Volume with new size to make it known to Swarm again
94+
95+
```
96+
docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes <new-size> --type mount --sharing onewriter --scope single <volume-name>
97+
```
98+
99+
9. Verify that volume exists again:
100+
101+
```
102+
docker volume ls --cluster
103+
```
104+

‎deploy/docker-swarm/pkg/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM golang:1.19 as builder
2+
WORKDIR /csi
3+
ADD go.mod go.sum /csi/
4+
RUN go mod download
5+
ADD . /csi/
6+
RUN ls -al
7+
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o aio.bin github.com/hetznercloud/csi-driver/cmd/aio
8+
9+
FROM --platform=linux/amd64 alpine:3.15
10+
RUN apk add --no-cache ca-certificates e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra btrfs-progs cryptsetup
11+
ENV GOTRACEBACK=all
12+
RUN mkdir -p /plugin
13+
COPY --from=builder /csi/aio.bin /plugin
14+
15+
ENTRYPOINT [ "/plugin/aio.bin" ]

‎deploy/docker-swarm/pkg/LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2018 Leo Antunes (base packaging code from https://github.com/costela/docker-volume-hetzner)
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎deploy/docker-swarm/pkg/Makefile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
PLUGIN_NAME = hetznercloud/hcloud-csi-driver
2+
PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev)-swarm
3+
4+
all: create
5+
6+
clean:
7+
@rm -rf ./plugin
8+
@docker container rm -vf tmp_plugin_build || true
9+
10+
rootfs: clean
11+
docker image build -f Dockerfile -t ${PLUGIN_NAME}:rootfs ../../../
12+
mkdir -p ./plugin/rootfs
13+
docker container create --name tmp_plugin_build ${PLUGIN_NAME}:rootfs
14+
docker container export tmp_plugin_build | tar -x -C ./plugin/rootfs
15+
cp config.json ./plugin/
16+
docker container rm -vf tmp_plugin_build
17+
18+
create: rootfs
19+
docker plugin rm -f ${PLUGIN_NAME}:${PLUGIN_TAG} 2> /dev/null || true
20+
docker plugin create ${PLUGIN_NAME}:${PLUGIN_TAG} ./plugin
21+
22+
enable: create
23+
docker plugin enable ${PLUGIN_NAME}:${PLUGIN_TAG}
24+
25+
push: create
26+
docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG}
27+
28+
push_latest: create
29+
docker plugin push ${PLUGIN_NAME}:latest-swarm
30+
31+
.PHONY: clean rootfs create enable push

‎deploy/docker-swarm/pkg/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
a lot in this directory comes from work originally done
2+
by other awesome people.
3+
4+
Before CSI support, Docker Swarm volumes
5+
were graciously supported by @costela over at:
6+
https://github.com/costela/docker-volume-hetzner

0 commit comments

Comments
 (0)