Skip to content

Commit 058c5d8

Browse files
wangforthinkerwangforthinker
committed
support to choose storage driver
Signed-off-by: allen.wang <[email protected]>
1 parent 7101cf2 commit 058c5d8

File tree

19 files changed

+324
-35
lines changed

19 files changed

+324
-35
lines changed

‎cri/v1alpha1/cri.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
cni "github.com/alibaba/pouch/cri/ocicni"
1919
"github.com/alibaba/pouch/cri/stream"
2020
criutils "github.com/alibaba/pouch/cri/utils"
21+
"github.com/alibaba/pouch/ctrd"
2122
"github.com/alibaba/pouch/daemon/config"
2223
"github.com/alibaba/pouch/daemon/mgr"
2324
"github.com/alibaba/pouch/pkg/errtypes"
@@ -63,9 +64,6 @@ const (
6364
// resolvConfPath is the abs path of resolv.conf on host or container.
6465
resolvConfPath = "/etc/resolv.conf"
6566

66-
// defaultSnapshotterName is the default Snapshotter name.
67-
defaultSnapshotterName = "overlayfs"
68-
6967
// snapshotPlugin implements a snapshotter.
7068
snapshotPlugin = "io.containerd.snapshotter.v1"
7169

@@ -163,7 +161,7 @@ func NewCriManager(config *config.Config, ctrMgr mgr.ContainerMgr, imgMgr mgr.Im
163161
return nil, fmt.Errorf("failed to create sandbox meta store: %v", err)
164162
}
165163

166-
imageFSPath := imageFSPath(path.Join(config.HomeDir, "containerd/root"), defaultSnapshotterName)
164+
imageFSPath := imageFSPath(path.Join(config.HomeDir, "containerd/root"), ctrd.CurrentSnapshotterName())
167165
c.ImageFSUUID, err = getDeviceUUID(imageFSPath)
168166
if err != nil {
169167
return nil, fmt.Errorf("failed to get imagefs uuid of %q: %v", imageFSPath, err)

‎cri/v1alpha2/cri.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
cni "github.com/alibaba/pouch/cri/ocicni"
2121
"github.com/alibaba/pouch/cri/stream"
2222
criutils "github.com/alibaba/pouch/cri/utils"
23+
"github.com/alibaba/pouch/ctrd"
2324
"github.com/alibaba/pouch/daemon/config"
2425
"github.com/alibaba/pouch/daemon/mgr"
2526
"github.com/alibaba/pouch/hookplugins"
@@ -66,9 +67,6 @@ const (
6667
// resolvConfPath is the abs path of resolv.conf on host or container.
6768
resolvConfPath = "/etc/resolv.conf"
6869

69-
// defaultSnapshotterName is the default Snapshotter name.
70-
defaultSnapshotterName = "overlayfs"
71-
7270
// snapshotPlugin implements a snapshotter.
7371
snapshotPlugin = "io.containerd.snapshotter.v1"
7472

@@ -180,7 +178,7 @@ func NewCriManager(config *config.Config, ctrMgr mgr.ContainerMgr, imgMgr mgr.Im
180178
return nil, fmt.Errorf("failed to create sandbox meta store: %v", err)
181179
}
182180

183-
c.imageFSPath = imageFSPath(path.Join(config.HomeDir, "containerd/root"), defaultSnapshotterName)
181+
c.imageFSPath = imageFSPath(path.Join(config.HomeDir, "containerd/root"), ctrd.CurrentSnapshotterName())
184182
logrus.Infof("Get image filesystem path %q", c.imageFSPath)
185183

186184
if !config.CriConfig.DisableCriStatsCollect {

‎ctrd/client.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"github.com/containerd/containerd"
1414
eventstypes "github.com/containerd/containerd/api/events"
15+
"github.com/containerd/containerd/api/services/introspection/v1"
1516
"github.com/containerd/typeurl"
1617
"github.com/pkg/errors"
1718
"github.com/sirupsen/logrus"
@@ -21,6 +22,10 @@ const (
2122
unixSocketPath = "/run/containerd/containerd.sock"
2223
defaultGrpcClientPoolCapacity = 5
2324
defaultMaxStreamsClient = 100
25+
// PluginStatusOk means plugin status is ok
26+
PluginStatusOk = "ok"
27+
// PluginStatusError means plugin status is error
28+
PluginStatusError = "error"
2429
)
2530

2631
// ErrGetCtrdClient is an error returned when failed to get a containerd grpc client from clients pool.
@@ -42,6 +47,13 @@ type Client struct {
4247
eventsHooks []func(context.Context, string, string, map[string]string) error
4348
}
4449

50+
// Plugin is the containerd plugin type
51+
type Plugin struct {
52+
Type string
53+
ID string
54+
Status string
55+
}
56+
4557
// NewClient connect to containerd.
4658
func NewClient(opts ...ClientOpt) (APIClient, error) {
4759
// set default value for parameters
@@ -185,6 +197,39 @@ func (c *Client) Cleanup() error {
185197
return c.Close()
186198
}
187199

200+
// Plugins return info of containerd plugins
201+
func (c *Client) Plugins(ctx context.Context, filters []string) ([]Plugin, error) {
202+
cli, err := c.Get(ctx)
203+
if err != nil {
204+
return nil, fmt.Errorf("failed to get a containerd grpc client: %v", err)
205+
}
206+
207+
resp, err := cli.client.IntrospectionService().Plugins(ctx, &introspection.PluginsRequest{Filters: filters})
208+
if err != nil {
209+
return nil, err
210+
}
211+
212+
var (
213+
plugins = []Plugin{}
214+
)
215+
216+
for _, p := range resp.Plugins {
217+
plugin := Plugin{
218+
Type: p.Type,
219+
ID: p.ID,
220+
Status: PluginStatusOk,
221+
}
222+
223+
if p.InitErr != nil {
224+
plugin.Status = PluginStatusError
225+
}
226+
227+
plugins = append(plugins, plugin)
228+
}
229+
230+
return plugins, nil
231+
}
232+
188233
// collectContainerdEvents collects events generated by containerd.
189234
func (c *Client) collectContainerdEvents() {
190235
ctx := context.Background()

‎ctrd/container.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -503,6 +503,7 @@ func (c *Client) createContainer(ctx context.Context, ref, id, checkpointDir str
503503

504504
// create container
505505
options := []containerd.NewContainerOpts{
506+
containerd.WithSnapshotter(CurrentSnapshotterName()),
506507
containerd.WithContainerLabels(container.Labels),
507508
containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), &runctypes.RuncOptions{
508509
Runtime: container.Runtime,

‎ctrd/image.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/containerd/containerd/errdefs"
1717
ctrdmetaimages "github.com/containerd/containerd/images"
1818
"github.com/containerd/containerd/remotes"
19+
"github.com/containerd/containerd/snapshots"
1920
"github.com/opencontainers/go-digest"
2021
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2122
"github.com/pkg/errors"
@@ -168,7 +169,7 @@ func (c *Client) importImage(ctx context.Context, importer ctrdmetaimages.Import
168169
}
169170

170171
for _, img := range imgs {
171-
err = img.Unpack(ctx, containerd.DefaultSnapshotter)
172+
err = img.Unpack(ctx, CurrentSnapshotterName())
172173
if err != nil {
173174
return nil, err
174175
}
@@ -194,6 +195,8 @@ func (c *Client) PullImage(ctx context.Context, ref string, authConfig *types.Au
194195
containerd.WithPullUnpack,
195196
containerd.WithSchema1Conversion,
196197
containerd.WithResolver(resolver),
198+
containerd.WithPullSnapshotter(CurrentSnapshotterName()),
199+
containerd.WithPullLabel(snapshots.TypeLabelKey, snapshots.ImageType),
197200
}
198201

199202
handle := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {

‎ctrd/image_commit.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func (c *Client) Commit(ctx context.Context, config *CommitConfig) (_ digest.Dig
7777
client := wrapperCli.client
7878

7979
var (
80-
sn = client.SnapshotService(defaultSnapshotterName)
80+
sn = client.SnapshotService(CurrentSnapshotterName())
8181
cs = client.ContentStore()
8282
differ = client.DiffService()
8383
)
@@ -111,7 +111,7 @@ func (c *Client) Commit(ctx context.Context, config *CommitConfig) (_ digest.Dig
111111
defer func() {
112112
if err0 != nil {
113113
logrus.Warnf("remove snapshot %s cause commit image failed", snapshotKey)
114-
client.SnapshotService(defaultSnapshotterName).Remove(ctx, snapshotKey)
114+
client.SnapshotService(CurrentSnapshotterName()).Remove(ctx, snapshotKey)
115115
}
116116
}()
117117

@@ -266,6 +266,7 @@ func newSnapshot(ctx context.Context, pImg ocispec.Image, sn snapshots.Snapshott
266266
// avoid active snapshots cleaned by containerd 1.0.3 gc
267267
opt := snapshots.WithLabels(map[string]string{
268268
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339),
269+
snapshots.TypeLabelKey: snapshots.ImageType,
269270
})
270271
mount, err := sn.Prepare(ctx, key, parent, opt)
271272
if err != nil {

‎ctrd/interface.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type APIClient interface {
2626

2727
Version(ctx context.Context) (containerd.Version, error)
2828
Cleanup() error
29+
Plugins(ctx context.Context, filters []string) ([]Plugin, error)
2930
}
3031

3132
// ContainerAPIClient provides access to containerd container features.

‎ctrd/snapshot.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,25 @@ import (
1111
"github.com/opencontainers/image-spec/identity"
1212
)
1313

14-
const defaultSnapshotterName = "overlayfs"
14+
const (
15+
defaultSnapshotterName = "overlayfs"
16+
// SnapshotterKey is set as a key in context
17+
SnapshotterKey = "snapshotter"
18+
)
19+
20+
var (
21+
currentSnapshotterName = defaultSnapshotterName
22+
)
23+
24+
// SetSnapshotterName sets current snapshotter driver, it should be called only when daemon starts
25+
func SetSnapshotterName(name string) {
26+
currentSnapshotterName = name
27+
}
28+
29+
// CurrentSnapshotterName returns current snapshotter driver
30+
func CurrentSnapshotterName() string {
31+
return currentSnapshotterName
32+
}
1533

1634
// CreateSnapshot creates a active snapshot with image's name and id.
1735
func (c *Client) CreateSnapshot(ctx context.Context, id, ref string) error {
@@ -32,7 +50,7 @@ func (c *Client) CreateSnapshot(ctx context.Context, id, ref string) error {
3250
}
3351

3452
parent := identity.ChainID(diffIDs).String()
35-
_, err = wrapperCli.client.SnapshotService(defaultSnapshotterName).Prepare(ctx, id, parent)
53+
_, err = wrapperCli.client.SnapshotService(CurrentSnapshotterName()).Prepare(ctx, id, parent)
3654
return err
3755
}
3856

@@ -43,7 +61,7 @@ func (c *Client) GetSnapshot(ctx context.Context, id string) (snapshots.Info, er
4361
return snapshots.Info{}, fmt.Errorf("failed to get a containerd grpc client: %v", err)
4462
}
4563

46-
service := wrapperCli.client.SnapshotService(defaultSnapshotterName)
64+
service := wrapperCli.client.SnapshotService(CurrentSnapshotterName())
4765
defer service.Close()
4866

4967
return service.Stat(ctx, id)
@@ -56,7 +74,7 @@ func (c *Client) RemoveSnapshot(ctx context.Context, id string) error {
5674
return fmt.Errorf("failed to get a containerd grpc client: %v", err)
5775
}
5876

59-
service := wrapperCli.client.SnapshotService(defaultSnapshotterName)
77+
service := wrapperCli.client.SnapshotService(CurrentSnapshotterName())
6078
defer service.Close()
6179

6280
return service.Remove(ctx, id)
@@ -70,7 +88,7 @@ func (c *Client) GetMounts(ctx context.Context, id string) ([]mount.Mount, error
7088
return nil, fmt.Errorf("failed to get a containerd grpc client: %v", err)
7189
}
7290

73-
service := wrapperCli.client.SnapshotService(defaultSnapshotterName)
91+
service := wrapperCli.client.SnapshotService(CurrentSnapshotterName())
7492
defer service.Close()
7593

7694
return service.Mounts(ctx, id)
@@ -84,7 +102,7 @@ func (c *Client) GetSnapshotUsage(ctx context.Context, id string) (snapshots.Usa
84102
return snapshots.Usage{}, fmt.Errorf("failed to get a containerd grpc client: %v", err)
85103
}
86104

87-
service := wrapperCli.client.SnapshotService(defaultSnapshotterName)
105+
service := wrapperCli.client.SnapshotService(CurrentSnapshotterName())
88106
defer service.Close()
89107

90108
return service.Usage(ctx, id)
@@ -97,7 +115,13 @@ func (c *Client) WalkSnapshot(ctx context.Context, fn func(context.Context, snap
97115
return fmt.Errorf("failed to get a containerd grpc client: %v", err)
98116
}
99117

100-
service := wrapperCli.client.SnapshotService(defaultSnapshotterName)
118+
// Allow user to walk snapshots of specific snapshotter
119+
snapshotter := CurrentSnapshotterName()
120+
if sn, ok := ctx.Value(SnapshotterKey).(string); ok {
121+
snapshotter = sn
122+
}
123+
124+
service := wrapperCli.client.SnapshotService(snapshotter)
101125
defer service.Close()
102126

103127
return service.Walk(ctx, fn)

‎daemon/config/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ type Config struct {
117117

118118
// DefaultNamespace is passed to containerd.
119119
DefaultNamespace string `json:"default-namespace,omitempty"`
120+
121+
// Snapshotter is passed to containerd, default to overlayfs
122+
Snapshotter string `json:"snapshotter,omitempty"`
120123
}
121124

122125
// GetCgroupDriver gets cgroup driver used in runc.

‎daemon/daemon.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,17 @@ func NewDaemon(cfg *config.Config) *Daemon {
101101
return nil
102102
}
103103

104+
if cfg.Snapshotter != "" {
105+
ctrd.SetSnapshotterName(cfg.Snapshotter)
106+
}
107+
108+
if err = checkSnapshotter(ctrd.CurrentSnapshotterName(), ctrdClient); err != nil {
109+
logrus.Errorf("failed to check snapshotter driver: %v", err)
110+
return nil
111+
}
112+
113+
logrus.Infof("Snapshotter is set to be %s", ctrd.CurrentSnapshotterName())
114+
104115
return &Daemon{
105116
config: cfg,
106117
ctrdClient: ctrdClient,

0 commit comments

Comments
 (0)