image/cas: Add a generic CAS interface

And implement that interface for tarballs based on the specs
image-layout.  I plan on adding other backends later, but this is
enough for a proof of concept.

Also add a new oci-cas command so folks can access the new read
functionality from the command line.

In a subsequent commit, I'll replace the image/walker.go functionality
with this new API.

The Context interface follows the pattern recommended in [1], allowing
callers to cancel long running actions (e.g. push/pull over the
network for engine implementations that communicate with a remote
store).

blobPath's separator argument will allow us to use
string(os.PathSeparator)) once we add directory support.

[1]: https://blog.golang.org/context

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

.gitignore

@@ -1,3 +1,4 @@
+/oci-cas
 /oci-create-runtime-bundle
-/oci-unpack
 /oci-image-validate
+/oci-unpack

Makefile

@@ -5,6 +5,7 @@ COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 
 EPOCH_TEST_COMMIT ?= v0.2.0
 TOOLS := \
+	oci-cas \
 	oci-create-runtime-bundle \
 	oci-image-validate \
 	oci-unpack

cmd/oci-cas/get.go

@@ -0,0 +1,99 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas/layout"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type getCmd struct {
+	path   string
+	digest digest.Digest
+}
+
+func newGetCmd() *cobra.Command {
+	state := &getCmd{}
+
+	return &cobra.Command{
+		Use:   "get PATH DIGEST",
+		Short: "Retrieve a blob from the store",
+		Long:  "Retrieve a blob from the store and write it to stdout.",
+		Run:   state.Run,
+	}
+}
+
+func (state *getCmd) Run(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided")
+		if err := cmd.Usage(); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		os.Exit(1)
+	}
+
+	state.path = args[0]
+	var err error
+	state.digest, err = digest.Parse(args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	err = state.run()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func (state *getCmd) run() (err error) {
+	ctx := context.Background()
+
+	engine, err := layout.NewEngine(state.path)
+	if err != nil {
+		return err
+	}
+	defer engine.Close()
+
+	reader, err := engine.Get(ctx, state.digest)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	bytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	n, err := os.Stdout.Write(bytes)
+	if err != nil {
+		return err
+	}
+	if n < len(bytes) {
+		return fmt.Errorf("wrote %d of %d bytes", n, len(bytes))
+	}
+
+	return nil
+}

cmd/oci-cas/main.go

@@ -0,0 +1,39 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	_ "crypto/sha256"
+	_ "crypto/sha512"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	cmd := &cobra.Command{
+		Use:   "oci-cas",
+		Short: "Content-addressable storage manipulation",
+	}
+
+	cmd.AddCommand(newGetCmd())
+
+	err := cmd.Execute()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

image/cas/interface.go

@@ -0,0 +1,45 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cas implements generic content-addressable storage.
+package cas
+
+import (
+	"io"
+
+	"github.com/opencontainers/go-digest"
+	"golang.org/x/net/context"
+)
+
+// Engine represents a content-addressable storage engine.
+type Engine interface {
+
+	// Put adds a new blob to the store.  The action is idempotent; a
+	// nil return means "that content is stored at DIGEST" without
+	// implying "because of your Put()".
+	Put(ctx context.Context, reader io.Reader) (digest digest.Digest, err error)
+
+	// Get returns a reader for retrieving a blob from the store.
+	// Returns os.ErrNotExist if the digest is not found.
+	Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error)
+
+	// Delete removes a blob from the store. The action is idempotent; a
+	// nil return means "that content is not in the store" without
+	// implying "because of your Delete()".
+	Delete(ctx context.Context, digest digest.Digest) (err error)
+
+	// Close releases resources held by the engine.  Subsequent engine
+	// method calls will fail.
+	Close() (err error)
+}

image/cas/layout/interface.go

@@ -0,0 +1,25 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package layout
+
+import (
+	"io"
+)
+
+// ReadWriteSeekCloser wraps the Read, Write, Seek, and Close methods.
+type ReadWriteSeekCloser interface {
+	io.ReadWriteSeeker
+	io.Closer
+}

image/cas/layout/main.go

@@ -0,0 +1,50 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package layout implements the cas interface using the image-spec's
+// image-layout [1].
+//
+// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md
+package layout
+
+import (
+	"os"
+	"strings"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas"
+)
+
+// NewEngine instantiates an engine with the appropriate backend (tar,
+// HTTP, ...).
+func NewEngine(path string) (engine cas.Engine, err error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewTarEngine(file)
+}
+
+// blobPath returns the PATH to the DIGEST blob.  SEPARATOR selects
+// the path separator used between components.
+func blobPath(digest digest.Digest, separator string) (path string, err error) {
+	err = digest.Validate()
+	if err != nil {
+		return "", err
+	}
+	algorithm := digest.Algorithm().String()
+	components := []string{".", "blobs", algorithm, digest.Hex()}
+	return strings.Join(components, separator), nil
+}

image/cas/layout/tar.go

@@ -0,0 +1,91 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package layout
+
+import (
+	"archive/tar"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas"
+	"golang.org/x/net/context"
+)
+
+// TarEngine is a cas.Engine backed by a tar file.
+type TarEngine struct {
+	file ReadWriteSeekCloser
+}
+
+// NewTarEngine returns a new TarEngine.
+func NewTarEngine(file ReadWriteSeekCloser) (engine cas.Engine, err error) {
+	engine = &TarEngine{
+		file: file,
+	}
+
+	return engine, nil
+}
+
+// Put adds a new blob to the store.
+func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest digest.Digest, err error) {
+	// FIXME
+	return "", errors.New("TarEngine.Put is not supported yet")
+}
+
+// Get returns a reader for retrieving a blob from the store.
+func (engine *TarEngine) Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) {
+	targetName, err := blobPath(digest, "/")
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = engine.file.Seek(0, os.SEEK_SET)
+	if err != nil {
+		return nil, err
+	}
+
+	tarReader := tar.NewReader(engine.file)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			return nil, os.ErrNotExist
+		} else if err != nil {
+			return nil, err
+		}
+
+		if header.Name == targetName {
+			return ioutil.NopCloser(tarReader), nil
+		}
+	}
+}
+
+// Delete removes a blob from the store.
+func (engine *TarEngine) Delete(ctx context.Context, digest digest.Digest) (err error) {
+	// FIXME
+	return errors.New("TarEngine.Delete is not supported yet")
+}
+
+// Close releases resources held by the engine.
+func (engine *TarEngine) Close() (err error) {
+	return engine.file.Close()
+}