Test reusable storage

helper/testhelpers/testhelpers.go

@@ -277,7 +277,7 @@ func WaitForNCoresUnsealed(t testing.T, cluster *vault.TestCluster, n int) {
 		time.Sleep(time.Second)
 	}
 
-	t.Fatalf("%d cores were not sealed", n)
+	t.Fatalf("%d cores were not unsealed", n)
 }
 
 func WaitForNCoresSealed(t testing.T, cluster *vault.TestCluster, n int) {
@@ -386,6 +386,12 @@ func RekeyCluster(t testing.T, cluster *vault.TestCluster, recovery bool) [][]by
 	return newKeys
 }
 
+// TestRaftServerAddressProvider is a ServerAddressProvider that uses the
+// ClusterAddr() of each node to provide raft addresses.
+//
+// Note that TestRaftServerAddressProvider should only be used in cases where
+// cores that are part of a raft configuration have already had
+// startClusterListener() called (via either unsealing or raft joining).
 type TestRaftServerAddressProvider struct {
 	Cluster *vault.TestCluster
 }
@@ -457,3 +463,49 @@ func RaftClusterJoinNodes(t testing.T, cluster *vault.TestCluster) {
 
 	WaitForNCoresUnsealed(t, cluster, 3)
 }
+
+// HardcodedServerAddressProvider is a ServerAddressProvider that uses
+// a hardcoded map of raft node addresses.
+//
+// It is useful in cases where the raft configuration is known ahead of time,
+// but some of the cores have not yet had startClusterListener() called (via
+// either unsealing or raft joining), and thus do not yet have a ClusterAddr()
+// assigned.
+type HardcodedServerAddressProvider struct {
+	Entries map[raftlib.ServerID]raftlib.ServerAddress
+}
+
+func (p *HardcodedServerAddressProvider) ServerAddr(id raftlib.ServerID) (raftlib.ServerAddress, error) {
+	if addr, ok := p.Entries[id]; ok {
+		return addr, nil
+	}
+	return "", errors.New("could not find cluster addr")
+}
+
+// NewHardcodedServerAddressProvider is a convenience function that makes a
+// ServerAddressProvider from a given cluster address base port.
+func NewHardcodedServerAddressProvider(baseClusterPort int) raftlib.ServerAddressProvider {
+
+	entries := make(map[raftlib.ServerID]raftlib.ServerAddress)
+
+	for i := 0; i < vault.DefaultNumCores; i++ {
+		id := fmt.Sprintf("core-%d", i)
+		addr := fmt.Sprintf("127.0.0.1:%d", baseClusterPort+i)
+		entries[raftlib.ServerID(id)] = raftlib.ServerAddress(addr)
+	}
+
+	return &HardcodedServerAddressProvider{
+		entries,
+	}
+}
+
+// SetRaftAddressProviders sets a ServerAddressProvider for all the nodes in a
+// cluster.
+func SetRaftAddressProviders(t testing.T, cluster *vault.TestCluster, provider raftlib.ServerAddressProvider) {
+
+	atomic.StoreUint32(&vault.UpdateClusterAddrForTests, 1)
+
+	for _, core := range cluster.Cores {
+		core.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(provider)
+	}
+}

helper/testhelpers/teststorage/teststorage_reusable.go

@@ -0,0 +1,142 @@
+package teststorage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime/debug"
+
+	"github.com/mitchellh/go-testing-interface"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/physical/raft"
+	"github.com/hashicorp/vault/vault"
+)
+
+// ReusableStorage is a physical backend that can be re-used across
+// multiple test clusters in sequence.  It is useful for testing things like
+// seal migration, wherein a given physical backend must be re-used as several
+// test clusters are sequentially created, tested, and discarded.
+type ReusableStorage struct {
+
+	// IsRaft specifies whether the storage is using a raft backend.
+	IsRaft bool
+
+	// Setup should be called just before a new TestCluster is created.
+	Setup ClusterSetupMutator
+
+	// Cleanup should be called after a TestCluster is no longer
+	// needed -- generally in a defer, just before the call to
+	// cluster.Cleanup().
+	Cleanup func(t testing.T, cluster *vault.TestCluster)
+}
+
+// StorageCleanup is a function that should be called once -- at the very end
+// of a given unit test, after each of the sequence of clusters have been
+// created, tested, and discarded.
+type StorageCleanup func()
+
+// MakeReusableStorage makes a physical backend that can be re-used across
+// multiple test clusters in sequence.
+func MakeReusableStorage(t testing.T, logger hclog.Logger, bundle *vault.PhysicalBackendBundle) (ReusableStorage, StorageCleanup) {
+
+	storage := ReusableStorage{
+		IsRaft: false,
+
+		Setup: func(conf *vault.CoreConfig, opts *vault.TestClusterOptions) {
+			opts.PhysicalFactory = func(t testing.T, coreIdx int, logger hclog.Logger) *vault.PhysicalBackendBundle {
+				if coreIdx == 0 {
+					// We intentionally do not clone the backend's Cleanup func,
+					// because we don't want it to be run until the entire test has
+					// been completed.
+					return &vault.PhysicalBackendBundle{
+						Backend:   bundle.Backend,
+						HABackend: bundle.HABackend,
+					}
+				}
+				return nil
+			}
+		},
+
+		// No-op
+		Cleanup: func(t testing.T, cluster *vault.TestCluster) {
+		},
+	}
+
+	cleanup := func() {
+		if bundle.Cleanup != nil {
+			bundle.Cleanup()
+		}
+	}
+
+	return storage, cleanup
+}
+
+// MakeReusableRaftStorage makes a physical raft backend that can be re-used
+// across multiple test clusters in sequence.
+func MakeReusableRaftStorage(t testing.T, logger hclog.Logger) (ReusableStorage, StorageCleanup) {
+
+	raftDirs := make([]string, vault.DefaultNumCores)
+	for i := 0; i < vault.DefaultNumCores; i++ {
+		raftDirs[i] = makeRaftDir(t)
+	}
+
+	storage := ReusableStorage{
+		IsRaft: true,
+
+		Setup: func(conf *vault.CoreConfig, opts *vault.TestClusterOptions) {
+			conf.DisablePerformanceStandby = true
+			opts.KeepStandbysSealed = true
+			opts.PhysicalFactory = func(t testing.T, coreIdx int, logger hclog.Logger) *vault.PhysicalBackendBundle {
+				return makeReusableRaftBackend(t, coreIdx, logger, raftDirs[coreIdx])
+			}
+		},
+
+		// Close open files being used by raft.
+		Cleanup: func(t testing.T, cluster *vault.TestCluster) {
+			for _, core := range cluster.Cores {
+				raftStorage := core.UnderlyingRawStorage.(*raft.RaftBackend)
+				if err := raftStorage.Close(); err != nil {
+					t.Fatal(err)
+				}
+			}
+		},
+	}
+
+	cleanup := func() {
+		for _, rd := range raftDirs {
+			os.RemoveAll(rd)
+		}
+	}
+
+	return storage, cleanup
+}
+
+func makeRaftDir(t testing.T) string {
+	raftDir, err := ioutil.TempDir("", "vault-raft-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("raft dir: %s", raftDir)
+	return raftDir
+}
+
+func makeReusableRaftBackend(t testing.T, coreIdx int, logger hclog.Logger, raftDir string) *vault.PhysicalBackendBundle {
+
+	nodeID := fmt.Sprintf("core-%d", coreIdx)
+	conf := map[string]string{
+		"path":                   raftDir,
+		"node_id":                nodeID,
+		"performance_multiplier": "8",
+	}
+
+	backend, err := raft.NewRaftBackend(conf, logger)
+	if err != nil {
+		debug.PrintStack()
+		t.Fatal(err)
+	}
+
+	return &vault.PhysicalBackendBundle{
+		Backend: backend,
+	}
+}