feat: support gRPC and http apis

buf.gen.yaml

@@ -0,0 +1,13 @@
+version: v1
+plugins:
+  - name: go
+    out: ./server
+    opt: paths=source_relative
+  - name: go-grpc
+    out: ./server
+    opt: paths=source_relative
+  - name: grpc-gateway
+    out: ./server
+    opt: paths=source_relative
+  - name: openapiv2
+    out: ./server

server/api/adapter.go

@@ -0,0 +1,47 @@
+package api
+
+import (
+	"github.com/odpf/stencil/server/models"
+	stencilv1 "github.com/odpf/stencil/server/odpf/stencil/v1"
+	"github.com/odpf/stencil/server/snapshot"
+)
+
+func fromProtoToSnapshot(g *stencilv1.Snapshot) *snapshot.Snapshot {
+	return &snapshot.Snapshot{
+		ID:        g.GetId(),
+		Namespace: g.GetNamespace(),
+		Name:      g.GetName(),
+		Version:   g.GetVersion(),
+		Latest:    g.GetLatest(),
+	}
+}
+
+func fromSnapshotToProto(g *snapshot.Snapshot) *stencilv1.Snapshot {
+	return &stencilv1.Snapshot{
+		Id:        g.ID,
+		Namespace: g.Namespace,
+		Name:      g.Name,
+		Version:   g.Version,
+		Latest:    g.Latest,
+	}
+}
+
+func toRulesList(r *stencilv1.Checks) []string {
+	var rules []string
+	if r == nil {
+		return rules
+	}
+	for _, rule := range r.Except {
+		rules = append(rules, rule.String())
+	}
+	return rules
+}
+
+func toFileDownloadRequest(g *stencilv1.DownloadDescriptorRequest) *models.FileDownloadRequest {
+	return &models.FileDownloadRequest{
+		Namespace: g.Namespace,
+		Name:      g.Name,
+		Version:   g.Version,
+		FullNames: g.GetFullnames(),
+	}
+}

server/api/api.go

@@ -3,7 +3,9 @@ package api
 import (
 	"context"
 
+	stencilv1 "github.com/odpf/stencil/server/odpf/stencil/v1"
 	"github.com/odpf/stencil/server/snapshot"
+	"google.golang.org/grpc/health/grpc_health_v1"
 )
 
 //StoreService Service Interface for storage and validation
@@ -16,14 +18,16 @@ type StoreService interface {
 // MetadataService Service Interface for metadata store
 type MetadataService interface {
 	Exists(context.Context, *snapshot.Snapshot) bool
-	ListNames(context.Context, string) ([]string, error)
-	ListVersions(context.Context, string, string) ([]string, error)
-	GetSnapshot(context.Context, string, string, string, bool) (*snapshot.Snapshot, error)
+	List(context.Context, *snapshot.Snapshot) ([]*snapshot.Snapshot, error)
+	GetSnapshotByFields(context.Context, string, string, string, bool) (*snapshot.Snapshot, error)
+	GetSnapshotByID(context.Context, int64) (*snapshot.Snapshot, error)
 	UpdateLatestVersion(context.Context, *snapshot.Snapshot) error
 }
 
 //API holds all handlers
 type API struct {
+	stencilv1.UnimplementedStencilServiceServer
+	grpc_health_v1.UnimplementedHealthServer
 	Store    StoreService
 	Metadata MetadataService
 }

server/api/api_test.go

@@ -1,15 +1,16 @@
 package api_test
 
 import (
+	"net/http"
+
 	"github.com/odpf/stencil/server/config"
 	server2 "github.com/odpf/stencil/server/server"
 
-	"github.com/gin-gonic/gin"
 	"github.com/odpf/stencil/server/api"
 	"github.com/odpf/stencil/server/api/mocks"
 )
 
-func setup() (*gin.Engine, *mocks.StoreService, *mocks.MetadataService, *api.API) {
+func setup() (http.Handler, *mocks.StoreService, *mocks.MetadataService, *api.API) {
 	mockService := &mocks.StoreService{}
 	mockMetadataService := &mocks.MetadataService{}
 	v1 := &api.API{

server/api/download.go

@@ -1,17 +1,21 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/url"
 
 	"github.com/gin-gonic/gin"
 	"github.com/odpf/stencil/server/models"
+	stencilv1 "github.com/odpf/stencil/server/odpf/stencil/v1"
 	"github.com/odpf/stencil/server/snapshot"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
-//Download downloads file
-func (a *API) Download(c *gin.Context) {
+//HTTPDownload http handler to download requested schema data
+func (a *API) HTTPDownload(c *gin.Context) {
 	ctx := c.Request.Context()
 	payload := models.FileDownloadRequest{
 		FullNames: c.QueryArray("fullnames"),
@@ -21,25 +25,44 @@ func (a *API) Download(c *gin.Context) {
 		return
 	}
 	s := payload.ToSnapshot()
-	st, err := a.Metadata.GetSnapshot(ctx, s.Namespace, s.Name, s.Version, s.Latest)
+	data, err := a.download(ctx, s, payload.FullNames)
+	if err != nil {
+		c.Error(err)
+		return
+	}
+	fileName := payload.Version
+	c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName)))
+	c.Data(http.StatusOK, "application/octet-stream", data)
+}
+
+// DownloadDescriptor grpc handler to download schema data
+func (a *API) DownloadDescriptor(ctx context.Context, req *stencilv1.DownloadDescriptorRequest) (*stencilv1.DownloadDescriptorResponse, error) {
+	payload := toFileDownloadRequest(req)
+	err := validate.Struct(payload)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	s := payload.ToSnapshot()
+	data, err := a.download(ctx, s, req.Fullnames)
+	return &stencilv1.DownloadDescriptorResponse{Data: data}, err
+}
+
+func (a *API) download(ctx context.Context, s *snapshot.Snapshot, fullNames []string) ([]byte, error) {
+	notfoundErr := status.Error(codes.NotFound, "not found")
+	var data []byte
+	st, err := a.Metadata.GetSnapshotByFields(ctx, s.Namespace, s.Name, s.Version, s.Latest)
 	if err != nil {
 		if err == snapshot.ErrNotFound {
-			c.JSON(http.StatusNotFound, gin.H{"message": err.Error()})
-			return
+			return data, notfoundErr
 		}
-		c.Error(err).SetMeta(models.ErrDownloadFailed)
-		return
+		return data, status.Convert(err).Err()
 	}
-	data, err := a.Store.Get(c.Request.Context(), st, payload.FullNames)
+	data, err = a.Store.Get(ctx, st, fullNames)
 	if err != nil {
-		c.Error(err).SetMeta(models.ErrDownloadFailed)
-		return
+		return data, status.Convert(err).Err()
 	}
 	if len(data) == 0 {
-		c.JSON(http.StatusNotFound, gin.H{"message": "not found"})
-		return
+		return data, notfoundErr
 	}
-	fileName := payload.Version
-	c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName)))
-	c.Data(http.StatusOK, "application/octet-stream", data)
+	return data, nil
 }

server/api/download_test.go

@@ -1,15 +1,19 @@
 package api_test
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	stencilv1 "github.com/odpf/stencil/server/odpf/stencil/v1"
 	"github.com/odpf/stencil/server/snapshot"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"google.golang.org/grpc/status"
 )
 
 var downloadFail = errors.New("download fail")
@@ -31,11 +35,11 @@ func TestDownload(t *testing.T) {
 		{"should return 200 if download succeeded", "name1", "1.0.1", nil, nil, 200},
 		{"should be able to download with latest version", "name1", "latest", nil, nil, 200},
 	} {
-		t.Run(test.desc, func(t *testing.T) {
+		t.Run(fmt.Sprintf("http: %s", test.desc), func(t *testing.T) {
 			router, mockService, mockMetadata, _ := setup()
 
 			fileData := []byte("File contents")
-			mockMetadata.On("GetSnapshot", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&snapshot.Snapshot{}, test.notFoundErr)
+			mockMetadata.On("GetSnapshotByFields", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&snapshot.Snapshot{}, test.notFoundErr)
 			mockService.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(fileData, test.downloadErr)
 			w := httptest.NewRecorder()
 			req, _ := http.NewRequest("GET", fmt.Sprintf("/v1/namespaces/namespace/descriptors/%s/versions/%s", test.name, test.version), nil)
@@ -49,11 +53,27 @@ func TestDownload(t *testing.T) {
 				assert.Equal(t, expectedHeader, w.Header().Get("Content-Disposition"))
 			}
 		})
+		t.Run(fmt.Sprintf("gRPC: %s", test.desc), func(t *testing.T) {
+			ctx := context.Background()
+			_, mockService, mockMetadata, a := setup()
+
+			fileData := []byte("File contents")
+			mockMetadata.On("GetSnapshotByFields", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&snapshot.Snapshot{}, test.notFoundErr)
+			mockService.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(fileData, test.downloadErr)
+			req := &stencilv1.DownloadDescriptorRequest{Namespace: "namespace", Name: test.name, Version: test.version}
+			res, err := a.DownloadDescriptor(ctx, req)
+			if test.expectedCode != 200 {
+				e := status.Convert(err)
+				assert.Equal(t, test.expectedCode, runtime.HTTPStatusFromCode(e.Code()))
+			} else {
+				assert.Equal(t, res.Data, []byte("File contents"))
+			}
+		})
 	}
 	t.Run("should return 404 if file content not found", func(t *testing.T) {
 		router, mockService, mockMetadata, _ := setup()
 		fileData := []byte("")
-		mockMetadata.On("GetSnapshot", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&snapshot.Snapshot{}, nil)
+		mockMetadata.On("GetSnapshotByFields", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&snapshot.Snapshot{}, nil)
 		mockService.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(fileData, nil)
 		w := httptest.NewRecorder()
 		req, _ := http.NewRequest("GET", "/v1/namespaces/namespace/descriptors/n/versions/latest", nil)

server/api/metadata.go

@@ -1,66 +1,41 @@
 package api
 
 import (
-	"net/http"
+	"context"
 
-	"github.com/gin-gonic/gin"
-	"github.com/odpf/stencil/server/models"
+	stencilv1 "github.com/odpf/stencil/server/odpf/stencil/v1"
 	"github.com/odpf/stencil/server/snapshot"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
-// ListNames lists descriptor entries
-func (a *API) ListNames(c *gin.Context) {
-	namespace := c.Param("namespace")
-	result, err := a.Metadata.ListNames(c.Request.Context(), namespace)
+// ListSnapshots returns list of snapshots. If filters applied it will return filtered snapshot list
+func (a *API) ListSnapshots(ctx context.Context, req *stencilv1.ListSnapshotsRequest) (*stencilv1.ListSnapshotsResponse, error) {
+	res := &stencilv1.ListSnapshotsResponse{}
+	list, err := a.Metadata.List(ctx, &snapshot.Snapshot{Namespace: req.Namespace, Name: req.Name, Version: req.Version, Latest: req.Latest})
 	if err != nil {
-		c.Error(err).SetMeta(models.ErrUnknown)
-		return
+		return res, err
 	}
-	c.JSON(http.StatusOK, result)
-}
-
-// ListVersions lists version numbers for specific name
-func (a *API) ListVersions(c *gin.Context) {
-	namespace := c.Param("namespace")
-	name := c.Param("name")
-	result, err := a.Metadata.ListVersions(c.Request.Context(), namespace, name)
-	if err != nil {
-		c.Error(err).SetMeta(models.ErrUnknown)
-		return
+	for _, j := range list {
+		res.Snapshots = append(res.Snapshots, fromSnapshotToProto(j))
 	}
-	c.JSON(http.StatusOK, result)
+	return res, nil
 }
 
-//GetLatestVersion return latest version number
-func (a *API) GetLatestVersion(c *gin.Context) {
-	namespace := c.Param("namespace")
-	name := c.Param("name")
-	snapshot, err := a.Metadata.GetSnapshot(c.Request.Context(), namespace, name, "", true)
+// PromoteSnapshot marks specified snapshot as latest
+func (a *API) PromoteSnapshot(ctx context.Context, req *stencilv1.PromoteSnapshotRequest) (*stencilv1.PromoteSnapshotResponse, error) {
+	st, err := a.Metadata.GetSnapshotByID(ctx, req.Id)
 	if err != nil {
-		c.Error(err).SetMeta(models.ErrGetMetadataFailed)
-		return
-	}
-	c.JSON(http.StatusOK, gin.H{"version": snapshot.Version})
-}
-
-//UpdateLatestVersion return latest version number
-func (a *API) UpdateLatestVersion(c *gin.Context) {
-	namespace := c.Param("namespace")
-	payload := &models.MetadataUpdateRequest{
-		Namespace: namespace,
-	}
-	if err := c.ShouldBind(payload); err != nil {
-		c.Error(err).SetMeta(models.ErrMissingFormData)
-		return
+		if err == snapshot.ErrNotFound {
+			return nil, status.Error(codes.NotFound, err.Error())
+		}
+		return nil, status.Error(codes.Internal, err.Error())
 	}
-	err := a.Metadata.UpdateLatestVersion(c.Request.Context(), &snapshot.Snapshot{
-		Namespace: namespace,
-		Name:      payload.Name,
-		Version:   payload.Version,
-	})
+	err = a.Metadata.UpdateLatestVersion(ctx, st)
 	if err != nil {
-		c.Error(err).SetMeta(models.ErrMetadataUpdateFailed)
-		return
+		return nil, err
 	}
-	c.JSON(http.StatusOK, gin.H{"message": "success"})
+	return &stencilv1.PromoteSnapshotResponse{
+		Snapshot: fromSnapshotToProto(st),
+	}, nil
 }