[18.03 backport] makefile and linting changes, bump go 1.11

.circleci/config.yml

@@ -11,7 +11,7 @@ jobs:
       # Needed to install go
       OS: linux
       ARCH: amd64
-      GOVERSION: 1.10.3
+      GOVERSION: 1.11
       # Needed to install protoc
       PROTOC: https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
 

.gitignore

@@ -31,3 +31,6 @@ bin/swarmkitstate
 
 # ignore code coverage output
 *coverage.txt
+
+# dev sync, if used
+/.docker-sync/

.golangci.yml

@@ -0,0 +1,14 @@
+run:
+  tests: false
+linters: 
+  disable-all: true
+  enable:
+    - misspell
+    - gofmt
+    - goimports
+    - golint
+    - ineffassign
+    - deadcode
+    - unconvert
+    - govet
+

BUILDING.md

@@ -113,3 +113,15 @@ NB: As of version 3.0.0-7 the Debian `protobuf-compiler` package lacks
 a dependency on `libprotobuf-dev` which contains some standard proto
 definitions, be sure to install both packages. This is [Debian bug
 #842158](https://bugs.debian.org/842158).
+
+### Build in a container instead of your local environment
+
+You can also choose to use a container to build SwarmKit and run tests. Simply
+set the `DOCKER_SWARMKIT_USE_CONTAINER` environment variable to any value,
+export it, then run `make` targets as you would have done within your local
+environment.
+
+Additionally, if your OS is not Linux, you might want to set and export the
+`DOCKER_SWARMKIT_USE_DOCKER_SYNC` environment variable, which will make use of
+[docker-sync](https://github.com/EugenMayer/docker-sync) to sync the code to
+the container, instead of native mounted volumes.

Dockerfile

@@ -0,0 +1,30 @@
+# NOTE(dperny): for some reason, alpine was giving me trouble
+FROM golang:1.11.0-stretch
+
+RUN apt-get update && apt-get install -y make git unzip
+
+# should stay consistent with the version we use in Circle builds
+ARG PROTOC_VERSION=3.5.0
+# make a directory to do these operations in
+RUN export PROTOC_TMP_DIR=protoc && mkdir -p $PROTOC_TMP_DIR && cd $PROTOC_TMP_DIR \
+  # download the pre-built protoc binary
+  && curl --silent --show-error --location --output protoc.zip \
+    https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-x86_64.zip \
+  # move the binary to /bin. move the well-known types ot /usr/local/include
+  && unzip protoc.zip && mv bin/protoc /bin/protoc && mv include/* /usr/local/include \
+  # remove all of the installation files
+  && cd .. && rm -rf $PROTOC_TMP_DIR
+
+WORKDIR /go/src/github.com/docker/swarmkit/
+
+# install the dependencies from `make setup`
+# we only copy `direct.mk` to avoid busting the cache too easily
+COPY direct.mk .
+RUN make --file=direct.mk setup
+
+# now we can copy the rest
+COPY . .
+
+# default to just `make`. If you want to change the default command, change the
+# default make command, not this command.
+CMD ["make"]

Makefile

@@ -1,159 +1,17 @@
 # Root directory of the project (absolute path).
 ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
-# Base path used to install.
-DESTDIR=/usr/local
-
-# Used to populate version variable in main package.
-VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always)
-
 PROJECT_ROOT=github.com/docker/swarmkit
 
-# Race detector is only supported on amd64.
-RACE := $(shell test $$(go env GOARCH) != "amd64" || (echo "-race"))
-
-# Project packages.
-PACKAGES=$(shell go list ./... | grep -v /vendor/)
-INTEGRATION_PACKAGE=${PROJECT_ROOT}/integration
-
-# Project binaries.
-COMMANDS=swarmd swarmctl swarm-bench swarm-rafttool protoc-gen-gogoswarm
-BINARIES=$(addprefix bin/,$(COMMANDS))
-
-VNDR=$(shell which vndr || echo '')
-
-GO_LDFLAGS=-ldflags "-X `go list ./version`.Version=$(VERSION)"
-
-.PHONY: clean all AUTHORS fmt vet lint build binaries test integration setup generate protos checkprotos coverage ci check help install uninstall dep-validate
-.DEFAULT: default
-
-all: check binaries test integration ## run fmt, vet, lint, build the binaries and run the tests
-
-check: fmt vet lint ineffassign misspell ## run fmt, vet, lint, ineffassign, misspell
-
-ci: check binaries checkprotos coverage coverage-integration ## to be used by the CI
-
-AUTHORS: .mailmap .git/HEAD
-	git log --format='%aN <%aE>' | sort -fu > $@
-
-# This only needs to be generated by hand when cutting full releases.
-version/version.go:
-	./version/version.sh > $@
-
-setup: ## install dependencies
-	@echo "🐳 $@"
-	# TODO(stevvooe): Install these from the vendor directory
-	@go get -u github.com/golang/lint/golint
-	#@go get -u github.com/kisielk/errcheck
-	@go get -u github.com/gordonklaus/ineffassign
-	@go get -u github.com/client9/misspell/cmd/misspell
-	@go get -u github.com/lk4d4/vndr
-	@go get -u github.com/stevvooe/protobuild
-
-generate: protos
-	@echo "🐳 $@"
-	@PATH=${ROOTDIR}/bin:${PATH} go generate -x ${PACKAGES}
-
-protos: bin/protoc-gen-gogoswarm ## generate protobuf
-	@echo "🐳 $@"
-	@PATH=${ROOTDIR}/bin:${PATH} protobuild ${PACKAGES}
-
-checkprotos: generate ## check if protobufs needs to be generated again
-	@echo "🐳 $@"
-	@test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \
-		((git diff | cat) && \
-		(echo "👹 please run 'make generate' when making changes to proto files" && false))
-
-# Depends on binaries because vet will silently fail if it can't load compiled
-# imports
-vet: binaries ## run go vet
-	@echo "🐳 $@"
-	@test -z "$$(go vet ${PACKAGES} 2>&1 | grep -v 'constant [0-9]* not a string in call to Errorf' | egrep -v '(timestamp_test.go|duration_test.go|exit status 1)' | tee /dev/stderr)"
-
-misspell:
-	@echo "🐳 $@"
-	@test -z "$$(find . -type f | grep -v vendor/ | grep -v bin/ | grep -v .git/ | grep -v MAINTAINERS | xargs misspell | tee /dev/stderr)"
-
-fmt: ## run go fmt
-	@echo "🐳 $@"
-	@test -z "$$(gofmt -s -l . | grep -v vendor/ | grep -v ".pb.go$$" | tee /dev/stderr)" || \
-		(echo "👹 please format Go code with 'gofmt -s -w'" && false)
-	@test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
-		(echo "👹 please indent proto files with tabs only" && false)
-	@test -z "$$(find . -path ./vendor -prune -o -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
-		(echo "👹 meta fields in proto files must have option (gogoproto.nullable) = false" && false)
-
-lint: ## run go lint
-	@echo "🐳 $@"
-	@test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
-
-ineffassign: ## run ineffassign
-	@echo "🐳 $@"
-	@test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
-
-#errcheck: ## run go errcheck
-#	@echo "🐳 $@"
-#	@test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)"
-
-build: ## build the go packages
-	@echo "🐳 $@"
-	@go build -i -tags "${DOCKER_BUILDTAGS}" -v ${GO_LDFLAGS} ${GO_GCFLAGS} ${PACKAGES}
-
-test: ## run tests, except integration tests
-	@echo "🐳 $@"
-	@go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES})
-
-integration: ## run integration tests
-	@echo "🐳 $@"
-	@go test -parallel 8 ${RACE} -tags "${DOCKER_BUILDTAGS}" ${INTEGRATION_PACKAGE}
-
-FORCE:
-
-# Build a binary from a cmd.
-bin/%: cmd/% FORCE
-	@test $$(go list) = "${PROJECT_ROOT}" || \
-		(echo "👹 Please correctly set up your Go build environment. This project must be located at <GOPATH>/src/${PROJECT_ROOT}" && false)
-	@echo "🐳 $@"
-	@go build -i -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS}  ${GO_GCFLAGS} ./$<
-
-binaries: $(BINARIES) ## build binaries
-	@echo "🐳 $@"
-
-clean: ## clean up binaries
-	@echo "🐳 $@"
-	@rm -f $(BINARIES)
-
-install: $(BINARIES) ## install binaries
-	@echo "🐳 $@"
-	@mkdir -p $(DESTDIR)/bin
-	@install $(BINARIES) $(DESTDIR)/bin
-
-uninstall:
-	@echo "🐳 $@"
-	@rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARIES)))
-
-coverage: ## generate coverprofiles from the unit tests
-	@echo "🐳 $@"
-	@( for pkg in $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES}); do \
-		go test -i ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \
-		go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../$$pkg/coverage.txt" -covermode=atomic $$pkg || exit; \
-	done )
-
-coverage-integration: ## generate coverprofiles from the integration tests
-	@echo "🐳 $@"
-	go test ${RACE} -tags "${DOCKER_BUILDTAGS}" -test.short -coverprofile="../../../${INTEGRATION_PACKAGE}/coverage.txt" -covermode=atomic ${INTEGRATION_PACKAGE}
-
-help: ## this help
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
-
-dep-validate:
-	@echo "+ $@"
-	$(if $(VNDR), , \
-		$(error Please install vndr: go get github.com/lk4d4/vndr))
-	@rm -Rf .vendor.bak
-	@mv vendor .vendor.bak
-	@$(VNDR)
-	@test -z "$$(diff -r vendor .vendor.bak 2>&1 | tee /dev/stderr)" || \
-		(echo >&2 "+ inconsistent dependencies! what you have in vendor.conf does not match with what you have in vendor" && false)
-	@rm -Rf vendor
-	@mv .vendor.bak vendor
+SHELL := /bin/bash
+
+# stop here. do we want to run everything inside of a container, or do we want
+# to run it directly on the host? if the user has set ANY non-empty value for
+# the variable DOCKER_SWARMKIT_USE_CONTAINER, then we do all of the making
+# inside of a container. We will default to using no container, to avoid
+# breaking anyone's workflow
+ifdef DOCKER_SWARMKIT_USE_CONTAINER
+include containerized.mk
+else
+include direct.mk
+endif

agent/agent.go

@@ -573,7 +573,7 @@ func (a *Agent) nodeDescriptionWithHostname(ctx context.Context, tlsInfo *api.No
 
 	// Override hostname and TLS info
 	if desc != nil {
-		if a.config.Hostname != "" && desc != nil {
+		if a.config.Hostname != "" {
 			desc.Hostname = a.config.Hostname
 		}
 		desc.TLSInfo = tlsInfo

agent/errors.go

@@ -13,10 +13,5 @@ var (
 	errAgentStarted    = errors.New("agent: already started")
 	errAgentNotStarted = errors.New("agent: not started")
 
-	errTaskNoController         = errors.New("agent: no task controller")
-	errTaskNotAssigned          = errors.New("agent: task not assigned")
-	errTaskStatusUpdateNoChange = errors.New("agent: no change in task status")
-	errTaskUnknown              = errors.New("agent: task unknown")
-
-	errTaskInvalid = errors.New("task: invalid")
+	errTaskUnknown = errors.New("agent: task unknown")
 )

agent/exec/containerd/adapter.go

@@ -17,7 +17,7 @@ import (
 	"github.com/docker/swarmkit/api/naming"
 	"github.com/docker/swarmkit/log"
 	gogotypes "github.com/gogo/protobuf/types"
-	"github.com/opencontainers/runtime-spec/specs-go"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -310,7 +310,7 @@ func (c *containerAdapter) shutdown(ctx context.Context) error {
 
 	var (
 		sig     syscall.Signal
-		timeout = time.Duration(10 * time.Second)
+		timeout = 10 * time.Second
 		err     error
 	)
 

agent/exec/controller_stub.go

@@ -1,10 +1,11 @@
 package exec
 
 import (
-	"github.com/docker/swarmkit/api"
-	"golang.org/x/net/context"
 	"runtime"
 	"strings"
+
+	"github.com/docker/swarmkit/api"
+	"golang.org/x/net/context"
 )
 
 // StubController implements the Controller interface,

agent/exec/dockerapi/adapter.go

@@ -144,15 +144,13 @@ func (c *containerAdapter) removeNetworks(ctx context.Context) error {
 }
 
 func (c *containerAdapter) create(ctx context.Context) error {
-	if _, err := c.client.ContainerCreate(ctx,
+	_, err := c.client.ContainerCreate(ctx,
 		c.container.config(),
 		c.container.hostConfig(),
 		c.container.networkingConfig(),
-		c.container.name()); err != nil {
-		return err
-	}
+		c.container.name())
 
-	return nil
+	return err
 }
 
 func (c *containerAdapter) start(ctx context.Context) error {

agent/exec/dockerapi/controller.go

@@ -654,7 +654,7 @@ func parsePortMap(portMap nat.PortMap) ([]*api.PortConfig, error) {
 			return nil, err
 		}
 
-		protocol := api.ProtocolTCP
+		var protocol api.PortConfig_Protocol
 		switch strings.ToLower(parts[1]) {
 		case "tcp":
 			protocol = api.ProtocolTCP

agent/exec/dockerapi/docker_client_stub.go

@@ -1,16 +1,17 @@
 package dockerapi
 
 import (
+	"io"
+	"runtime"
+	"strings"
+	"time"
+
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"golang.org/x/net/context"
-	"io"
-	"runtime"
-	"strings"
-	"time"
 )
 
 // StubAPIClient implements the client.APIClient interface, but allows

agent/exec/dockerapi/executor.go

@@ -3,6 +3,7 @@ package dockerapi
 import (
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/docker/docker/api/types/filters"
 	engineapi "github.com/docker/docker/client"
@@ -11,7 +12,6 @@ import (
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/log"
 	"golang.org/x/net/context"
-	"sync"
 )
 
 type executor struct {

agent/session.go

@@ -16,7 +16,6 @@ import (
 
 var (
 	dispatcherRPCTimeout = 5 * time.Second
-	errSessionDisconnect = errors.New("agent: session disconnect") // instructed to disconnect
 	errSessionClosed     = errors.New("agent: session closed")
 )
 
@@ -129,7 +128,7 @@ func (s *session) start(ctx context.Context, description *api.NodeDescription) e
 	// `ctx` is done and hence fail to propagate the timeout error to the agent.
 	// If the error is not propogated to the agent, the agent will not close
 	// the session or rebuild a new sesssion.
-	sessionCtx, cancelSession := context.WithCancel(ctx)
+	sessionCtx, cancelSession := context.WithCancel(ctx) //nolint:govet
 
 	// Need to run Session in a goroutine since there's no way to set a
 	// timeout for an individual Recv call in a stream.
@@ -152,7 +151,7 @@ func (s *session) start(ctx context.Context, description *api.NodeDescription) e
 	select {
 	case err := <-errChan:
 		if err != nil {
-			return err
+			return err //nolint:govet
 		}
 	case <-time.After(dispatcherRPCTimeout):
 		cancelSession()