From fbe92224692d871fc0acfaacb99249062a7578ab Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 9 Jun 2018 20:59:33 +0800 Subject: [PATCH 1/7] refactor: allow container mgr to use custom log driver 1. use logger.Info to contain container's information 2. add log driver validation in daemon 3. make the openContainerIO/attachContainerIO friendly Signed-off-by: Wei Fu --- daemon/containerio/jsonfile.go | 5 +-- daemon/containerio/options.go | 12 ++++--- daemon/logger/info.go | 17 +++++++++ daemon/mgr/container.go | 58 +++++++++++++++--------------- daemon/mgr/container_logger.go | 44 +++++++++++++++++++++++ daemon/mgr/container_validation.go | 16 +++++++++ 6 files changed, 118 insertions(+), 34 deletions(-) create mode 100644 daemon/logger/info.go create mode 100644 daemon/mgr/container_logger.go diff --git a/daemon/containerio/jsonfile.go b/daemon/containerio/jsonfile.go index 8f1bfc908..ff913785b 100644 --- a/daemon/containerio/jsonfile.go +++ b/daemon/containerio/jsonfile.go @@ -37,11 +37,12 @@ func (jf *jsonFile) Name() string { } func (jf *jsonFile) Init(opt *Option) error { - if _, err := os.Stat(opt.rootDir); err != nil { + rootDir := opt.info.ContainerRootDir + if _, err := os.Stat(rootDir); err != nil { return err } - logPath := filepath.Join(opt.rootDir, jsonFilePathName) + logPath := filepath.Join(rootDir, jsonFilePathName) w, err := jsonfile.NewJSONLogFile(logPath, 0644) if err != nil { return err diff --git a/daemon/containerio/options.go b/daemon/containerio/options.go index 58b085460..34265414c 100644 --- a/daemon/containerio/options.go +++ b/daemon/containerio/options.go @@ -6,12 +6,16 @@ import ( "os" "github.com/alibaba/pouch/cri/stream/remotecommand" + "github.com/alibaba/pouch/daemon/logger" ) // Option is used to pass some data into ContainerIO. +// +// FIXME(fuwei): use logger.Info to separate options and backends. type Option struct { + info logger.Info + id string - rootDir string stdin bool muxDisabled bool backends map[string]struct{} @@ -41,10 +45,10 @@ func WithID(id string) func(*Option) { } } -// WithRootDir specified the container's root dir. -func WithRootDir(dir string) func(*Option) { +// WithLoggerInfo specified the container's logger information. +func WithLoggerInfo(info logger.Info) func(*Option) { return func(opt *Option) { - opt.rootDir = dir + opt.info = info } } diff --git a/daemon/logger/info.go b/daemon/logger/info.go new file mode 100644 index 000000000..bc6bbb57f --- /dev/null +++ b/daemon/logger/info.go @@ -0,0 +1,17 @@ +package logger + +import "github.com/alibaba/pouch/pkg/utils" + +// Info provides container information for log driver. +type Info struct { + LogConfig map[string]string + + ContainerID string + ContainerLabels map[string]string + ContainerRootDir string +} + +// ID returns the container truncated ID. +func (i *Info) ID() string { + return utils.TruncateID(i.ContainerID) +} diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index b2e617ecd..9fb7f1c4b 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -189,12 +189,13 @@ func NewContainerManager(ctx context.Context, store *meta.Store, cli ctrd.APICli func (mgr *ContainerManager) Restore(ctx context.Context) error { fn := func(obj meta.Object) error { container, ok := obj.(*Container) - id := container.ID if !ok { // object has not type of Container return nil } + id := container.ID + // map container's name to id. mgr.NameToID.Put(container.Name, id) @@ -207,7 +208,7 @@ func (mgr *ContainerManager) Restore(ctx context.Context) error { } // recover the running or paused container. - io, err := mgr.openContainerIO(id, container.Config.OpenStdin) + io, err := mgr.openContainerIO(container) if err != nil { logrus.Errorf("failed to recover container: %s, %v", id, err) } @@ -298,9 +299,14 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty } } - // FIXME(fuwei): only support LogConfig is json-file right now - config.HostConfig.LogConfig = &types.LogConfig{ - LogDriver: types.LogConfigLogDriverJSONFile, + // set default log driver and validate for logger driver + if config.HostConfig.LogConfig == nil { + config.HostConfig.LogConfig = &types.LogConfig{ + LogDriver: types.LogConfigLogDriverJSONFile, + } + } + if err := validateLogConfig(config.HostConfig.LogConfig); err != nil { + return nil, err } container := &Container{ @@ -554,7 +560,7 @@ func (mgr *ContainerManager) createContainerdContainer(ctx context.Context, c *C } // open container's stdio. - io, err := mgr.openContainerIO(c.ID, c.Config.OpenStdin) + io, err := mgr.openContainerIO(c) if err != nil { return errors.Wrap(err, "failed to open io") } @@ -706,7 +712,7 @@ func (mgr *ContainerManager) Attach(ctx context.Context, name string, attach *At return err } - _, err = mgr.openAttachIO(c.ID, attach) + _, err = mgr.openAttachIO(c, attach) if err != nil { return err } @@ -1455,23 +1461,22 @@ func (mgr *ContainerManager) Disconnect(ctx context.Context, containerName, netw return nil } -func (mgr *ContainerManager) openContainerIO(id string, stdin bool) (*containerio.IO, error) { - if io := mgr.IOs.Get(id); io != nil { +func (mgr *ContainerManager) openContainerIO(c *Container) (*containerio.IO, error) { + if io := mgr.IOs.Get(c.ID); io != nil { return io, nil } - root := mgr.Store.Path(id) + logInfo := mgr.convContainerToLoggerInfo(c) options := []func(*containerio.Option){ - containerio.WithID(id), - containerio.WithRootDir(root), - containerio.WithJSONFile(), - containerio.WithStdin(stdin), + containerio.WithID(c.ID), + containerio.WithLoggerInfo(logInfo), + containerio.WithStdin(c.Config.OpenStdin), } - io := containerio.NewIO(containerio.NewOption(options...)) - - mgr.IOs.Put(id, io) + options = append(options, optionsForContainerio(c)...) + io := containerio.NewIO(containerio.NewOption(options...)) + mgr.IOs.Put(c.ID, io) return io, nil } @@ -1542,6 +1547,7 @@ func (mgr *ContainerManager) connectToNetwork(ctx context.Context, container *Co return mgr.updateNetworkConfig(container, networkIDOrName, endpoint.EndpointConfig) } +// FIXME: remove this useless functions func (mgr *ContainerManager) updateNetworkSettings(container *Container, n libnetwork.Network) error { if container.NetworkSettings == nil { container.NetworkSettings = &types.NetworkSettings{Networks: make(map[string]*types.EndpointSettings)} @@ -1594,19 +1600,17 @@ func (mgr *ContainerManager) openExecIO(id string, attach *AttachConfig) (*conta } io := containerio.NewIO(containerio.NewOption(options...)) - mgr.IOs.Put(id, io) - return io, nil } -func (mgr *ContainerManager) openAttachIO(id string, attach *AttachConfig) (*containerio.IO, error) { - rootDir := mgr.Store.Path(id) +func (mgr *ContainerManager) openAttachIO(c *Container, attach *AttachConfig) (*containerio.IO, error) { + logInfo := mgr.convContainerToLoggerInfo(c) options := []func(*containerio.Option){ - containerio.WithID(id), - containerio.WithRootDir(rootDir), - containerio.WithJSONFile(), + containerio.WithID(c.ID), + containerio.WithLoggerInfo(logInfo), } + options = append(options, optionsForContainerio(c)...) if attach != nil { options = append(options, attachConfigToOptions(attach)...) @@ -1615,15 +1619,13 @@ func (mgr *ContainerManager) openAttachIO(id string, attach *AttachConfig) (*con options = append(options, containerio.WithDiscard()) } - io := mgr.IOs.Get(id) + io := mgr.IOs.Get(c.ID) if io != nil { io.AddBackend(containerio.NewOption(options...)) } else { io = containerio.NewIO(containerio.NewOption(options...)) } - - mgr.IOs.Put(id, io) - + mgr.IOs.Put(c.ID, io) return io, nil } diff --git a/daemon/mgr/container_logger.go b/daemon/mgr/container_logger.go new file mode 100644 index 000000000..1a02111d8 --- /dev/null +++ b/daemon/mgr/container_logger.go @@ -0,0 +1,44 @@ +package mgr + +import ( + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/daemon/containerio" + "github.com/alibaba/pouch/daemon/logger" + + "github.com/sirupsen/logrus" +) + +func optionsForContainerio(c *Container) []func(*containerio.Option) { + optFuncs := make([]func(*containerio.Option), 0, 1) + + cfg := c.HostConfig.LogConfig + if cfg == nil || cfg.LogDriver == types.LogConfigLogDriverNone { + return optFuncs + } + + switch cfg.LogDriver { + case types.LogConfigLogDriverJSONFile: + optFuncs = append(optFuncs, containerio.WithJSONFile()) + default: + logrus.Warnf("not support %v log driver yet", cfg.LogDriver) + } + return optFuncs +} + +// convContainerToLoggerInfo uses logger.Info to wrap container information. +func (mgr *ContainerManager) convContainerToLoggerInfo(c *Container) logger.Info { + logCfg := make(map[string]string) + if cfg := c.HostConfig.LogConfig; cfg != nil && cfg.LogDriver != types.LogConfigLogDriverNone { + logCfg = cfg.LogOpts + } + + // TODO: + // 1. add more fields into logger.Info + // 2. separate the logic about retrieving container root dir from mgr. + return logger.Info{ + LogConfig: logCfg, + ContainerID: c.ID, + ContainerLabels: c.Config.Labels, + ContainerRootDir: mgr.Store.Path(c.ID), + } +} diff --git a/daemon/mgr/container_validation.go b/daemon/mgr/container_validation.go index a0cb97fad..a608080b8 100644 --- a/daemon/mgr/container_validation.go +++ b/daemon/mgr/container_validation.go @@ -1,6 +1,8 @@ package mgr import ( + "fmt" + "github.com/alibaba/pouch/apis/types" ) @@ -11,3 +13,17 @@ func (mgr *ContainerManager) verifyContainerSetting(hostConfig *types.HostConfig } return nil } + +// validateLogConfig is used to verify the correctness of log configuration. +func validateLogConfig(logCfg *types.LogConfig) error { + if logCfg == nil { + return nil + } + + switch logCfg.LogDriver { + case types.LogConfigLogDriverNone, types.LogConfigLogDriverJSONFile: + return nil + default: + return fmt.Errorf("not support (%v) log driver yet", logCfg.LogDriver) + } +} From 39f8a4de5d3cdc8f2da48192d411589babe6155c Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sat, 9 Jun 2018 22:43:37 +0800 Subject: [PATCH 2/7] feature: allow cli to use custom log driver and opts Signed-off-by: Wei Fu --- apis/opts/log_options.go | 36 ++++++++ apis/opts/log_options_test.go | 74 +++++++++++++++++ cli/common_flags.go | 4 + cli/container.go | 13 +++ test/cli_create_log_options_test.go | 123 ++++++++++++++++++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 apis/opts/log_options.go create mode 100644 apis/opts/log_options_test.go create mode 100644 test/cli_create_log_options_test.go diff --git a/apis/opts/log_options.go b/apis/opts/log_options.go new file mode 100644 index 000000000..35ec095cf --- /dev/null +++ b/apis/opts/log_options.go @@ -0,0 +1,36 @@ +package opts + +import ( + "errors" + "fmt" + "strings" +) + +// ParseLogOptions parses [key=value] slice-type log options into map. +func ParseLogOptions(driver string, logOpts []string) (map[string]string, error) { + opts, err := convertKVStringsToMap(logOpts) + if err != nil { + return nil, err + } + + if driver == "none" && len(opts) > 0 { + return nil, fmt.Errorf("don't allow to set logging opts for driver %s", driver) + } + return opts, nil +} + +// convertKVStringsToMap converts ["key=value"] into {"key":"value"} +// +// TODO(fuwei): make it common in the opts.ParseXXX(). +func convertKVStringsToMap(values []string) (map[string]string, error) { + kvs := make(map[string]string, len(values)) + + for _, value := range values { + terms := strings.SplitN(value, "=", 2) + if len(terms) != 2 { + return nil, errors.New("the format must be key=value") + } + kvs[terms[0]] = terms[1] + } + return kvs, nil +} diff --git a/apis/opts/log_options_test.go b/apis/opts/log_options_test.go new file mode 100644 index 000000000..c061e4c06 --- /dev/null +++ b/apis/opts/log_options_test.go @@ -0,0 +1,74 @@ +package opts + +import ( + "reflect" + "testing" +) + +func TestParseLogOptions(t *testing.T) { + type tCases struct { + driver string + logOpts []string + + hasError bool + expected map[string]string + } + + for idx, tc := range []tCases{ + { + driver: "", + logOpts: nil, + hasError: false, + expected: map[string]string{}, + }, { + driver: "none", + logOpts: nil, + hasError: false, + expected: map[string]string{}, + }, { + driver: "none", + logOpts: []string{"haha"}, + hasError: true, + expected: nil, + }, { + driver: "none", + logOpts: []string{"test=1"}, + hasError: true, + expected: nil, + }, { + driver: "json-file", + logOpts: []string{"test=1"}, + hasError: false, + expected: map[string]string{ + "test": "1", + }, + }, { + driver: "json-file", + logOpts: []string{"test=1=1"}, + hasError: false, + expected: map[string]string{ + "test": "1=1", + }, + }, { + driver: "json-file", + logOpts: []string{"test=1", "flag=oops", "test=2"}, + hasError: false, + expected: map[string]string{ + "test": "2", + "flag": "oops", + }, + }, + } { + got, err := ParseLogOptions(tc.driver, tc.logOpts) + if err == nil && tc.hasError { + t.Fatalf("[%d case] should have error here, but got nothing", idx) + } + if err != nil && !tc.hasError { + t.Fatalf("[%d case] should have no error here, but got error(%v)", idx, err) + } + + if !reflect.DeepEqual(got, tc.expected) { + t.Fatalf("[%d case] should have (%v), but got (%v)", idx, tc.expected, got) + } + } +} diff --git a/cli/common_flags.go b/cli/common_flags.go index 5aab5f008..c64396dde 100644 --- a/cli/common_flags.go +++ b/cli/common_flags.go @@ -42,6 +42,10 @@ func addCommonFlags(flagSet *pflag.FlagSet) *container { flagSet.StringVar(&c.ipcMode, "ipc", "", "IPC namespace to use") flagSet.StringSliceVarP(&c.labels, "label", "l", nil, "Set labels for a container") + // log driver and log options + flagSet.StringVar(&c.logDriver, "log-driver", "json-file", "Logging driver for the container") + flagSet.StringSliceVar(&c.logOpts, "log-opt", nil, "Log driver options") + // memory flagSet.StringVarP(&c.memory, "memory", "m", "", "Memory limit") diff --git a/cli/container.go b/cli/container.go index 6750e3ace..d569da055 100644 --- a/cli/container.go +++ b/cli/container.go @@ -72,6 +72,10 @@ type container struct { ulimit Ulimit pidsLimit int64 + // log driver and log option + logDriver string + logOpts []string + //add for rich container mode rich bool richMode string @@ -172,6 +176,11 @@ func (c *container) config() (*types.ContainerCreateConfig, error) { return nil, err } + logOpts, err := opts.ParseLogOptions(c.logDriver, c.logOpts) + if err != nil { + return nil, err + } + config := &types.ContainerCreateConfig{ ContainerConfig: types.ContainerConfig{ Tty: c.tty, @@ -242,6 +251,10 @@ func (c *container) config() (*types.ContainerCreateConfig, error) { CapDrop: c.capDrop, PortBindings: portBindings, OomScoreAdj: c.oomScoreAdj, + LogConfig: &types.LogConfig{ + LogDriver: c.logDriver, + LogOpts: logOpts, + }, }, NetworkingConfig: networkingConfig, diff --git a/test/cli_create_log_options_test.go b/test/cli_create_log_options_test.go new file mode 100644 index 000000000..5154eb3eb --- /dev/null +++ b/test/cli_create_log_options_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/test/command" + "github.com/alibaba/pouch/test/environment" + + "github.com/go-check/check" + "github.com/gotestyourself/gotestyourself/icmd" +) + +// PouchCreateLogOptionsSuite is the test suite for create CLI with LogOptions. +type PouchCreateLogOptionsSuite struct{} + +func init() { + check.Suite(&PouchCreateLogOptionsSuite{}) +} + +// SetUpSuite does common setup in the beginning of each test suite. +func (suite *PouchCreateLogOptionsSuite) SetUpSuite(c *check.C) { + SkipIfFalse(c, environment.IsLinux) + + PullImage(c, busyboxImage) +} + +// TestFailNoneWithOpts fails. +func (suite *PouchCreateLogOptionsSuite) TestFailNoneWithOpts(c *check.C) { + cname := "TestCreateLogOptions_Fail_none_with_opts" + expected := "don't allow to set logging opts for driver none" + + args := []string{"create"} + args = append(args, getArgsForLogOptions("none", []string{"tag=1"})...) + args = append(args, "--name", cname, busyboxImage) + + res := command.PouchRun(args...) + if got := res.Combined(); !strings.Contains(got, expected) { + c.Fatalf("expected to contains (%v), but got (%v)", expected, got) + } +} + +// TestFailNotSupportDriver fails. +func (suite *PouchCreateLogOptionsSuite) TestFailNotSupportDriver(c *check.C) { + cname := "TestCreateLogOptions_Fail_not_support_driver" + + driver := "notyet" + expected := fmt.Sprintf("not support (%v) log driver yet", driver) + + args := []string{"create"} + args = append(args, getArgsForLogOptions(driver, nil)...) + args = append(args, "--name", cname, busyboxImage) + + res := command.PouchRun(args...) + if got := res.Combined(); !strings.Contains(got, expected) { + c.Fatalf("expected to contains (%v), but got (%v)", expected, got) + } +} + +// TestOK tests happy cases for log options +func (suite *PouchCreateLogOptionsSuite) TestOK(c *check.C) { + type tCase struct { + cname string + driver string + logOpts []string + expected map[string]string + } + + for _, tc := range []tCase{ + { + cname: "TestCreateLogOptions_none", + driver: "none", + expected: nil, + }, { + cname: "TestCreateLogOptions_jsonfile", + driver: "json-file", + expected: nil, + }, { + cname: "TestCreateLogOptions_jsonfile_tag=1", + driver: "json-file", + logOpts: []string{"tag=1"}, + expected: map[string]string{ + "tag": "1", + }, + }, + } { + args := []string{"create"} + args = append(args, getArgsForLogOptions(tc.driver, tc.logOpts)...) + args = append(args, "--name", tc.cname, busyboxImage) + + command.PouchRun(args...).Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, tc.cname) + + cfg := suite.getContainerLogConfig(c, tc.cname) + c.Assert(cfg.LogDriver, check.Equals, tc.driver) + + if !reflect.DeepEqual(cfg.LogOpts, tc.expected) { + c.Errorf("expected to have (%v), but got (%v)", tc.expected, cfg.LogOpts) + } + } +} + +func getArgsForLogOptions(driver string, logOpts []string) []string { + args := []string{"--log-driver", driver} + for _, opt := range logOpts { + args = append(args, "--log-opt", opt) + } + return args +} + +func (suite *PouchCreateLogOptionsSuite) getContainerLogConfig(c *check.C, idOrName string) *types.LogConfig { + output := command.PouchRun("inspect", idOrName).Combined() + result := []types.ContainerJSON{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + + c.Assert(result, check.HasLen, 1) + return result[0].HostConfig.LogConfig +} From bb7318b88bf6f4b96fecbb9fbe3711aea5810ee6 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 10 Jun 2018 09:50:27 +0800 Subject: [PATCH 3/7] feature: add github.com/RackSec/srslog dependence Signed-off-by: Wei Fu --- .../RackSec/srslog/CODE_OF_CONDUCT.md | 50 +++++ vendor/github.com/RackSec/srslog/LICENSE | 27 +++ vendor/github.com/RackSec/srslog/README.md | 131 ++++++++++++ vendor/github.com/RackSec/srslog/constants.go | 68 ++++++ vendor/github.com/RackSec/srslog/dialer.go | 87 ++++++++ vendor/github.com/RackSec/srslog/formatter.go | 58 +++++ vendor/github.com/RackSec/srslog/framer.go | 24 +++ vendor/github.com/RackSec/srslog/net_conn.go | 30 +++ vendor/github.com/RackSec/srslog/srslog.go | 97 +++++++++ .../github.com/RackSec/srslog/srslog_unix.go | 54 +++++ vendor/github.com/RackSec/srslog/writer.go | 198 ++++++++++++++++++ vendor/vendor.json | 6 + 12 files changed, 830 insertions(+) create mode 100644 vendor/github.com/RackSec/srslog/CODE_OF_CONDUCT.md create mode 100644 vendor/github.com/RackSec/srslog/LICENSE create mode 100644 vendor/github.com/RackSec/srslog/README.md create mode 100644 vendor/github.com/RackSec/srslog/constants.go create mode 100644 vendor/github.com/RackSec/srslog/dialer.go create mode 100644 vendor/github.com/RackSec/srslog/formatter.go create mode 100644 vendor/github.com/RackSec/srslog/framer.go create mode 100644 vendor/github.com/RackSec/srslog/net_conn.go create mode 100644 vendor/github.com/RackSec/srslog/srslog.go create mode 100644 vendor/github.com/RackSec/srslog/srslog_unix.go create mode 100644 vendor/github.com/RackSec/srslog/writer.go diff --git a/vendor/github.com/RackSec/srslog/CODE_OF_CONDUCT.md b/vendor/github.com/RackSec/srslog/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..18ac49fc7 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/CODE_OF_CONDUCT.md @@ -0,0 +1,50 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer at [sirsean@gmail.com]. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/vendor/github.com/RackSec/srslog/LICENSE b/vendor/github.com/RackSec/srslog/LICENSE new file mode 100644 index 000000000..9269338fb --- /dev/null +++ b/vendor/github.com/RackSec/srslog/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2015 Rackspace. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/RackSec/srslog/README.md b/vendor/github.com/RackSec/srslog/README.md new file mode 100644 index 000000000..1ae1fd4ef --- /dev/null +++ b/vendor/github.com/RackSec/srslog/README.md @@ -0,0 +1,131 @@ +[![Build Status](https://travis-ci.org/RackSec/srslog.svg?branch=master)](https://travis-ci.org/RackSec/srslog) + +# srslog + +Go has a `syslog` package in the standard library, but it has the following +shortcomings: + +1. It doesn't have TLS support +2. [According to bradfitz on the Go team, it is no longer being maintained.](https://github.com/golang/go/issues/13449#issuecomment-161204716) + +I agree that it doesn't need to be in the standard library. So, I've +followed Brad's suggestion and have made a separate project to handle syslog. + +This code was taken directly from the Go project as a base to start from. + +However, this _does_ have TLS support. + +# Usage + +Basic usage retains the same interface as the original `syslog` package. We +only added to the interface where required to support new functionality. + +Switch from the standard library: + +``` +import( + //"log/syslog" + syslog "github.com/RackSec/srslog" +) +``` + +You can still use it for local syslog: + +``` +w, err := syslog.Dial("", "", syslog.LOG_ERR, "testtag") +``` + +Or to unencrypted UDP: + +``` +w, err := syslog.Dial("udp", "192.168.0.50:514", syslog.LOG_ERR, "testtag") +``` + +Or to unencrypted TCP: + +``` +w, err := syslog.Dial("tcp", "192.168.0.51:514", syslog.LOG_ERR, "testtag") +``` + +But now you can also send messages via TLS-encrypted TCP: + +``` +w, err := syslog.DialWithTLSCertPath("tcp+tls", "192.168.0.52:514", syslog.LOG_ERR, "testtag", "/path/to/servercert.pem") +``` + +And if you need more control over your TLS configuration : + +``` +pool := x509.NewCertPool() +serverCert, err := ioutil.ReadFile("/path/to/servercert.pem") +if err != nil { + return nil, err +} +pool.AppendCertsFromPEM(serverCert) +config := tls.Config{ + RootCAs: pool, +} + +w, err := DialWithTLSConfig(network, raddr, priority, tag, &config) +``` + +(Note that in both TLS cases, this uses a self-signed certificate, where the +remote syslog server has the keypair and the client has only the public key.) + +And then to write log messages, continue like so: + +``` +if err != nil { + log.Fatal("failed to connect to syslog:", err) +} +defer w.Close() + +w.Alert("this is an alert") +w.Crit("this is critical") +w.Err("this is an error") +w.Warning("this is a warning") +w.Notice("this is a notice") +w.Info("this is info") +w.Debug("this is debug") +w.Write([]byte("these are some bytes")) +``` + +# Generating TLS Certificates + +We've provided a script that you can use to generate a self-signed keypair: + +``` +pip install cryptography +python script/gen-certs.py +``` + +That outputs the public key and private key to standard out. Put those into +`.pem` files. (And don't put them into any source control. The certificate in +the `test` directory is used by the unit tests, and please do not actually use +it anywhere else.) + +# Running Tests + +Run the tests as usual: + +``` +go test +``` + +But we've also provided a test coverage script that will show you which +lines of code are not covered: + +``` +script/coverage --html +``` + +That will open a new browser tab showing coverage information. + +# License + +This project uses the New BSD License, the same as the Go project itself. + +# Code of Conduct + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. diff --git a/vendor/github.com/RackSec/srslog/constants.go b/vendor/github.com/RackSec/srslog/constants.go new file mode 100644 index 000000000..600801ee8 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/constants.go @@ -0,0 +1,68 @@ +package srslog + +import ( + "errors" +) + +// Priority is a combination of the syslog facility and +// severity. For example, LOG_ALERT | LOG_FTP sends an alert severity +// message from the FTP facility. The default severity is LOG_EMERG; +// the default facility is LOG_KERN. +type Priority int + +const severityMask = 0x07 +const facilityMask = 0xf8 + +const ( + // Severity. + + // From /usr/include/sys/syslog.h. + // These are the same on Linux, BSD, and OS X. + LOG_EMERG Priority = iota + LOG_ALERT + LOG_CRIT + LOG_ERR + LOG_WARNING + LOG_NOTICE + LOG_INFO + LOG_DEBUG +) + +const ( + // Facility. + + // From /usr/include/sys/syslog.h. + // These are the same up to LOG_FTP on Linux, BSD, and OS X. + LOG_KERN Priority = iota << 3 + LOG_USER + LOG_MAIL + LOG_DAEMON + LOG_AUTH + LOG_SYSLOG + LOG_LPR + LOG_NEWS + LOG_UUCP + LOG_CRON + LOG_AUTHPRIV + LOG_FTP + _ // unused + _ // unused + _ // unused + _ // unused + LOG_LOCAL0 + LOG_LOCAL1 + LOG_LOCAL2 + LOG_LOCAL3 + LOG_LOCAL4 + LOG_LOCAL5 + LOG_LOCAL6 + LOG_LOCAL7 +) + +func validatePriority(p Priority) error { + if p < 0 || p > LOG_LOCAL7|LOG_DEBUG { + return errors.New("log/syslog: invalid priority") + } else { + return nil + } +} diff --git a/vendor/github.com/RackSec/srslog/dialer.go b/vendor/github.com/RackSec/srslog/dialer.go new file mode 100644 index 000000000..47a7b2bea --- /dev/null +++ b/vendor/github.com/RackSec/srslog/dialer.go @@ -0,0 +1,87 @@ +package srslog + +import ( + "crypto/tls" + "net" +) + +// dialerFunctionWrapper is a simple object that consists of a dialer function +// and its name. This is primarily for testing, so we can make sure that the +// getDialer method returns the correct dialer function. However, if you ever +// find that you need to check which dialer function you have, this would also +// be useful for you without having to use reflection. +type dialerFunctionWrapper struct { + Name string + Dialer func() (serverConn, string, error) +} + +// Call the wrapped dialer function and return its return values. +func (df dialerFunctionWrapper) Call() (serverConn, string, error) { + return df.Dialer() +} + +// getDialer returns a "dialer" function that can be called to connect to a +// syslog server. +// +// Each dialer function is responsible for dialing the remote host and returns +// a serverConn, the hostname (or a default if the Writer has not specified a +// hostname), and an error in case dialing fails. +// +// The reason for separate dialers is that different network types may need +// to dial their connection differently, yet still provide a net.Conn interface +// that you can use once they have dialed. Rather than an increasingly long +// conditional, we have a map of network -> dialer function (with a sane default +// value), and adding a new network type is as easy as writing the dialer +// function and adding it to the map. +func (w *Writer) getDialer() dialerFunctionWrapper { + dialers := map[string]dialerFunctionWrapper{ + "": dialerFunctionWrapper{"unixDialer", w.unixDialer}, + "tcp+tls": dialerFunctionWrapper{"tlsDialer", w.tlsDialer}, + } + dialer, ok := dialers[w.network] + if !ok { + dialer = dialerFunctionWrapper{"basicDialer", w.basicDialer} + } + return dialer +} + +// unixDialer uses the unixSyslog method to open a connection to the syslog +// daemon running on the local machine. +func (w *Writer) unixDialer() (serverConn, string, error) { + sc, err := unixSyslog() + hostname := w.hostname + if hostname == "" { + hostname = "localhost" + } + return sc, hostname, err +} + +// tlsDialer connects to TLS over TCP, and is used for the "tcp+tls" network +// type. +func (w *Writer) tlsDialer() (serverConn, string, error) { + c, err := tls.Dial("tcp", w.raddr, w.tlsConfig) + var sc serverConn + hostname := w.hostname + if err == nil { + sc = &netConn{conn: c} + if hostname == "" { + hostname = c.LocalAddr().String() + } + } + return sc, hostname, err +} + +// basicDialer is the most common dialer for syslog, and supports both TCP and +// UDP connections. +func (w *Writer) basicDialer() (serverConn, string, error) { + c, err := net.Dial(w.network, w.raddr) + var sc serverConn + hostname := w.hostname + if err == nil { + sc = &netConn{conn: c} + if hostname == "" { + hostname = c.LocalAddr().String() + } + } + return sc, hostname, err +} diff --git a/vendor/github.com/RackSec/srslog/formatter.go b/vendor/github.com/RackSec/srslog/formatter.go new file mode 100644 index 000000000..e306fd671 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/formatter.go @@ -0,0 +1,58 @@ +package srslog + +import ( + "fmt" + "os" + "time" +) + +const appNameMaxLength = 48 // limit to 48 chars as per RFC5424 + +// Formatter is a type of function that takes the consituent parts of a +// syslog message and returns a formatted string. A different Formatter is +// defined for each different syslog protocol we support. +type Formatter func(p Priority, hostname, tag, content string) string + +// DefaultFormatter is the original format supported by the Go syslog package, +// and is a non-compliant amalgamation of 3164 and 5424 that is intended to +// maximize compatibility. +func DefaultFormatter(p Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(time.RFC3339) + msg := fmt.Sprintf("<%d> %s %s %s[%d]: %s", + p, timestamp, hostname, tag, os.Getpid(), content) + return msg +} + +// UnixFormatter omits the hostname, because it is only used locally. +func UnixFormatter(p Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(time.Stamp) + msg := fmt.Sprintf("<%d>%s %s[%d]: %s", + p, timestamp, tag, os.Getpid(), content) + return msg +} + +// RFC3164Formatter provides an RFC 3164 compliant message. +func RFC3164Formatter(p Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(time.Stamp) + msg := fmt.Sprintf("<%d>%s %s %s[%d]: %s", + p, timestamp, hostname, tag, os.Getpid(), content) + return msg +} + +// if string's length is greater than max, then use the last part +func truncateStartStr(s string, max int) string { + if (len(s) > max) { + return s[len(s) - max:] + } + return s +} + +// RFC5424Formatter provides an RFC 5424 compliant message. +func RFC5424Formatter(p Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(time.RFC3339) + pid := os.Getpid() + appName := truncateStartStr(os.Args[0], appNameMaxLength) + msg := fmt.Sprintf("<%d>%d %s %s %s %d %s - %s", + p, 1, timestamp, hostname, appName, pid, tag, content) + return msg +} diff --git a/vendor/github.com/RackSec/srslog/framer.go b/vendor/github.com/RackSec/srslog/framer.go new file mode 100644 index 000000000..ab46f0de7 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/framer.go @@ -0,0 +1,24 @@ +package srslog + +import ( + "fmt" +) + +// Framer is a type of function that takes an input string (typically an +// already-formatted syslog message) and applies "message framing" to it. We +// have different framers because different versions of the syslog protocol +// and its transport requirements define different framing behavior. +type Framer func(in string) string + +// DefaultFramer does nothing, since there is no framing to apply. This is +// the original behavior of the Go syslog package, and is also typically used +// for UDP syslog. +func DefaultFramer(in string) string { + return in +} + +// RFC5425MessageLengthFramer prepends the message length to the front of the +// provided message, as defined in RFC 5425. +func RFC5425MessageLengthFramer(in string) string { + return fmt.Sprintf("%d %s", len(in), in) +} diff --git a/vendor/github.com/RackSec/srslog/net_conn.go b/vendor/github.com/RackSec/srslog/net_conn.go new file mode 100644 index 000000000..75e4c3ca1 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/net_conn.go @@ -0,0 +1,30 @@ +package srslog + +import ( + "net" +) + +// netConn has an internal net.Conn and adheres to the serverConn interface, +// allowing us to send syslog messages over the network. +type netConn struct { + conn net.Conn +} + +// writeString formats syslog messages using time.RFC3339 and includes the +// hostname, and sends the message to the connection. +func (n *netConn) writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, msg string) error { + if framer == nil { + framer = DefaultFramer + } + if formatter == nil { + formatter = DefaultFormatter + } + formattedMessage := framer(formatter(p, hostname, tag, msg)) + _, err := n.conn.Write([]byte(formattedMessage)) + return err +} + +// close the network connection +func (n *netConn) close() error { + return n.conn.Close() +} diff --git a/vendor/github.com/RackSec/srslog/srslog.go b/vendor/github.com/RackSec/srslog/srslog.go new file mode 100644 index 000000000..b404dff7c --- /dev/null +++ b/vendor/github.com/RackSec/srslog/srslog.go @@ -0,0 +1,97 @@ +package srslog + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "log" + "os" +) + +// This interface allows us to work with both local and network connections, +// and enables Solaris support (see syslog_unix.go). +type serverConn interface { + writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, s string) error + close() error +} + +// New establishes a new connection to the system log daemon. Each +// write to the returned Writer sends a log message with the given +// priority and prefix. +func New(priority Priority, tag string) (w *Writer, err error) { + return Dial("", "", priority, tag) +} + +// Dial establishes a connection to a log daemon by connecting to +// address raddr on the specified network. Each write to the returned +// Writer sends a log message with the given facility, severity and +// tag. +// If network is empty, Dial will connect to the local syslog server. +func Dial(network, raddr string, priority Priority, tag string) (*Writer, error) { + return DialWithTLSConfig(network, raddr, priority, tag, nil) +} + +// DialWithTLSCertPath establishes a secure connection to a log daemon by connecting to +// address raddr on the specified network. It uses certPath to load TLS certificates and configure +// the secure connection. +func DialWithTLSCertPath(network, raddr string, priority Priority, tag, certPath string) (*Writer, error) { + serverCert, err := ioutil.ReadFile(certPath) + if err != nil { + return nil, err + } + + return DialWithTLSCert(network, raddr, priority, tag, serverCert) +} + +// DialWIthTLSCert establishes a secure connection to a log daemon by connecting to +// address raddr on the specified network. It uses serverCert to load a TLS certificate +// and configure the secure connection. +func DialWithTLSCert(network, raddr string, priority Priority, tag string, serverCert []byte) (*Writer, error) { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(serverCert) + config := tls.Config{ + RootCAs: pool, + } + + return DialWithTLSConfig(network, raddr, priority, tag, &config) +} + +// DialWithTLSConfig establishes a secure connection to a log daemon by connecting to +// address raddr on the specified network. It uses tlsConfig to configure the secure connection. +func DialWithTLSConfig(network, raddr string, priority Priority, tag string, tlsConfig *tls.Config) (*Writer, error) { + if err := validatePriority(priority); err != nil { + return nil, err + } + + if tag == "" { + tag = os.Args[0] + } + hostname, _ := os.Hostname() + + w := &Writer{ + priority: priority, + tag: tag, + hostname: hostname, + network: network, + raddr: raddr, + tlsConfig: tlsConfig, + } + + _, err := w.connect() + if err != nil { + return nil, err + } + return w, err +} + +// NewLogger creates a log.Logger whose output is written to +// the system log service with the specified priority. The logFlag +// argument is the flag set passed through to log.New to create +// the Logger. +func NewLogger(p Priority, logFlag int) (*log.Logger, error) { + s, err := New(p, "") + if err != nil { + return nil, err + } + return log.New(s, "", logFlag), nil +} diff --git a/vendor/github.com/RackSec/srslog/srslog_unix.go b/vendor/github.com/RackSec/srslog/srslog_unix.go new file mode 100644 index 000000000..a04d9396f --- /dev/null +++ b/vendor/github.com/RackSec/srslog/srslog_unix.go @@ -0,0 +1,54 @@ +package srslog + +import ( + "errors" + "io" + "net" +) + +// unixSyslog opens a connection to the syslog daemon running on the +// local machine using a Unix domain socket. This function exists because of +// Solaris support as implemented by gccgo. On Solaris you can not +// simply open a TCP connection to the syslog daemon. The gccgo +// sources have a syslog_solaris.go file that implements unixSyslog to +// return a type that satisfies the serverConn interface and simply calls the C +// library syslog function. +func unixSyslog() (conn serverConn, err error) { + logTypes := []string{"unixgram", "unix"} + logPaths := []string{"/dev/log", "/var/run/syslog", "/var/run/log"} + for _, network := range logTypes { + for _, path := range logPaths { + conn, err := net.Dial(network, path) + if err != nil { + continue + } else { + return &localConn{conn: conn}, nil + } + } + } + return nil, errors.New("Unix syslog delivery error") +} + +// localConn adheres to the serverConn interface, allowing us to send syslog +// messages to the local syslog daemon over a Unix domain socket. +type localConn struct { + conn io.WriteCloser +} + +// writeString formats syslog messages using time.Stamp instead of time.RFC3339, +// and omits the hostname (because it is expected to be used locally). +func (n *localConn) writeString(framer Framer, formatter Formatter, p Priority, hostname, tag, msg string) error { + if framer == nil { + framer = DefaultFramer + } + if formatter == nil { + formatter = UnixFormatter + } + _, err := n.conn.Write([]byte(framer(formatter(p, hostname, tag, msg)))) + return err +} + +// close the (local) network connection +func (n *localConn) close() error { + return n.conn.Close() +} diff --git a/vendor/github.com/RackSec/srslog/writer.go b/vendor/github.com/RackSec/srslog/writer.go new file mode 100644 index 000000000..35804e189 --- /dev/null +++ b/vendor/github.com/RackSec/srslog/writer.go @@ -0,0 +1,198 @@ +package srslog + +import ( + "crypto/tls" + "strings" + "sync" +) + +// A Writer is a connection to a syslog server. +type Writer struct { + priority Priority + tag string + hostname string + network string + raddr string + tlsConfig *tls.Config + framer Framer + formatter Formatter + + mu sync.RWMutex // guards conn + conn serverConn +} + +// getConn provides access to the internal conn, protected by a mutex. The +// conn is threadsafe, so it can be used while unlocked, but we want to avoid +// race conditions on grabbing a reference to it. +func (w *Writer) getConn() serverConn { + w.mu.RLock() + conn := w.conn + w.mu.RUnlock() + return conn +} + +// setConn updates the internal conn, protected by a mutex. +func (w *Writer) setConn(c serverConn) { + w.mu.Lock() + w.conn = c + w.mu.Unlock() +} + +// connect makes a connection to the syslog server. +func (w *Writer) connect() (serverConn, error) { + conn := w.getConn() + if conn != nil { + // ignore err from close, it makes sense to continue anyway + conn.close() + w.setConn(nil) + } + + var hostname string + var err error + dialer := w.getDialer() + conn, hostname, err = dialer.Call() + if err == nil { + w.setConn(conn) + w.hostname = hostname + + return conn, nil + } else { + return nil, err + } +} + +// SetFormatter changes the formatter function for subsequent messages. +func (w *Writer) SetFormatter(f Formatter) { + w.formatter = f +} + +// SetFramer changes the framer function for subsequent messages. +func (w *Writer) SetFramer(f Framer) { + w.framer = f +} + +// SetHostname changes the hostname for syslog messages if needed. +func (w *Writer) SetHostname(hostname string) { + w.hostname = hostname +} + +// Write sends a log message to the syslog daemon using the default priority +// passed into `srslog.New` or the `srslog.Dial*` functions. +func (w *Writer) Write(b []byte) (int, error) { + return w.writeAndRetry(w.priority, string(b)) +} + +// WriteWithPriority sends a log message with a custom priority. +func (w *Writer) WriteWithPriority(p Priority, b []byte) (int, error) { + return w.writeAndRetryWithPriority(p, string(b)) +} + +// Close closes a connection to the syslog daemon. +func (w *Writer) Close() error { + conn := w.getConn() + if conn != nil { + err := conn.close() + w.setConn(nil) + return err + } + return nil +} + +// Emerg logs a message with severity LOG_EMERG; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Emerg(m string) (err error) { + _, err = w.writeAndRetry(LOG_EMERG, m) + return err +} + +// Alert logs a message with severity LOG_ALERT; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Alert(m string) (err error) { + _, err = w.writeAndRetry(LOG_ALERT, m) + return err +} + +// Crit logs a message with severity LOG_CRIT; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Crit(m string) (err error) { + _, err = w.writeAndRetry(LOG_CRIT, m) + return err +} + +// Err logs a message with severity LOG_ERR; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Err(m string) (err error) { + _, err = w.writeAndRetry(LOG_ERR, m) + return err +} + +// Warning logs a message with severity LOG_WARNING; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Warning(m string) (err error) { + _, err = w.writeAndRetry(LOG_WARNING, m) + return err +} + +// Notice logs a message with severity LOG_NOTICE; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Notice(m string) (err error) { + _, err = w.writeAndRetry(LOG_NOTICE, m) + return err +} + +// Info logs a message with severity LOG_INFO; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Info(m string) (err error) { + _, err = w.writeAndRetry(LOG_INFO, m) + return err +} + +// Debug logs a message with severity LOG_DEBUG; this overrides the default +// priority passed to `srslog.New` and the `srslog.Dial*` functions. +func (w *Writer) Debug(m string) (err error) { + _, err = w.writeAndRetry(LOG_DEBUG, m) + return err +} + +// writeAndRetry takes a severity and the string to write. Any facility passed to +// it as part of the severity Priority will be ignored. +func (w *Writer) writeAndRetry(severity Priority, s string) (int, error) { + pr := (w.priority & facilityMask) | (severity & severityMask) + + return w.writeAndRetryWithPriority(pr, s) +} + +// writeAndRetryWithPriority differs from writeAndRetry in that it allows setting +// of both the facility and the severity. +func (w *Writer) writeAndRetryWithPriority(p Priority, s string) (int, error) { + conn := w.getConn() + if conn != nil { + if n, err := w.write(conn, p, s); err == nil { + return n, err + } + } + + var err error + if conn, err = w.connect(); err != nil { + return 0, err + } + return w.write(conn, p, s) +} + +// write generates and writes a syslog formatted string. It formats the +// message based on the current Formatter and Framer. +func (w *Writer) write(conn serverConn, p Priority, msg string) (int, error) { + // ensure it ends in a \n + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + + err := conn.writeString(w.framer, w.formatter, p, w.hostname, w.tag, msg) + if err != nil { + return 0, err + } + // Note: return the length of the input, not the number of + // bytes printed by Fprintf, because this must behave like + // an io.Writer. + return len(msg), nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index c57c8bad4..11c635fb0 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -19,6 +19,12 @@ "revision": "de5bf2ad457846296e2031421a34e2568e304e35", "revisionTime": "2017-08-10T14:37:23Z" }, + { + "checksumSHA1": "rvYLucIwmyrfZxGKj24GLylbBls=", + "path": "github.com/RackSec/srslog", + "revision": "1f7cff998e92763c026b26ab7c80a39e19acc368", + "revisionTime": "2018-05-14T15:09:17Z" + }, { "path": "github.com/alibaba/pouch/libcontainerd", "revision": "" From 9e2bbf4377b2fc08173ac922801ea0ab7af94a6e Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 10 Jun 2018 16:40:43 +0800 Subject: [PATCH 4/7] feature: add syslog driver and add some fields in logger.Info Signed-off-by: Wei Fu --- daemon/logger/info.go | 30 ++++- daemon/logger/loggerutils/tag.go | 28 ++++ daemon/logger/loggerutils/tag_test.go | 61 +++++++++ daemon/logger/syslog/const.go | 57 ++++++++ daemon/logger/syslog/fmt.go | 25 ++++ daemon/logger/syslog/syslog.go | 102 ++++++++++++++ daemon/logger/syslog/syslog_test.go | 50 +++++++ daemon/logger/syslog/validate.go | 184 ++++++++++++++++++++++++++ daemon/logger/syslog/validate_test.go | 174 ++++++++++++++++++++++++ 9 files changed, 710 insertions(+), 1 deletion(-) create mode 100644 daemon/logger/loggerutils/tag.go create mode 100644 daemon/logger/loggerutils/tag_test.go create mode 100644 daemon/logger/syslog/const.go create mode 100644 daemon/logger/syslog/fmt.go create mode 100644 daemon/logger/syslog/syslog.go create mode 100644 daemon/logger/syslog/syslog_test.go create mode 100644 daemon/logger/syslog/validate.go create mode 100644 daemon/logger/syslog/validate_test.go diff --git a/daemon/logger/info.go b/daemon/logger/info.go index bc6bbb57f..57a73ce1a 100644 --- a/daemon/logger/info.go +++ b/daemon/logger/info.go @@ -1,17 +1,45 @@ package logger -import "github.com/alibaba/pouch/pkg/utils" +import ( + "github.com/alibaba/pouch/pkg/utils" +) // Info provides container information for log driver. +// +// TODO(fuwei): add more fields. type Info struct { LogConfig map[string]string ContainerID string + ContainerName string + ContainerImageID string ContainerLabels map[string]string ContainerRootDir string + + DaemonName string } // ID returns the container truncated ID. func (i *Info) ID() string { return utils.TruncateID(i.ContainerID) } + +// FullID returns the container ID. +func (i *Info) FullID() string { + return i.ContainerID +} + +// Name returns the container name. +func (i *Info) Name() string { + return i.ContainerName +} + +// ImageID returns the container's image truncated ID. +func (i *Info) ImageID() string { + return utils.TruncateID(i.ContainerImageID) +} + +// ImageFullID returns the container's image ID. +func (i *Info) ImageFullID() string { + return i.ContainerImageID +} diff --git a/daemon/logger/loggerutils/tag.go b/daemon/logger/loggerutils/tag.go new file mode 100644 index 000000000..ce8d09403 --- /dev/null +++ b/daemon/logger/loggerutils/tag.go @@ -0,0 +1,28 @@ +package loggerutils + +import ( + "bytes" + + "github.com/alibaba/pouch/daemon/logger" + "github.com/alibaba/pouch/pkg/utils/templates" +) + +// GenerateLogTag returns a tag which can be used for different log drivers +// based on the container. +func GenerateLogTag(info logger.Info, defaultTemplate string) (string, error) { + tagTemplate := info.LogConfig["tag"] + if tagTemplate == "" { + tagTemplate = defaultTemplate + } + + tmpl, err := templates.NewParse("logtag", tagTemplate) + if err != nil { + return "", err + } + + buf := bytes.Buffer{} + if err := tmpl.Execute(&buf, &info); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/daemon/logger/loggerutils/tag_test.go b/daemon/logger/loggerutils/tag_test.go new file mode 100644 index 000000000..eafd6d622 --- /dev/null +++ b/daemon/logger/loggerutils/tag_test.go @@ -0,0 +1,61 @@ +package loggerutils + +import ( + "testing" + + "github.com/alibaba/pouch/daemon/logger" +) + +func TestGenerateLogTag(t *testing.T) { + info := logger.Info{ + LogConfig: map[string]string{}, + ContainerID: "pouchcontainer-20180610", + ContainerName: "created_by_$(whois)", + ContainerImageID: "8c811b4aec35", + DaemonName: "pouch daemon", + } + + defaultTemplate := "{{.ID}}" + for idx, tc := range []struct { + tag string + expected string + hasError bool + }{ + { + tag: "", + expected: info.ID(), + hasError: false, + }, { + tag: "{{.FullID}}", + expected: "pouchcontainer-20180610", + hasError: false, + }, { + tag: "{{.FullID}} - {{.Name}} - {{.ImageID}}", + expected: "pouchcontainer-20180610 - created_by_$(whois) - 8c811b4aec35", + hasError: false, + }, { + tag: "{{.DaemonName}}", + expected: "pouch daemon", + hasError: false, + }, { + tag: "{{.NotSupport}}", + expected: "", + hasError: true, + }, + } { + info.LogConfig["tag"] = tc.tag + got, err := GenerateLogTag(info, defaultTemplate) + + if err != nil && !tc.hasError { + t.Fatalf("[%d case] expect no error here, but got error: %v", idx, err) + } + + if err == nil && tc.hasError { + t.Fatalf("[%d case] expect error here, but got nothing", idx) + } + + if got != tc.expected { + t.Fatalf("[%d case] expect value (%v), but got (%v)", idx, tc.expected, got) + } + } +} diff --git a/daemon/logger/syslog/const.go b/daemon/logger/syslog/const.go new file mode 100644 index 000000000..43af806c8 --- /dev/null +++ b/daemon/logger/syslog/const.go @@ -0,0 +1,57 @@ +package syslog + +import ( + "crypto/tls" + + "github.com/RackSec/srslog" +) + +var ( + // rfc5424 provides millisecond resolution. + timeRfc5424fmt = "2006-01-02T15:04:05.999999Z07:00" + + secureProto = "tcp+tls" + + defaultTagTemplate = "{{.ID}}" + defaultSyslogPriority = srslog.LOG_DAEMON + + // facilityAliasMap allows user to use alias to set the syslog priority. + facilityAliasMap = map[string]srslog.Priority{ + "kern": srslog.LOG_KERN, + "user": srslog.LOG_USER, + "mail": srslog.LOG_MAIL, + "daemon": srslog.LOG_DAEMON, + "auth": srslog.LOG_AUTH, + "syslog": srslog.LOG_SYSLOG, + "lpr": srslog.LOG_LPR, + "news": srslog.LOG_NEWS, + "uucp": srslog.LOG_UUCP, + "cron": srslog.LOG_CRON, + "authpriv": srslog.LOG_AUTHPRIV, + "ftp": srslog.LOG_FTP, + "local0": srslog.LOG_LOCAL0, + "local1": srslog.LOG_LOCAL1, + "local2": srslog.LOG_LOCAL2, + "local3": srslog.LOG_LOCAL3, + "local4": srslog.LOG_LOCAL4, + "local5": srslog.LOG_LOCAL5, + "local6": srslog.LOG_LOCAL6, + "local7": srslog.LOG_LOCAL7, + } + + validTransportURLPrefix = []string{ + "udp://", + "tcp://", + "tcp+tls://", + "unix://", + "unixgram://", + } + + // tls client cipher suites + defaultCipherSuites = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + } +) diff --git a/daemon/logger/syslog/fmt.go b/daemon/logger/syslog/fmt.go new file mode 100644 index 000000000..797c6401b --- /dev/null +++ b/daemon/logger/syslog/fmt.go @@ -0,0 +1,25 @@ +package syslog + +import ( + "fmt" + "os" + "time" + + "github.com/RackSec/srslog" +) + +func rfc5424FormatterWithTagAsAppName(p srslog.Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(time.RFC3339) + pid := os.Getpid() + msg := fmt.Sprintf("<%d>%d %s %s %s %d %s - %s", + p, 1, timestamp, hostname, tag, pid, tag, content) + return msg +} + +func rfc5424MicroFormatterWithTagAsAppName(p srslog.Priority, hostname, tag, content string) string { + timestamp := time.Now().Format(timeRfc5424fmt) + pid := os.Getpid() + msg := fmt.Sprintf("<%d>%d %s %s %s %d %s - %s", + p, 1, timestamp, hostname, tag, pid, tag, content) + return msg +} diff --git a/daemon/logger/syslog/syslog.go b/daemon/logger/syslog/syslog.go new file mode 100644 index 000000000..da61c2eab --- /dev/null +++ b/daemon/logger/syslog/syslog.go @@ -0,0 +1,102 @@ +package syslog + +import ( + "crypto/tls" + + "github.com/alibaba/pouch/daemon/logger" + "github.com/alibaba/pouch/daemon/logger/loggerutils" + + "github.com/RackSec/srslog" +) + +// Syslog writes the log data into syslog. +type Syslog struct { + writer *srslog.Writer +} + +type options struct { + tag string + proto string + address string + priority srslog.Priority + formatter srslog.Formatter + framer srslog.Framer + tlsCfg *tls.Config +} + +func defaultOptions() *options { + return &options{ + priority: defaultSyslogPriority, + } +} + +// NewSyslog returns new Syslog based on the log config. +func NewSyslog(info logger.Info) (*Syslog, error) { + opts, err := parseOptions(info) + if err != nil { + return nil, err + } + + var w *srslog.Writer + if opts.proto == secureProto { + w, err = srslog.DialWithTLSConfig(opts.proto, opts.address, opts.priority, opts.tag, opts.tlsCfg) + } else { + w, err = srslog.Dial(opts.proto, opts.address, opts.priority, opts.tag) + } + + if err != nil { + return nil, err + } + + w.SetFormatter(opts.formatter) + w.SetFramer(opts.framer) + return &Syslog{writer: w}, nil +} + +// WriteLogMessage will write the LogMessage. +func (s *Syslog) WriteLogMessage(msg *logger.LogMessage) error { + line := string(msg.Line) + if msg.Source == "stderr" { + return s.writer.Err(line) + } + return s.writer.Info(line) +} + +// Close closes the Syslog. +func (s *Syslog) Close() error { + return s.writer.Close() +} + +// parseOptions parses the log config into options. +func parseOptions(info logger.Info) (*options, error) { + var err error + opts := defaultOptions() + + opts.priority, err = parseFacility(info.LogConfig["syslog-facility"]) + if err != nil { + return nil, err + } + + opts.tag, err = loggerutils.GenerateLogTag(info, defaultTagTemplate) + if err != nil { + return nil, err + } + + opts.proto, opts.address, err = parseTargetAddress(info.LogConfig["syslog-address"]) + if err != nil { + return nil, err + } + + if opts.proto == secureProto { + opts.tlsCfg, err = parseTLSConfig(info) + if err != nil { + return nil, err + } + } + + opts.formatter, opts.framer, err = parseLogFormat(info.LogConfig["syslog-format"], opts.proto) + if err != nil { + return nil, err + } + return opts, nil +} diff --git a/daemon/logger/syslog/syslog_test.go b/daemon/logger/syslog/syslog_test.go new file mode 100644 index 000000000..710ba9324 --- /dev/null +++ b/daemon/logger/syslog/syslog_test.go @@ -0,0 +1,50 @@ +package syslog + +import ( + "testing" + + "github.com/alibaba/pouch/daemon/logger" + + "github.com/RackSec/srslog" +) + +func TestParseOptions(t *testing.T) { + info := logger.Info{ + LogConfig: map[string]string{ + "syslog-address": "tcp://localhost:8080", + "syslog-facility": "daemon", + "syslog-format": "rfc3164", + "tag": "{{.FullID}}", + }, + ContainerID: "container-20180610", + } + + opts, err := parseOptions(info) + if err != nil { + t.Fatalf("failed to parse options: %v", err) + } + + // check formatter and framer + if !isSameFunc(opts.formatter, srslog.RFC3164Formatter) || !isSameFunc(opts.framer, srslog.DefaultFramer) { + t.Fatalf("expect formatter(%v) & framer(%v), but got formatter(%v) & framer(%v)", + srslog.RFC3164Formatter, srslog.DefaultFramer, opts.formatter, opts.framer, + ) + } + + // check tag + if expected := info.ContainerID; opts.tag != expected { + t.Fatalf("expect tag(%v), but got tag(%v)", expected, opts.tag) + } + + // check priority + if expected := srslog.LOG_DAEMON; opts.priority != expected { + t.Fatalf("expect priority(%v), but got priority(%v)", expected, opts.priority) + } + + // check proto and address + if proto, addr := "tcp", "localhost:8080"; opts.proto != proto || opts.address != addr { + t.Fatalf("expect proto(%v) & address(%v), but got proto(%v) & address(%v)", + proto, addr, opts.proto, opts.address, + ) + } +} diff --git a/daemon/logger/syslog/validate.go b/daemon/logger/syslog/validate.go new file mode 100644 index 000000000..14b87dc8f --- /dev/null +++ b/daemon/logger/syslog/validate.go @@ -0,0 +1,184 @@ +package syslog + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "net" + "net/url" + "os" + "strconv" + "strings" + + "github.com/alibaba/pouch/daemon/logger" + + "github.com/RackSec/srslog" + pkgerrors "github.com/pkg/errors" +) + +var ( + // ErrInvalidSyslogFacility represents the invalid facility. + ErrInvalidSyslogFacility = errors.New("invalid syslog facility") + + // ErrInvalidSyslogFormat represents the invalid format. + ErrInvalidSyslogFormat = errors.New("invalid syslog format") + + // ErrFailedToLoadX509KeyPair is to used to indicate that it's failed to load x590 key pair. + ErrFailedToLoadX509KeyPair = errors.New("fail to load x509 key pair") + + fmtErrInvalidAddressFormat = "syslog-address must be in form proto://address, but got %v" +) + +// ValidateSyslogOption validates the syslog config. +func ValidateSyslogOption(info logger.Info) error { + _, err := parseOptions(info) + return err +} + +// parseFacility parses facility into syslog priority. +func parseFacility(f string) (srslog.Priority, error) { + if f == "" { + return defaultSyslogPriority, nil + } + + if priority, ok := facilityAliasMap[f]; ok { + return priority, nil + } + + fInt, err := strconv.Atoi(f) + if err == nil && 0 <= fInt && fInt <= 23 { + return srslog.Priority(fInt << 3), nil + } + return srslog.Priority(0), ErrInvalidSyslogFacility +} + +// parseTargetAddress parses the address into proto and host:port or path. +func parseTargetAddress(address string) (string, string, error) { + if address == "" { + return "", "", nil + } + + if !isTransportURL(address) { + return "", "", fmt.Errorf(fmtErrInvalidAddressFormat, address) + } + + url, err := url.Parse(address) + if err != nil { + return "", "", err + } + + switch url.Scheme { + case "unix", "unixgram": + if _, err := os.Stat(url.Path); err != nil { + return "", "", err + } + return url.Scheme, url.Path, nil + default: + h := url.Host + if _, _, err := net.SplitHostPort(h); err != nil { + if !strings.Contains(err.Error(), "missing port in address") { + return "", "", err + } + h = h + ":514" + } + return url.Scheme, h, nil + } +} + +// parseLogFormat parse log format into syslog formatter and framer. +func parseLogFormat(logFmt, proto string) (srslog.Formatter, srslog.Framer, error) { + switch logFmt { + case "": + return srslog.UnixFormatter, srslog.DefaultFramer, nil + case "rfc3164": + return srslog.RFC3164Formatter, srslog.DefaultFramer, nil + case "rfc5424": + if proto == secureProto { + return rfc5424FormatterWithTagAsAppName, srslog.RFC5425MessageLengthFramer, nil + } + return rfc5424FormatterWithTagAsAppName, srslog.DefaultFramer, nil + case "rfc5424micro": + if proto == secureProto { + return rfc5424MicroFormatterWithTagAsAppName, srslog.RFC5425MessageLengthFramer, nil + } + return rfc5424MicroFormatterWithTagAsAppName, srslog.DefaultFramer, nil + default: + return nil, nil, ErrInvalidSyslogFormat + } +} + +// parseTLSConfig parses the config into tls.Config. +func parseTLSConfig(info logger.Info) (*tls.Config, error) { + var ( + skipVerify bool + caFile string = info.LogConfig["syslog-tls-ca-cert"] + certFile string = info.LogConfig["syslog-tls-cert"] + keyFile string = info.LogConfig["syslog-tls-key"] + err error + ) + + _, skipVerify = info.LogConfig["syslog-tls-skip-verify"] + + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: defaultCipherSuites, + InsecureSkipVerify: skipVerify, + } + + if !skipVerify && caFile != "" { + pool, err := getCertPool(caFile) + if err != nil { + return nil, err + } + tlsCfg.RootCAs = pool + } + + if tlsCfg.Certificates, err = getCert(certFile, keyFile); err != nil { + return nil, err + } + return tlsCfg, nil +} + +// isTransportURL returns true if the address has the prefix like +// tcp|udp|unix|unixgram proto. +func isTransportURL(address string) bool { + for _, pre := range validTransportURLPrefix { + if strings.HasPrefix(address, pre) { + return true + } + } + return false +} + +// getCertPool returns an X.509 certificate pool from the certificate file. +func getCertPool(caFile string) (*x509.CertPool, error) { + pool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("failed to read system certificates: %v", err) + } + + pem, err := ioutil.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate %v: %v", caFile, err) + } + + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("failed to append certificates from the PEM file: %v", caFile) + } + return pool, nil +} + +// getCert returns the certificate. +func getCert(certFile string, keyFile string) ([]tls.Certificate, error) { + if certFile == "" && keyFile == "" { + return nil, nil + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, pkgerrors.Wrap(err, ErrFailedToLoadX509KeyPair.Error()) + } + return []tls.Certificate{cert}, nil +} diff --git a/daemon/logger/syslog/validate_test.go b/daemon/logger/syslog/validate_test.go new file mode 100644 index 000000000..d742c67ab --- /dev/null +++ b/daemon/logger/syslog/validate_test.go @@ -0,0 +1,174 @@ +package syslog + +import ( + "io/ioutil" + "log" + "os" + "reflect" + "testing" + + "github.com/RackSec/srslog" +) + +func TestParseFacility(t *testing.T) { + for idx, tc := range []struct { + input string + expected srslog.Priority + err error + }{ + { + input: "", + expected: defaultSyslogPriority, + err: nil, + }, { + input: "local0", + expected: srslog.LOG_LOCAL0, + err: nil, + }, { + input: "1", + expected: srslog.Priority(8), + err: nil, + }, { + input: "invalid", + expected: srslog.Priority(0), + err: ErrInvalidSyslogFacility, + }, + } { + got, err := parseFacility(tc.input) + if err != tc.err { + t.Fatalf("[%d case] expected error(%v), but got error(%v)", idx, tc.err, err) + } + + if got != tc.expected { + t.Fatalf("[%d case] expected priority(%v), but got(%v)", idx, tc.expected, got) + } + } +} + +func TestParseTargetAddress(t *testing.T) { + // create file for unix socket + sockFile, err := ioutil.TempFile("", "testXXX.sock") + if err != nil { + log.Fatal(err) + } + defer func() { + sockFile.Close() + os.Remove(sockFile.Name()) + }() + + for idx, tc := range []struct { + input string + proto, address string + hasError bool + }{ + { + input: "http://localhost:8080", + hasError: true, + }, { + input: "", + }, { + input: "invalid url", + hasError: true, + }, { + input: "tcp://localhost:8080", + proto: "tcp", + address: "localhost:8080", + }, { + input: "udp://localhost", + proto: "udp", + address: "localhost:514", // use default port + }, { + input: "unix://" + sockFile.Name(), + proto: "unix", + address: sockFile.Name(), + }, { + input: "unixgram://" + sockFile.Name(), + proto: "unixgram", + address: sockFile.Name(), + }, { + input: "unixgram://" + sockFile.Name() + "dont_exist", + hasError: true, + }, + } { + gotProto, gotAddress, err := parseTargetAddress(tc.input) + if err != nil && !tc.hasError { + t.Fatalf("[%d case] expect no error here, but got error: %v", idx, err) + } + + if err == nil && tc.hasError { + t.Fatalf("[%d case] expect error here, but got nothing", idx) + } + + if gotProto != tc.proto { + t.Fatalf("[%d case] expected proto(%v), but got(%v)", idx, tc.proto, gotProto) + } + + if gotAddress != tc.address { + t.Fatalf("[%d case] expected address(%v), but got(%v)", idx, tc.address, gotAddress) + } + } +} + +func TestParseLogFormat(t *testing.T) { + for idx, tc := range []struct { + fmtTyp string + proto string + + formatter srslog.Formatter + framer srslog.Framer + hasError bool + }{ + { + fmtTyp: "", + formatter: srslog.UnixFormatter, + framer: srslog.DefaultFramer, + }, { + fmtTyp: "rfc3164", + proto: secureProto, + formatter: srslog.RFC3164Formatter, + framer: srslog.DefaultFramer, + }, { + fmtTyp: "rfc5424", + proto: "tcp", + formatter: rfc5424FormatterWithTagAsAppName, + framer: srslog.DefaultFramer, + }, { + fmtTyp: "rfc5424", + proto: "tcp+tls", + formatter: rfc5424FormatterWithTagAsAppName, + framer: srslog.RFC5425MessageLengthFramer, + }, { + fmtTyp: "rfc5424micro", + proto: "tcp+tls", + formatter: rfc5424MicroFormatterWithTagAsAppName, + framer: srslog.RFC5425MessageLengthFramer, + }, { + fmtTyp: "rfc5424micro", + proto: "tcp", + formatter: rfc5424MicroFormatterWithTagAsAppName, + framer: srslog.DefaultFramer, + }, { + fmtTyp: "not support yet", + hasError: true, + }, + } { + gotFmter, gotFramer, err := parseLogFormat(tc.fmtTyp, tc.proto) + if err != nil && !tc.hasError { + t.Fatalf("[%d case] expect no error here, but got error: %v", idx, err) + } + + if err == nil && tc.hasError { + t.Fatalf("[%d case] expect error here, but got nothing", idx) + } + + if !isSameFunc(tc.formatter, gotFmter) || !isSameFunc(tc.framer, gotFramer) { + t.Fatalf("[%d case] expect formatter(%v) & framer(%v), but got formatter(%v) & framer(%v)", + idx, tc.formatter, tc.framer, gotFmter, gotFramer, + ) + } + } +} + +func isSameFunc(aFunc interface{}, bFunc interface{}) bool { + return reflect.ValueOf(aFunc).Pointer() == reflect.ValueOf(bFunc).Pointer() +} From 26f3405e5173f42d410ade78abf25a9fe6e3b78f Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 10 Jun 2018 17:17:32 +0800 Subject: [PATCH 5/7] feature: allow user to use syslog log driver 1. add option in containerio 2. add syslog config validation in container mgr 3. add syslog option during init containerio Signed-off-by: Wei Fu --- daemon/containerio/options.go | 10 ++++ daemon/containerio/syslog.go | 73 ++++++++++++++++++++++++++++++ daemon/mgr/container.go | 8 ++-- daemon/mgr/container_logger.go | 7 ++- daemon/mgr/container_validation.go | 7 ++- 5 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 daemon/containerio/syslog.go diff --git a/daemon/containerio/options.go b/daemon/containerio/options.go index 34265414c..5efe2ec3f 100644 --- a/daemon/containerio/options.go +++ b/daemon/containerio/options.go @@ -86,6 +86,16 @@ func WithJSONFile() func(*Option) { } } +// WithSyslog specified the syslog backend. +func WithSyslog() func(*Option) { + return func(opt *Option) { + if opt.backends == nil { + opt.backends = make(map[string]struct{}) + } + opt.backends["syslog"] = struct{}{} + } +} + // WithHijack specified the hijack backend. func WithHijack(hi http.Hijacker, upgrade bool) func(*Option) { return func(opt *Option) { diff --git a/daemon/containerio/syslog.go b/daemon/containerio/syslog.go new file mode 100644 index 000000000..68cb42065 --- /dev/null +++ b/daemon/containerio/syslog.go @@ -0,0 +1,73 @@ +package containerio + +import ( + "io" + + "github.com/alibaba/pouch/daemon/logger" + "github.com/alibaba/pouch/daemon/logger/syslog" +) + +func init() { + Register(func() Backend { + return &syslogging{} + }) +} + +type customWriter struct { + w func(p []byte) (int, error) +} + +func (cw *customWriter) Write(p []byte) (int, error) { + return cw.w(p) +} + +type syslogging struct { + w *syslog.Syslog +} + +func (s *syslogging) Init(opt *Option) error { + w, err := syslog.NewSyslog(opt.info) + if err != nil { + return err + } + s.w = w + return nil +} + +func (s *syslogging) Name() string { + return "syslog" +} + +func (s *syslogging) In() io.Reader { + return nil +} + +func (s *syslogging) Out() io.Writer { + return &customWriter{w: s.sourceWriteFunc("stdout")} +} + +func (s *syslogging) Err() io.Writer { + return &customWriter{w: s.sourceWriteFunc("stderr")} +} + +func (s *syslogging) Close() error { + return s.w.Close() +} + +func (s *syslogging) sourceWriteFunc(source string) func([]byte) (int, error) { + return func(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + msg := &logger.LogMessage{ + Source: source, + Line: p, + } + + if err := s.w.WriteLogMessage(msg); err != nil { + return 0, err + } + return len(p), nil + } +} diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 9fb7f1c4b..d8ad22354 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -305,9 +305,6 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty LogDriver: types.LogConfigLogDriverJSONFile, } } - if err := validateLogConfig(config.HostConfig.LogConfig); err != nil { - return nil, err - } container := &Container{ State: &types.ContainerState{ @@ -323,6 +320,11 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty HostConfig: config.HostConfig, } + // validate log config + if err := mgr.validateLogConfig(container); err != nil { + return nil, err + } + // parse volume config if err := mgr.generateMountPoints(ctx, container); err != nil { return nil, errors.Wrap(err, "failed to parse volume argument") diff --git a/daemon/mgr/container_logger.go b/daemon/mgr/container_logger.go index 1a02111d8..2a0633a4a 100644 --- a/daemon/mgr/container_logger.go +++ b/daemon/mgr/container_logger.go @@ -19,8 +19,10 @@ func optionsForContainerio(c *Container) []func(*containerio.Option) { switch cfg.LogDriver { case types.LogConfigLogDriverJSONFile: optFuncs = append(optFuncs, containerio.WithJSONFile()) + case types.LogConfigLogDriverSyslog: + optFuncs = append(optFuncs, containerio.WithSyslog()) default: - logrus.Warnf("not support %v log driver yet", cfg.LogDriver) + logrus.Warnf("not support (%v) log driver yet", cfg.LogDriver) } return optFuncs } @@ -38,7 +40,10 @@ func (mgr *ContainerManager) convContainerToLoggerInfo(c *Container) logger.Info return logger.Info{ LogConfig: logCfg, ContainerID: c.ID, + ContainerName: c.Name, + ContainerImageID: c.Image, ContainerLabels: c.Config.Labels, ContainerRootDir: mgr.Store.Path(c.ID), + DaemonName: "pouchd", } } diff --git a/daemon/mgr/container_validation.go b/daemon/mgr/container_validation.go index a608080b8..d51c5c7ff 100644 --- a/daemon/mgr/container_validation.go +++ b/daemon/mgr/container_validation.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/daemon/logger/syslog" ) // verifyContainerSetting is to verify the correctness of hostconfig and config. @@ -15,7 +16,8 @@ func (mgr *ContainerManager) verifyContainerSetting(hostConfig *types.HostConfig } // validateLogConfig is used to verify the correctness of log configuration. -func validateLogConfig(logCfg *types.LogConfig) error { +func (mgr *ContainerManager) validateLogConfig(c *Container) error { + logCfg := c.HostConfig.LogConfig if logCfg == nil { return nil } @@ -23,6 +25,9 @@ func validateLogConfig(logCfg *types.LogConfig) error { switch logCfg.LogDriver { case types.LogConfigLogDriverNone, types.LogConfigLogDriverJSONFile: return nil + case types.LogConfigLogDriverSyslog: + info := mgr.convContainerToLoggerInfo(c) + return syslog.ValidateSyslogOption(info) default: return fmt.Errorf("not support (%v) log driver yet", logCfg.LogDriver) } From cf0e2e76f93c81ef5ef46f982d9e24e2953d6dd1 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Sun, 10 Jun 2018 17:37:02 +0800 Subject: [PATCH 6/7] feature: limit the logs API which only can be used for jsonfile Signed-off-by: Wei Fu --- daemon/mgr/container_logs.go | 7 +++++++ test/api_container_logs_test.go | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/daemon/mgr/container_logs.go b/daemon/mgr/container_logs.go index 717fe5b0d..4cde3afcb 100644 --- a/daemon/mgr/container_logs.go +++ b/daemon/mgr/container_logs.go @@ -28,6 +28,13 @@ func (mgr *ContainerManager) Logs(ctx context.Context, name string, logOpt *type return nil, false, pkgerrors.Wrapf(errtypes.ErrInvalidParam, "you must choose at least one stream") } + if c.HostConfig.LogConfig.LogDriver != types.LogConfigLogDriverJSONFile { + return nil, false, pkgerrors.Wrapf( + errtypes.ErrInvalidParam, + "only support for the %v log driver", types.LogConfigLogDriverJSONFile, + ) + } + cfg, err := convContainerLogsOptionsToReadConfig(logOpt) if err != nil { return nil, false, err diff --git a/test/api_container_logs_test.go b/test/api_container_logs_test.go index e6d17a2be..26d040317 100644 --- a/test/api_container_logs_test.go +++ b/test/api_container_logs_test.go @@ -47,6 +47,23 @@ func (suite *APIContainerLogsSuite) TestNoSuchContainer(c *check.C) { CheckRespStatus(c, resp, http.StatusNotFound) } +// TestOnlySupportForJSONFile tests for non-jsonfile log driver. +func (suite *APIContainerLogsSuite) TestOnlySupportForJSONFile(c *check.C) { + name := "logs_with_none_driver" + command.PouchRun("run", "--log-driver", "none", "-d", "--name", name, busyboxImage, "ls").Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, name) + + resp, err := request.Get(fmt.Sprintf("/containers/%s/logs", name), + request.WithQuery( + url.Values(map[string][]string{ + "stdout": {"1"}, + }), + ), + ) + c.Assert(err, check.IsNil) + CheckRespStatus(c, resp, http.StatusBadRequest) +} + // TestNoShowStdoutAndShowStderr tests logs API without ShowStderr and // ShowStdout should return 401. func (suite *APIContainerLogsSuite) TestNoShowStdoutAndShowStderr(c *check.C) { @@ -57,8 +74,6 @@ func (suite *APIContainerLogsSuite) TestNoShowStdoutAndShowStderr(c *check.C) { resp, err := request.Get(fmt.Sprintf("/containers/%s/logs", name)) c.Assert(err, check.IsNil) CheckRespStatus(c, resp, http.StatusBadRequest) - - DelContainerForceOk(c, name) } // TestStdout tests stdout stream. From 5d9dc50ec9cdff03510f23dd00e33e9b7fee8604 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Tue, 12 Jun 2018 19:24:43 +0800 Subject: [PATCH 7/7] refactor: change for the reviewer 1. s/optionsForContainerio/logOptionsForContainerio/g 2. make the convertKVStringsToMap into pkg/utils 3. fix typo Signed-off-by: Wei Fu --- apis/opts/log_options.go | 22 ++------------ daemon/logger/syslog/validate.go | 2 +- daemon/mgr/container.go | 4 +-- daemon/mgr/container_logger.go | 2 +- pkg/utils/utils.go | 14 +++++++++ pkg/utils/utils_test.go | 51 ++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 23 deletions(-) diff --git a/apis/opts/log_options.go b/apis/opts/log_options.go index 35ec095cf..c12c1f0ed 100644 --- a/apis/opts/log_options.go +++ b/apis/opts/log_options.go @@ -1,14 +1,14 @@ package opts import ( - "errors" "fmt" - "strings" + + "github.com/alibaba/pouch/pkg/utils" ) // ParseLogOptions parses [key=value] slice-type log options into map. func ParseLogOptions(driver string, logOpts []string) (map[string]string, error) { - opts, err := convertKVStringsToMap(logOpts) + opts, err := utils.ConvertKVStringsToMap(logOpts) if err != nil { return nil, err } @@ -18,19 +18,3 @@ func ParseLogOptions(driver string, logOpts []string) (map[string]string, error) } return opts, nil } - -// convertKVStringsToMap converts ["key=value"] into {"key":"value"} -// -// TODO(fuwei): make it common in the opts.ParseXXX(). -func convertKVStringsToMap(values []string) (map[string]string, error) { - kvs := make(map[string]string, len(values)) - - for _, value := range values { - terms := strings.SplitN(value, "=", 2) - if len(terms) != 2 { - return nil, errors.New("the format must be key=value") - } - kvs[terms[0]] = terms[1] - } - return kvs, nil -} diff --git a/daemon/logger/syslog/validate.go b/daemon/logger/syslog/validate.go index 14b87dc8f..8f52ff326 100644 --- a/daemon/logger/syslog/validate.go +++ b/daemon/logger/syslog/validate.go @@ -25,7 +25,7 @@ var ( // ErrInvalidSyslogFormat represents the invalid format. ErrInvalidSyslogFormat = errors.New("invalid syslog format") - // ErrFailedToLoadX509KeyPair is to used to indicate that it's failed to load x590 key pair. + // ErrFailedToLoadX509KeyPair is to used to indicate that it's failed to load x509 key pair. ErrFailedToLoadX509KeyPair = errors.New("fail to load x509 key pair") fmtErrInvalidAddressFormat = "syslog-address must be in form proto://address, but got %v" diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index d8ad22354..86befa7ec 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -1475,7 +1475,7 @@ func (mgr *ContainerManager) openContainerIO(c *Container) (*containerio.IO, err containerio.WithStdin(c.Config.OpenStdin), } - options = append(options, optionsForContainerio(c)...) + options = append(options, logOptionsForContainerio(c)...) io := containerio.NewIO(containerio.NewOption(options...)) mgr.IOs.Put(c.ID, io) @@ -1612,7 +1612,7 @@ func (mgr *ContainerManager) openAttachIO(c *Container, attach *AttachConfig) (* containerio.WithID(c.ID), containerio.WithLoggerInfo(logInfo), } - options = append(options, optionsForContainerio(c)...) + options = append(options, logOptionsForContainerio(c)...) if attach != nil { options = append(options, attachConfigToOptions(attach)...) diff --git a/daemon/mgr/container_logger.go b/daemon/mgr/container_logger.go index 2a0633a4a..ee2bf38ab 100644 --- a/daemon/mgr/container_logger.go +++ b/daemon/mgr/container_logger.go @@ -8,7 +8,7 @@ import ( "github.com/sirupsen/logrus" ) -func optionsForContainerio(c *Container) []func(*containerio.Option) { +func logOptionsForContainerio(c *Container) []func(*containerio.Option) { optFuncs := make([]func(*containerio.Option), 0, 1) cfg := c.HostConfig.LogConfig diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a20f0c047..ffbd3ec83 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -336,3 +336,17 @@ func SetOOMScore(pid, score int) error { f.Close() return err } + +// ConvertKVStringsToMap converts ["key=value"] into {"key":"value"} +func ConvertKVStringsToMap(values []string) (map[string]string, error) { + kvs := make(map[string]string, len(values)) + + for _, value := range values { + terms := strings.SplitN(value, "=", 2) + if len(terms) != 2 { + return nil, errors.New("the format must be key=value") + } + kvs[terms[0]] = terms[1] + } + return kvs, nil +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 43dc3acf3..4d9b6cf61 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -430,3 +430,54 @@ func TestParseTimestamp(t *testing.T) { } } } + +func TestConvertKVStringsToMap(t *testing.T) { + type tCases struct { + input []string + expected map[string]string + hasError bool + } + + for idx, tc := range []tCases{ + { + input: nil, + expected: map[string]string{}, + hasError: false, + }, { + input: []string{"withoutValue"}, + expected: nil, + hasError: true, + }, { + input: []string{"key=value"}, + expected: map[string]string{ + "key": "value", + }, + hasError: false, + }, { + input: []string{"key=key=value"}, + expected: map[string]string{ + "key": "key=value", + }, + hasError: false, + }, { + input: []string{"test=1", "flag=oops", "test=2"}, + expected: map[string]string{ + "test": "2", + "flag": "oops", + }, + hasError: false, + }, + } { + got, err := ConvertKVStringsToMap(tc.input) + if err == nil && tc.hasError { + t.Fatalf("[%d case] should have error here, but got nothing", idx) + } + if err != nil && !tc.hasError { + t.Fatalf("[%d case] should have no error here, but got error(%v)", idx, err) + } + + if !reflect.DeepEqual(got, tc.expected) { + t.Fatalf("[%d case] should have (%v), but got (%v)", idx, tc.expected, got) + } + } +}