Skip to content

Commit 62b6bf1

Browse files
authored
Merge pull request #101 from onepanelio/feat/password.generators.params
feat: optional database configuration
2 parents 9720077 + 95b4034 commit 62b6bf1

File tree

9 files changed

+195
-39
lines changed

9 files changed

+195
-39
lines changed

cmd/apply.go

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,30 @@ var applyCmd = &cobra.Command{
3535
return
3636
}
3737

38+
yamlFile, err := util.LoadDynamicYamlFromFile(config.Spec.Params)
39+
if err != nil {
40+
fmt.Printf("Unable to read params.yaml: %v", err.Error())
41+
return
42+
}
43+
44+
var database *opConfig.Database = nil
45+
if !yamlFile.HasKey("database") {
46+
database, err = GetDatabaseConfigurationFromCluster()
47+
if err != nil {
48+
fmt.Printf("Unable to connect to cluster to check information: %v", err.Error())
49+
return
50+
}
51+
}
52+
53+
options := &GenerateKustomizeResultOptions{
54+
Database: database,
55+
Config: config,
56+
}
57+
3858
overlayComponentFirst := filepath.Join("common", "application", "base")
3959
baseOverlayComponent := config.GetOverlayComponent(overlayComponentFirst)
4060
applicationBaseKustomizeTemplate := TemplateFromSimpleOverlayedComponents(baseOverlayComponent)
41-
applicationResult, err := GenerateKustomizeResult(*config, applicationBaseKustomizeTemplate)
61+
applicationResult, err := GenerateKustomizeResult(applicationBaseKustomizeTemplate, options)
4262
if err != nil {
4363
fmt.Printf("%s\n", HumanizeKustomizeError(err))
4464
return
@@ -146,7 +166,7 @@ var applyCmd = &cobra.Command{
146166
//Apply the rest of the yaml
147167
kustomizeTemplate := TemplateFromSimpleOverlayedComponents(config.GetOverlayComponents(overlayComponentFirst))
148168

149-
result, err := GenerateKustomizeResult(*config, kustomizeTemplate)
169+
result, err := GenerateKustomizeResult(kustomizeTemplate, options)
150170
if err != nil {
151171
fmt.Printf("%s\n", HumanizeKustomizeError(err))
152172
return
@@ -200,7 +220,7 @@ var applyCmd = &cobra.Command{
200220
log.Printf("%v", errRes)
201221
}
202222

203-
yamlFile, err := util.LoadDynamicYamlFromFile(config.Spec.Params)
223+
yamlFile, err = util.LoadDynamicYamlFromFile(config.Spec.Params)
204224
if err != nil {
205225
fmt.Println("Error parsing configuration file.")
206226
return

cmd/build.go

Lines changed: 121 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@ import (
55
"errors"
66
"fmt"
77
"github.com/onepanelio/cli/cloud/storage"
8+
"github.com/sethvargo/go-password/password"
89
"golang.org/x/crypto/bcrypt"
910
"io/ioutil"
11+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1012
"k8s.io/apimachinery/pkg/util/rand"
13+
"k8s.io/client-go/kubernetes"
14+
"k8s.io/client-go/tools/clientcmd"
1115
"log"
1216
"os"
1317
"path/filepath"
@@ -51,8 +55,17 @@ var generateCmd = &cobra.Command{
5155

5256
kustomizeTemplate := TemplateFromSimpleOverlayedComponents(config.GetOverlayComponents(""))
5357

58+
databaseConfig, err := GetDatabaseConfigurationFromCluster()
59+
if err != nil {
60+
fmt.Printf("[error] %v", err.Error())
61+
return
62+
}
63+
5464
log.Printf("Building...")
55-
result, err := GenerateKustomizeResult(*config, kustomizeTemplate)
65+
result, err := GenerateKustomizeResult(kustomizeTemplate, &GenerateKustomizeResultOptions{
66+
Config: config,
67+
Database: databaseConfig,
68+
})
5669
if err != nil {
5770
fmt.Printf("%s\n", HumanizeKustomizeError(err))
5871
return
@@ -67,10 +80,112 @@ func init() {
6780
generateCmd.Flags().BoolVarP(&Dev, "latest", "", false, "Sets conditions to allow development testing.")
6881
}
6982

83+
// GenerateKustomizeResultOptions is configuration for the GenerateKustomizeResult function
84+
type GenerateKustomizeResultOptions struct {
85+
Database *opConfig.Database
86+
Config *opConfig.Config
87+
}
88+
89+
// generateDatabaseConfiguration checks to see if database configuration is already present
90+
// if not, it'll randomly generate some.
91+
func generateDatabaseConfiguration(yaml *util.DynamicYaml, database *opConfig.Database) error {
92+
if yaml.HasKey("database") {
93+
return nil
94+
}
95+
96+
if database == nil {
97+
dbPath := filepath.Join(".onepanel", "manifests", "cache", "common", "onepanel", "base", "vars.yaml")
98+
data, err := ioutil.ReadFile(dbPath)
99+
if err != nil {
100+
log.Fatal(err)
101+
}
102+
103+
wrapper := &opConfig.DatabaseWrapper{}
104+
if err := yaml2.Unmarshal(data, wrapper); err != nil {
105+
log.Fatal(err)
106+
}
107+
108+
database = wrapper.Database
109+
110+
pass, err := password.Generate(16, 6, 0, false, false)
111+
if err != nil {
112+
return err
113+
}
114+
database.Password.Value = pass
115+
116+
username, err := password.Generate(8, 6, 0, false, false)
117+
if err != nil {
118+
return err
119+
}
120+
database.Username.Value = "onepanel" + username
121+
}
122+
123+
yaml.Put("database.host", database.Host.Value)
124+
yaml.Put("database.username", database.Username.Value)
125+
yaml.Put("database.password", database.Password.Value)
126+
yaml.Put("database.port", fmt.Sprintf(`"%s"`, database.Port.Value))
127+
yaml.Put("database.databaseName", database.DatabaseName.Value)
128+
yaml.Put("database.driverName", database.DriverName.Value)
129+
130+
return nil
131+
}
132+
133+
// GetDatabaseConfigurationFromCluster attempts to load the database configuration from a deployed cluster
134+
// If there is no configuration (not found) no error is returned
135+
func GetDatabaseConfigurationFromCluster() (database *opConfig.Database, err error) {
136+
config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
137+
clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig()
138+
if err != nil {
139+
return
140+
}
141+
// check if there is any, and load that. Otherwise don't.
142+
c, err := kubernetes.NewForConfig(config)
143+
if err != nil {
144+
return
145+
}
146+
147+
secret, err := c.CoreV1().Secrets("onepanel").Get("onepanel", v1.GetOptions{})
148+
if err != nil {
149+
if strings.Contains(err.Error(), "not found") {
150+
return nil, nil
151+
}
152+
return
153+
}
154+
155+
databaseUsername := string(secret.Data["databaseUsername"])
156+
databasePassword := string(secret.Data["databasePassword"])
157+
158+
configMap, err := c.CoreV1().ConfigMaps("onepanel").Get("onepanel", v1.GetOptions{})
159+
if err != nil {
160+
if strings.Contains(err.Error(), "not found") {
161+
return nil, nil
162+
}
163+
return
164+
}
165+
166+
driverName := configMap.Data["databaseDriverName"]
167+
databaseHost := configMap.Data["databaseHost"]
168+
databaseName := configMap.Data["databaseName"]
169+
databasePort := configMap.Data["databasePort"]
170+
171+
database = &opConfig.Database{
172+
Host: opConfig.RequiredManifestVar(databaseHost),
173+
Username: opConfig.RequiredManifestVar(databaseUsername),
174+
Password: opConfig.RequiredManifestVar(databasePassword),
175+
Port: opConfig.RequiredManifestVar(databasePort),
176+
DatabaseName: opConfig.RequiredManifestVar(databaseName),
177+
DriverName: opConfig.RequiredManifestVar(driverName),
178+
}
179+
180+
return
181+
}
182+
70183
// GenerateKustomizeResult Given the path to the manifests, and a kustomize config, creates the final kustomization file.
71184
// It does this by copying the manifests into a temporary directory, inserting the kustomize template
72185
// and running the kustomize command
73-
func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template.Kustomize) (string, error) {
186+
func GenerateKustomizeResult(kustomizeTemplate template.Kustomize, options *GenerateKustomizeResultOptions) (string, error) {
187+
config := *options.Config
188+
74189
yamlFile, err := util.LoadDynamicYamlFromFile(config.Spec.Params)
75190
if err != nil {
76191
return "", err
@@ -269,6 +384,10 @@ func GenerateKustomizeResult(config opConfig.Config, kustomizeTemplate template.
269384
yamlFile.Put("workflowEngineContainerRuntimeExecutor", valueNode.Value)
270385
}
271386

387+
if err := generateDatabaseConfiguration(yamlFile, options.Database); err != nil {
388+
return "", err
389+
}
390+
272391
flatMap := yamlFile.FlattenToKeyValue(util.LowerCamelCaseFlatMapKeyFormatter)
273392
if err := mapLinkedVars(flatMap, localManifestsCopyPath, &config, true); err != nil {
274393
return "", err

cmd/init.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ var (
3535
EnableHTTPS bool
3636
EnableCertManager bool
3737
EnableMetalLb bool
38+
Database bool
3839
GPUDevicePlugins []string
3940
Services []string
4041
)
@@ -66,7 +67,6 @@ var initCmd = &cobra.Command{
6667
Use: "init",
6768
Short: "Gets latest manifests and generates params.yaml file.",
6869
Run: func(cmd *cobra.Command, args []string) {
69-
7070
if err := validateInput(); err != nil {
7171
log.Println(err.Error())
7272
return
@@ -245,6 +245,21 @@ var initCmd = &cobra.Command{
245245
log.Printf("[error] setting comments: %v", err.Error())
246246
return
247247
}
248+
249+
// Workflow Engine defaults to pns, but can be overwritten
250+
if err := mergedParams.Delete("workflowEngine"); err != nil {
251+
log.Printf("[error] %v", err.Error())
252+
return
253+
}
254+
255+
// By default, random credentials for a Postgres database are generated
256+
if !Database {
257+
if err := mergedParams.Delete("database"); err != nil {
258+
log.Printf("[error] %v", err.Error())
259+
return
260+
}
261+
}
262+
248263
paramsString, err := mergedParams.String()
249264
if err != nil {
250265
log.Printf("[error] unable to write params to a string")
@@ -305,6 +320,7 @@ func init() {
305320
initCmd.Flags().BoolVarP(&EnableMetalLb, "enable-metallb", "", false, "Automatically create a LoadBalancer for non-cloud deployments.")
306321
initCmd.Flags().StringSliceVarP(&GPUDevicePlugins, "gpu-device-plugins", "", nil, "Install NVIDIA and/or AMD gpu device plugins. Valid values can be comma separated and are: amd, nvidia")
307322
initCmd.Flags().StringSliceVarP(&Services, "services", "", nil, "Install additional services. Valid values can be comma separated and are: modeldb")
323+
initCmd.Flags().BoolVarP(&Database, "database", "", false, "Use a pre-existing database, set up configuration in params.yaml")
308324
}
309325

310326
func validateInput() error {

config/models.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package config
2+
3+
// ManifestVar represents a variable from the manifest
4+
type ManifestVar struct {
5+
Required bool `yaml:"required"`
6+
Value string `yaml:"default"`
7+
}
8+
9+
// RequiredManifestVar returns a ManifestVar that is required with the input value
10+
func RequiredManifestVar(val string) *ManifestVar {
11+
return &ManifestVar{
12+
Required: true,
13+
Value: val,
14+
}
15+
}
16+
17+
// Database represents the configuration available for the database
18+
type Database struct {
19+
Host *ManifestVar
20+
Username *ManifestVar
21+
Password *ManifestVar
22+
Port *ManifestVar
23+
DatabaseName *ManifestVar `yaml:"databaseName"`
24+
DriverName *ManifestVar `yaml:"driverName"`
25+
}
26+
27+
// DatabaseWrapper is a utility for loading the database key from yaml
28+
type DatabaseWrapper struct {
29+
Database *Database `yaml:"database"`
30+
}

files/files.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"strings"
1111
)
1212

13-
// exists returns whether the given file or directory exists
13+
// Exists returns whether the given file or directory exists
1414
func Exists(path string) (bool, error) {
1515
_, err := os.Stat(path)
1616
if err == nil {

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
99
github.com/mitchellh/go-homedir v1.1.0
1010
github.com/pkg/errors v0.9.1
11+
github.com/sethvargo/go-password v0.2.0
1112
github.com/spf13/cobra v0.0.5
1213
github.com/spf13/viper v1.5.0
1314
github.com/stretchr/testify v1.4.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
506506
github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
507507
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
508508
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
509+
github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
510+
github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
509511
github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
510512
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
511513
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=

manifest/builder.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,6 @@ func (b *Builder) GetYamls() []*util.DynamicYaml {
182182
}
183183

184184
temp.FlattenRequiredDefault()
185-
if err := temp.HideHidden(); err != nil {
186-
log.Printf("[error] %v", err.Error())
187-
continue
188-
}
189185

190186
varsArray = append(varsArray, temp)
191187
}

util/dynamic_yaml.go

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -587,34 +587,6 @@ func (d *DynamicYaml) FlattenRequiredDefault() {
587587
}
588588
}
589589

590-
// HideHidden goes through the vars in the yaml and removes any where hide is set to true
591-
func (d *DynamicYaml) HideHidden() error {
592-
flatMap := d.Flatten(AppendDotFlatMapKeyFormatter)
593-
594-
for key := range flatMap {
595-
lastIndex := strings.LastIndex(key, ".")
596-
if lastIndex < 0 {
597-
continue
598-
}
599-
postfixDefault := ".hide"
600-
if key[lastIndex:] != postfixDefault {
601-
continue
602-
}
603-
defaultIndex := strings.LastIndex(key, postfixDefault)
604-
if defaultIndex < 0 {
605-
continue
606-
}
607-
608-
partialKey := key[0:defaultIndex]
609-
610-
if err := d.Delete(partialKey); err != nil {
611-
return err
612-
}
613-
}
614-
615-
return nil
616-
}
617-
618590
func (d *DynamicYaml) mergeSingle(y *DynamicYaml) {
619591
if len(y.node.Content) == 0 || len(y.node.Content[0].Content) == 0 {
620592
return

0 commit comments

Comments
 (0)