From ceaf41d8d8a4d3e3b989cdabc5f5b41b14aedada Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 23 Apr 2021 21:43:22 +0900 Subject: [PATCH] Support Sobol's quasirandom sequence --- README.md | 1 + docs/proposals/suggestion.md | 6 ++ examples/v1beta1/sobol-example.yaml | 64 +++++++++++++++++++ .../components/controller/katib-config.yaml | 3 + .../constants/algorithms-settings.const.ts | 3 + .../app/constants/algorithms-types.const.ts | 1 + .../src/app/enumerations/algorithms.enum.ts | 1 + pkg/suggestion/v1beta1/goptuna/converter.go | 3 + pkg/suggestion/v1beta1/goptuna/service.go | 3 +- 9 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 examples/v1beta1/sobol-example.yaml diff --git a/README.md b/README.md index fe3cf7c4852..6c3d1696e28 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ to know more about each algorithm. - [Hyperband](https://arxiv.org/pdf/1603.06560.pdf) - [Bayesian Optimization](https://arxiv.org/pdf/1012.2599.pdf) - [Covariance Matrix Adaptation Evolution Strategy (CMA-ES)](https://arxiv.org/abs/1604.00772) +- [Sobol's Quasirandom Sequence](https://dl.acm.org/doi/10.1145/641876.641879) #### Neural Architecture Search diff --git a/docs/proposals/suggestion.md b/docs/proposals/suggestion.md index c93cef6f192..6a4abecc607 100644 --- a/docs/proposals/suggestion.md +++ b/docs/proposals/suggestion.md @@ -21,6 +21,8 @@ Table of Contents * [BOHB](#bohb) * [TPE](#tpe) * [SMAC](#smac) + * [CMA-ES](#cma-es) + * [Sobol](#sobol) Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc) @@ -395,3 +397,7 @@ We can use [SMAC3](https://github.com/automl/SMAC3) to run SMAC. ### CMA-ES We can use [goptuna](https://github.com/c-bata/goptuna) to run CMA-ES. + +### Sobol + +We can use [goptuna](https://github.com/c-bata/goptuna) to run Sobol. diff --git a/examples/v1beta1/sobol-example.yaml b/examples/v1beta1/sobol-example.yaml new file mode 100644 index 00000000000..d7ca316a747 --- /dev/null +++ b/examples/v1beta1/sobol-example.yaml @@ -0,0 +1,64 @@ +apiVersion: "kubeflow.org/v1beta1" +kind: Experiment +metadata: + namespace: kubeflow + name: sobol-example +spec: + objective: + type: maximize + goal: 0.99 + objectiveMetricName: Validation-accuracy + additionalMetricNames: + - Train-accuracy + algorithm: + algorithmName: sobol + parallelTrialCount: 3 + maxTrialCount: 12 + maxFailedTrialCount: 3 + parameters: + - name: lr + parameterType: double + feasibleSpace: + min: "0.01" + max: "0.03" + - name: num-layers + parameterType: int + feasibleSpace: + min: "2" + max: "5" + - name: optimizer + parameterType: categorical + feasibleSpace: + list: + - sgd + - adam + - ftrl + trialTemplate: + primaryContainerName: training-container + trialParameters: + - name: learningRate + description: Learning rate for the training model + reference: lr + - name: numberLayers + description: Number of training model layers + reference: num-layers + - name: optimizer + description: Training model optimizer (sdg, adam or ftrl) + reference: optimizer + trialSpec: + apiVersion: batch/v1 + kind: Job + spec: + template: + spec: + containers: + - name: training-container + image: docker.io/kubeflowkatib/mxnet-mnist:v1beta1-45c5727 + command: + - "python3" + - "/opt/mxnet-mnist/mnist.py" + - "--batch-size=64" + - "--lr=${trialParameters.learningRate}" + - "--num-layers=${trialParameters.numberLayers}" + - "--optimizer=${trialParameters.optimizer}" + restartPolicy: Never diff --git a/manifests/v1beta1/components/controller/katib-config.yaml b/manifests/v1beta1/components/controller/katib-config.yaml index 26bf87baebe..833626917b8 100644 --- a/manifests/v1beta1/components/controller/katib-config.yaml +++ b/manifests/v1beta1/components/controller/katib-config.yaml @@ -41,6 +41,9 @@ data: "cmaes": { "image": "docker.io/kubeflowkatib/suggestion-goptuna:latest" }, + "sobol": { + "image": "docker.io/kubeflowkatib/suggestion-goptuna:latest" + }, "enas": { "image": "docker.io/kubeflowkatib/suggestion-enas:latest", "resources": { diff --git a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts index f755749186f..7992caf6a7d 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts @@ -97,6 +97,8 @@ export const CMAESSettings: AlgorithmSetting[] = [ }, ]; +export const SOBOLSettings: AlgorithmSetting[] = []; + export const ENASSettings: AlgorithmSetting[] = [ { name: 'controller_hidden_size', @@ -248,6 +250,7 @@ export const AlgorithmSettingsMap: { [key: string]: AlgorithmSetting[] } = { [AlgorithmsEnum.BAYESIAN_OPTIMIZATION]: BayesianOptimizationSettings, [AlgorithmsEnum.TPE]: TPESettings, [AlgorithmsEnum.CMAES]: CMAESSettings, + [AlgorithmsEnum.SOBOL]: SOBOLSettings, [AlgorithmsEnum.ENAS]: ENASSettings, [AlgorithmsEnum.DARTS]: DartsSettings, }; diff --git a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts index df20e380b9b..c689544fb90 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts @@ -10,6 +10,7 @@ export const AlgorithmNames = { [AlgorithmsEnum.BAYESIAN_OPTIMIZATION]: 'Bayesian Optimization', [AlgorithmsEnum.TPE]: 'Tree of Parzen Estimators', [AlgorithmsEnum.CMAES]: 'Covariance Matrix Adaptation: Evolution Strategy', + [AlgorithmsEnum.SOBOL]: 'Sobol Quasirandom Sequence', }; export const NasAlgorithmNames = { diff --git a/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts b/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts index c99d0eed667..c7634b360c2 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts @@ -5,6 +5,7 @@ export enum AlgorithmsEnum { BAYESIAN_OPTIMIZATION = 'bayesianoptimization', TPE = 'tpe', CMAES = 'cmaes', + SOBOL = 'sobol', ENAS = 'enas', DARTS = 'darts', } diff --git a/pkg/suggestion/v1beta1/goptuna/converter.go b/pkg/suggestion/v1beta1/goptuna/converter.go index 31ca75f8a4c..7ebab1b002b 100644 --- a/pkg/suggestion/v1beta1/goptuna/converter.go +++ b/pkg/suggestion/v1beta1/goptuna/converter.go @@ -23,6 +23,7 @@ import ( "github.com/c-bata/goptuna" "github.com/c-bata/goptuna/cmaes" + "github.com/c-bata/goptuna/sobol" "github.com/c-bata/goptuna/tpe" api_v1_beta1 "github.com/kubeflow/katib/pkg/apis/manager/v1beta1" ) @@ -89,6 +90,8 @@ func toGoptunaSampler(algorithm *api_v1_beta1.AlgorithmSpec) (goptuna.Sampler, g } } return tpe.NewSampler(opts...), nil, nil + } else if name == AlgorithmSobol { + return nil, sobol.NewSampler(), nil } else { opts := make([]goptuna.RandomSamplerOption, 0, len(algorithm.GetAlgorithmSettings())) for _, s := range algorithm.GetAlgorithmSettings() { diff --git a/pkg/suggestion/v1beta1/goptuna/service.go b/pkg/suggestion/v1beta1/goptuna/service.go index 36f8435d1ed..789e9ca6259 100644 --- a/pkg/suggestion/v1beta1/goptuna/service.go +++ b/pkg/suggestion/v1beta1/goptuna/service.go @@ -31,6 +31,7 @@ const ( AlgorithmCMAES = "cmaes" AlgorithmTPE = "tpe" AlgorithmRandom = "random" + AlgorithmSobol = "sobol" defaultStudyName = "Katib" ) @@ -178,7 +179,7 @@ func (s *SuggestionService) ValidateAlgorithmSettings( } algorithmName := req.GetExperiment().GetSpec().GetAlgorithm().GetAlgorithmName() - if algorithmName != AlgorithmRandom && algorithmName != AlgorithmCMAES && algorithmName != AlgorithmTPE { + if algorithmName != AlgorithmRandom && algorithmName != AlgorithmCMAES && algorithmName != AlgorithmTPE && algorithmName != AlgorithmSobol { return nil, status.Error(codes.InvalidArgument, "unsupported algorithm") }