Skip to content

Commit d7e1386

Browse files
Zhulikiwik
Zhuli
authored and
kiwik
committed
Add job cloud-provider-openstack-acceptance-test-k8s-cinder (apache#111)
1 parent d93e408 commit d7e1386

4 files changed

Lines changed: 190 additions & 0 deletions

File tree

‎playbooks/cloud-provider-openstack-acceptance-test-k8s-cinder/post.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
- hosts: all
2+
become: yes
3+
tasks:
4+
- name: Clean up resources
5+
shell:
6+
cmd: |
7+
set -e
8+
set -x
9+
cd ${GOPATH}/src/k8s.io/kubernetes
10+
cluster/kubectl.sh config use-context local
11+
cluster/kubectl.sh delete -f '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml'
12+
executable: /bin/bash
13+
environment: '{{ golang_env }}'

‎playbooks/cloud-provider-openstack-acceptance-test-k8s-cinder/run.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
- hosts: all
2+
become: yes
3+
tasks:
4+
- shell:
5+
cmd: |
6+
# use echo to delete extra spaces
7+
export OS_AUTH_TYPE=$(echo '{{ vexxhost_credentials.auth_type }}')
8+
export OS_IDENTITY_API_VERSION=$(echo '{{ vexxhost_credentials.identity_api_version }}')
9+
export OS_VOLUME_API_VERSION=$(echo '{{ vexxhost_credentials.volume_api_version }}')
10+
export OS_INTERFACE=$(echo '{{ vexxhost_credentials.interface }}')
11+
export OS_AUTH_URL=$(echo '{{ vexxhost_credentials.auth_url }}')
12+
export OS_PROJECT_ID=$(echo '{{ vexxhost_credentials.project_id }}')
13+
export OS_PROJECT_NAME=$(echo '{{ vexxhost_credentials.project_name }}')
14+
export OS_USER_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}')
15+
export OS_PROJECT_DOMAIN_ID=$(echo '{{ vexxhost_credentials.project_domain_id }}')
16+
export OS_USERNAME=$(echo '{{ vexxhost_credentials.username }}')
17+
export OS_PASSWORD=$(echo '{{ vexxhost_credentials.password }}')
18+
export OS_REGION_NAME=$(echo '{{ vexxhost_credentials.region_name }}')
19+
20+
set -x
21+
set -e
22+
set -o pipefail
23+
24+
# Create cloud-config
25+
mkdir -p /etc/kubernetes/
26+
cat << EOF >> /etc/kubernetes/cloud-config
27+
[Global]
28+
domain-name = $OS_USER_DOMAIN_NAME
29+
tenant-id = $OS_PROJECT_ID
30+
auth-url = $OS_AUTH_URL
31+
password = $OS_PASSWORD
32+
username = $OS_USERNAME
33+
region = $OS_REGION_NAME
34+
35+
[BlockStorage]
36+
bs-version = v2
37+
ignore-volume-az = yes
38+
EOF
39+
40+
# Go where we cloned kubernetes repository
41+
cd $GOPATH/src/k8s.io/kubernetes/
42+
export API_HOST_IP=$(ifconfig | awk '/^docker0/ {getline; print $2}' | awk -F ':' '{print $2}')
43+
export KUBELET_HOST="0.0.0.0"
44+
export ALLOW_SECURITY_CONTEXT=true
45+
export ENABLE_CRI=false
46+
export ENABLE_HOSTPATH_PROVISIONER=true
47+
export ENABLE_SINGLE_CA_SIGNER=true
48+
export KUBE_ENABLE_CLUSTER_DNS=false
49+
export LOG_LEVEL=4
50+
# We want to use the openstack cloud provider
51+
export CLOUD_PROVIDER=openstack
52+
# We want to run a separate cloud-controller-manager for openstack
53+
export EXTERNAL_CLOUD_PROVIDER=true
54+
# DO NOT change the location of the cloud-config file. It is important for the old cinder provider to work
55+
export CLOUD_CONFIG=/etc/kubernetes/cloud-config
56+
# Specify the OCCM binary
57+
export EXTERNAL_CLOUD_PROVIDER_BINARY='{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/openstack-cloud-controller-manager'
58+
59+
# location of where the kubernetes processes log their output
60+
mkdir -p '{{ ansible_user_dir }}/workspace/logs/kubernetes'
61+
export LOG_DIR='{{ ansible_user_dir }}/workspace/logs/kubernetes'
62+
# We need this for one of the conformance tests
63+
export ALLOW_PRIVILEGED=true
64+
# Just kick off all the processes and drop down to the command line
65+
export ENABLE_DAEMON=true
66+
export HOSTNAME_OVERRIDE=$(curl http://169.254.169.254/openstack/latest/meta_data.json | python -c "import sys, json; print json.load(sys.stdin)['name']")
67+
export MAX_TIME_FOR_URL_API_SERVER=5
68+
69+
# -E preserves the current env vars, but we need to special case PATH
70+
sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh -O
71+
nohup '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/cinder-provisioner' --cloud-config "$CLOUD_CONFIG" --kubeconfig /var/run/kubernetes/admin.kubeconfig --id cinder > "$LOG_DIR/cinder-provisioner.log" 2>&1 &
72+
73+
# set up the config we need for kubectl to work
74+
cluster/kubectl.sh config set-cluster local --server=https://localhost:6443 --certificate-authority=/var/run/kubernetes/server-ca.crt
75+
cluster/kubectl.sh config set-credentials myself --client-key=/var/run/kubernetes/client-admin.key --client-certificate=/var/run/kubernetes/client-admin.crt
76+
cluster/kubectl.sh config set-context local --cluster=local --user=myself
77+
cluster/kubectl.sh config use-context local
78+
79+
# Hack for RBAC for all for the new cloud-controller process, we need to do better than this
80+
cluster/kubectl.sh create clusterrolebinding --user system:serviceaccount:kube-system:default kube-system-cluster-admin-1 --clusterrole cluster-admin
81+
cluster/kubectl.sh create clusterrolebinding --user system:serviceaccount:kube-system:pvl-controller kube-system-cluster-admin-2 --clusterrole cluster-admin
82+
cluster/kubectl.sh create clusterrolebinding --user system:serviceaccount:kube-system:cloud-node-controller kube-system-cluster-admin-3 --clusterrole cluster-admin
83+
cluster/kubectl.sh create clusterrolebinding --user system:serviceaccount:kube-system:cloud-controller-manager kube-system-cluster-admin-4 --clusterrole cluster-admin
84+
cluster/kubectl.sh create clusterrolebinding --user system:serviceaccount:kube-system:shared-informers kube-system-cluster-admin-5 --clusterrole cluster-admin
85+
cluster/kubectl.sh create clusterrolebinding --user system:kube-controller-manager kube-system-cluster-admin-6 --clusterrole cluster-admin
86+
87+
# Sleep to wait for creating serviceaccount default/default complete
88+
sleep 5
89+
# Remove node taint
90+
cluster/kubectl.sh taint nodes "$HOSTNAME_OVERRIDE" node.cloudprovider.kubernetes.io/uninitialized-
91+
# Cinder test
92+
cluster/kubectl.sh apply -f '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml'
93+
94+
# If test passed
95+
if timeout 100 bash -c '
96+
while :
97+
do
98+
[[ $(cluster/kubectl.sh describe pods web | awk "/^Status:/ {print \$2}") == Running ]] && break
99+
sleep 1
100+
done
101+
'
102+
then
103+
echo 'Run test successful'
104+
else
105+
echo 'Run test failed'
106+
cluster/kubectl.sh describe pods web
107+
exit 1
108+
fi
109+
executable: /bin/bash
110+
chdir: '{{ zuul.project.src_dir }}'
111+
environment: '{{ golang_env }}'

‎playbooks/cloud-provider-openstack-acceptance-test/pre.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
- hosts: all
2+
become: yes
3+
tasks:
4+
- shell:
5+
cmd: |
6+
set -x
7+
set -e
8+
9+
# Install docker
10+
apt update -y
11+
apt install -y docker.io
12+
13+
# Custom Docker daemon options via drop-in file then restart
14+
mkdir /etc/systemd/system/docker.service.d
15+
cat << EOF > /etc/systemd/system/docker.service.d/local-up-cluster.conf
16+
[Service]
17+
Environment="DOCKER_OPTS=--iptables=false"
18+
EOF
19+
systemctl daemon-reload
20+
systemctl restart docker
21+
22+
# Install etcd
23+
wget https://github.com/coreos/etcd/releases/download/v3.3.0/etcd-v3.3.0-linux-amd64.tar.gz
24+
tar -zxf etcd-v3.3.0-linux-amd64.tar.gz
25+
cp etcd-v3.3.0-linux-amd64/etcd{,ctl} /usr/local/bin/
26+
27+
# Install dependencies
28+
go get -u github.com/Masterminds/glide
29+
30+
# Build binaries
31+
make build
32+
# Build k8s cmd
33+
git clone https://github.com/kubernetes/kubernetes ${GOPATH}/src/k8s.io/kubernetes
34+
make -C ${GOPATH}/src/k8s.io/kubernetes WHAT="cmd/kubectl cmd/hyperkube"
35+
36+
# Stopping firewall and allow all traffic
37+
iptables -F
38+
iptables -X
39+
iptables -t nat -F
40+
iptables -t nat -X
41+
iptables -t mangle -F
42+
iptables -t mangle -X
43+
iptables -P INPUT ACCEPT
44+
iptables -P FORWARD ACCEPT
45+
iptables -P OUTPUT ACCEPT
46+
executable: /bin/bash
47+
chdir: '{{ zuul.project.src_dir }}'
48+
environment: '{{ golang_env }}'

‎zuul.d/jobs.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,24 @@
252252
- vexxhost_credentials
253253
nodeset: ubuntu-xenial-vexxhost
254254

255+
- job:
256+
name: cloud-provider-openstack-acceptance-test
257+
parent: golang-test
258+
description: |
259+
Run acceptance test of cloud-provider-openstack
260+
pre-run: playbooks/cloud-provider-openstack-acceptance-test/pre.yaml
261+
nodeset: ubuntu-xenial-vexxhost
262+
263+
- job:
264+
name: cloud-provider-openstack-acceptance-test-k8s-cinder
265+
parent: cloud-provider-openstack-acceptance-test
266+
description: |
267+
Run cinder in-tree acceptance tests of cloud-provider-openstack
268+
run: playbooks/cloud-provider-openstack-acceptance-test-k8s-cinder/run.yaml
269+
post-run: playbooks/cloud-provider-openstack-acceptance-test-k8s-cinder/post.yaml
270+
secrets:
271+
- vexxhost_credentials
272+
255273
- job:
256274
name: cloud-provider-openstack-acceptance-test-lb-octavia
257275
parent: golang-test

0 commit comments

Comments
 (0)