Drop the use of custom KNI release images.

04_setup_ironic.sh

@@ -34,43 +34,22 @@ if  sudo podman pod exists ironic-pod ; then
     sudo podman pod rm ironic-pod -f
 fi
 
-# set password for mariadb
-mariadb_password=$(echo $(date;hostname)|sha256sum |cut -c-20)
-
 # Create pod
 sudo podman pod create -n ironic-pod 
 
-# Start dnsmasq, http, mariadb, and ironic containers using same image
-sudo podman run -d --net host --privileged --name dnsmasq  --pod ironic-pod \
-     -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE}
-
+# We start only the httpd and *downloader containers so that we can provide
+# cached images to the bootstrap VM
 sudo podman run -d --net host --privileged --name httpd --pod ironic-pod \
      -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/runhttpd ${IRONIC_IMAGE}
 
-sudo podman run -d --net host --privileged --name mariadb --pod ironic-pod \
-     -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/runmariadb \
-     --env MARIADB_PASSWORD=$mariadb_password ${IRONIC_IMAGE}
-
-sudo podman run -d --net host --privileged --name ironic-conductor --pod ironic-pod \
-     --env MARIADB_PASSWORD=$mariadb_password \
-     --env OS_CONDUCTOR__HEARTBEAT_TIMEOUT=120 \
-     --entrypoint /bin/runironic-conductor \
-     -v $IRONIC_DATA_DIR:/shared ${IRONIC_IMAGE}
-
-sudo podman run -d --net host --privileged --name ironic-api --pod ironic-pod \
-     --env MARIADB_PASSWORD=$mariadb_password \
-     --entrypoint /bin/runironic-api \
-     -v $IRONIC_DATA_DIR:/shared ${IRONIC_IMAGE}
-
 sudo podman run -d --net host --privileged --name ipa-downloader --pod ironic-pod \
      -v $IRONIC_DATA_DIR:/shared ${IPA_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh
 
 sudo podman run -d --net host --privileged --name coreos-downloader --pod ironic-pod \
      -v $IRONIC_DATA_DIR:/shared ${COREOS_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh $RHCOS_IMAGE_URL
 
-# Start Ironic Inspector 
-sudo podman run -d --net host --privileged --name ironic-inspector \
-     --pod ironic-pod -v $IRONIC_DATA_DIR:/shared "${IRONIC_INSPECTOR_IMAGE}"
-
 # Wait for images to be downloaded/ready
-while ! curl --fail http://localhost:80/images/rhcos-ootpa-latest.qcow2.md5sum ; do sleep 1 ; done
+while ! curl --fail http://localhost/images/rhcos-ootpa-latest.qcow2.md5sum ; do sleep 1 ; done
+while ! curl --fail --head http://localhost/images/ironic-python-agent.initramfs ; do sleep 1; done
+while ! curl --fail --head http://localhost/images/ironic-python-agent.tar.headers ; do sleep 1; done
+while ! curl --fail --head http://localhost/images/ironic-python-agent.kernel ; do sleep 1; done

06_create_cluster.sh

@@ -8,7 +8,7 @@ source common.sh
 source ocp_install_env.sh
 
 # Do some PULL_SECRET sanity checking
-if [[ "${OPENSHIFT_RELEASE_IMAGE}" == *"registry.svc.ci.openshift.org"* ]]; then
+if [[ "${OPENSHIFT_RELEASE_IMAGE_OVERRIDE}" == *"registry.svc.ci.openshift.org"* ]]; then
     if [[ "${PULL_SECRET}" != *"registry.svc.ci.openshift.org"* ]]; then
         echo "Please get a valid pull secret for registry.svc.ci.openshift.org."
         exit 1
@@ -37,7 +37,7 @@ if [ ! -d ocp ]; then
 
     if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
       # Extract openshift-install from the release image
-      extract_installer "${OPENSHIFT_RELEASE_IMAGE}" ocp/
+      extract_installer "${OPENSHIFT_RELEASE_IMAGE_OVERRIDE}" ocp/
     else
       # Clone and build the installer from source
       clone_installer
@@ -58,25 +58,14 @@ if [ ! -d ocp ]; then
     generate_ocp_install_config ocp
 fi
 
-# Make sure Ironic is up
-export OS_TOKEN=fake-token
-export OS_URL=http://localhost:6385
-
-wait_for_json ironic \
-    "${OS_URL}/v1/nodes" \
-    20 \
-    -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: wait-for-json" -H "X-Auth-Token: $OS_TOKEN"
-
-if [ $(sudo podman ps | grep -w -e "ironic-api$" -e "ironic-conductor$" -e "ironic-inspector$" -e "dnsmasq" -e "httpd" | wc -l) != 5 ]; then
-    echo "Can't find required containers"
-    exit 1
-fi
-
-# Run the fix_certs.sh script periodically as a workaround for
-# https://github.com/openshift-metalkube/dev-scripts/issues/260
-sudo systemd-run --on-active=30s --on-unit-active=1m --unit=fix_certs.service $(dirname $0)/fix_certs.sh
-
 # Call openshift-installer to deploy the bootstrap node and masters
 create_cluster ocp
 
+# Kill the dnsmasq container on the host since it is performing DHCP and doesn't
+# allow our pod in openshift to take over.  We don't want to take down all of ironic
+# as it makes cleanup "make clean" not work properly.
+for name in dnsmasq ironic-inspector ; do
+    sudo podman ps | grep -w "$name$" && sudo podman stop $name
+done
+
 echo "Cluster up, you can interact with it via oc --config ${KUBECONFIG} <command>"

11_register_hosts.sh

@@ -75,6 +75,10 @@ function make_bm_workers() {
 list_masters | make_bm_masters | tee $SCRIPTDIR/ocp/master_crs.yaml
 
 list_workers | make_bm_workers | tee $SCRIPTDIR/ocp/worker_crs.yaml
+# TODO - remove this once we set worker replicas to ${NUM_WORKERS} in
+# install-config, which will be after the machine-api-operator can deploy the
+# baremetal-operator
+oc scale machineset -n openshift-machine-api ${CLUSTER_NAME}-worker-0 --replicas=${NUM_WORKERS}
 
 oc --config ocp/auth/kubeconfig apply -f $SCRIPTDIR/ocp/master_crs.yaml --namespace=openshift-machine-api
 

Makefile

@@ -1,9 +1,9 @@
-.PHONY: default all requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell csr_hack
-default: requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts csr_hack bell
+.PHONY: default all requirements configure repo_sync ironic ocp_run register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell csr_hack
+default: requirements configure repo_sync ironic ocp_run register_hosts csr_hack bell
 
 all: default
 
-redeploy: ocp_cleanup ironic_cleanup ironic ocp_run deploy_bmo register_hosts csr_hack bell
+redeploy: ocp_cleanup ironic_cleanup ironic ocp_run register_hosts csr_hack bell
 
 requirements:
 	./01_install_requirements.sh
@@ -20,9 +20,6 @@ ironic:
 ocp_run:
 	./06_create_cluster.sh
 
-deploy_bmo:
-	./08_deploy_bmo.sh
-
 register_hosts:
 	./11_register_hosts.sh
 

README.md

@@ -112,12 +112,6 @@ Then you can interact with the k8s API on the bootstrap VM e.g
 You can also see the status of the bootkube.sh script which is running via
 `journalctl -b -f -u bootkube.service`.
 
-- `./08_deploy_bmo.sh`
-
-After running `./08_deploy_bmo.sh` the cluster that becomes active in the previous step
-is updated by deploying the baremetal-operator into the pre-existing "openshift-machine-api"
-project/namespace.
-
 ## Interacting with the deployed cluster
 
 When the master nodes are up and the cluster is active, you can interact with the API:

common.sh

@@ -47,7 +47,7 @@ export NUM_MASTERS=${NUM_MASTERS:-"3"}
 export NUM_WORKERS=${NUM_WORKERS:-"1"}
 export VM_EXTRADISKS=${VM_EXTRADISKS:-"false"}
 
-export RHCOS_INSTALLER_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.2/420.8.20190708.2/rhcos-420.8.20190708.2-openstack.qcow2"
+export RHCOS_INSTALLER_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.2/42.80.20190725.1/rhcos-42.80.20190725.1-openstack.qcow2"
 export RHCOS_IMAGE_URL=${RHCOS_IMAGE_URL:-${RHCOS_INSTALLER_IMAGE_URL}}
 export RHCOS_IMAGE_FILENAME_LATEST="rhcos-ootpa-latest.qcow2"
 

ironic_hosts.json.example

@@ -8,8 +8,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.1",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "09:e1:e4:56:44:e5",
@@ -28,8 +28,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.2",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "cb:77:ab:53:9c:30",
@@ -48,8 +48,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.3",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "8e:af:c4:d0:a3:b4",
@@ -68,8 +68,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.4",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "8e:af:c4:d0:a3:b5",
@@ -88,8 +88,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.5",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "8e:af:c4:d0:a3:b6",
@@ -108,8 +108,8 @@
         "ipmi_username": "root",
         "ipmi_password": "passw0rd",
         "ipmi_address": "1.1.1.6",
-        "deploy_kernel": "http://172.22.0.1/images/ironic-python-agent.kernel",
-        "deploy_ramdisk": "http://172.22.0.1/images/ironic-python-agent.initramfs"
+        "deploy_kernel": "http://172.22.0.2/images/ironic-python-agent.kernel",
+        "deploy_ramdisk": "http://172.22.0.2/images/ironic-python-agent.initramfs"
       },
       "ports": [{
         "address": "8e:af:c4:d0:a3:b7",

mariadb-password.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  password: cGFzc3dvcmQ=
+kind: Secret
+metadata:
+  name: mariadb-password
+  namespace: openshift-machine-api
+type: Opaque

metal3-config.yaml

@@ -2,6 +2,7 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: metal3-config
+  namespace: openshift-machine-api
 data:
   http_port: "6180"
   provisioning_interface: "ens3"

ocp_install_env.sh

@@ -12,10 +12,7 @@ export DNS_VIP=${DNS_VIP:-"192.168.111.2"}
 #
 # See https://origin-release.svc.ci.openshift.org/ for release details
 #
-# The release we default to here is pinned and known to work with the
-# baremetal platform in openshift-installer
-#
-export OPENSHIFT_RELEASE_IMAGE="${OPENSHIFT_RELEASE_IMAGE:-registry.svc.ci.openshift.org/kni/release:4.2.0-0.ci-2019-07-31-123929-kni.0}"
+export OPENSHIFT_RELEASE_IMAGE_OVERRIDE="${OPENSHIFT_RELEASE_IMAGE_OVERRIDE:-registry.svc.ci.openshift.org/ocp/release:4.2}"
 
 function extract_installer() {
     local release_image
@@ -25,14 +22,20 @@ function extract_installer() {
     outdir="$2"
 
     extract_dir=$(mktemp -d "installer--XXXXXXXXXX")
+    pullsecret_file=$(mktemp "pullsecret--XXXXXXXXXX")
 
+    echo "${PULL_SECRET}" > "${pullsecret_file}"
+    # FIXME: Find the pullspec for baremetal-installer image and extract the image, until
+    # https://github.com/openshift/oc/pull/57 is merged
+    baremetal_image=$(oc adm release info --registry-config "${pullsecret_file}" $OPENSHIFT_RELEASE_IMAGE_OVERRIDE -o json | jq -r '.references.spec.tags[] | select(.name == "baremetal-installer") | .from.name')
+    oc image extract --registry-config "${pullsecret_file}" $baremetal_image --path usr/bin/openshift-install:${extract_dir}
 
-    echo "${PULL_SECRET}" > "${extract_dir}/pullsecret"
-    oc adm release extract --registry-config "${extract_dir}/pullsecret" --command=openshift-install --to "${extract_dir}" "${release_image}"
+    chmod 755 "${extract_dir}/openshift-install"
     mv "${extract_dir}/openshift-install" "${outdir}"
     export OPENSHIFT_INSTALLER="${outdir}/openshift-install"
 
     rm -rf "${extract_dir}"
+    rm -rf "${pullsecret_file}"
 }
 
 function clone_installer() {
@@ -46,7 +49,7 @@ function build_installer() {
   # Build installer
   pushd .
   cd $OPENSHIFT_INSTALL_PATH
-  RELEASE_IMAGE="$OPENSHIFT_RELEASE_IMAGE" TAGS="libvirt baremetal" hack/build.sh
+  RELEASE_IMAGE="$OPENSHIFT_RELEASE_IMAGE_OVERRIDE" TAGS="libvirt baremetal" hack/build.sh
   popd
 
   export OPENSHIFT_INSTALLER="$OPENSHIFT_INSTALL_PATH/bin/openshift-install"
@@ -60,6 +63,13 @@ function generate_ocp_install_config() {
     deploy_kernel=$(master_node_val 0 "driver_info.deploy_kernel")
     deploy_ramdisk=$(master_node_val 0 "driver_info.deploy_ramdisk")
 
+    # Always deploy with 0 workers by default.  We do not yet support
+    # automatically deploying workers at install time anyway.  We can scale up
+    # the worker MachineSet after deploying the baremetal-operator
+    #
+    # TODO - Change worker replicas to ${NUM_WORKERS} once the machine-api-operator
+    # deploys the baremetal-operator
+
     cat > "${outdir}/install-config.yaml" << EOF
 apiVersion: v1beta4
 baseDomain: ${BASE_DOMAIN}
@@ -69,7 +79,7 @@ metadata:
   name: ${CLUSTER_NAME}
 compute:
 - name: worker
-  replicas: ${NUM_WORKERS}
+  replicas: 0
 controlPlane:
   name: master
   replicas: ${NUM_MASTERS}
@@ -80,11 +90,6 @@ platform:
     dnsVIP: ${DNS_VIP}
     hosts:
 $(master_node_map_to_install_config $NUM_MASTERS)
-    image:
-      source: "http://172.22.0.1/images/$RHCOS_IMAGE_FILENAME_LATEST"
-      checksum: $(curl http://172.22.0.1/images/$RHCOS_IMAGE_FILENAME_LATEST.md5sum)
-      deployKernel: ${deploy_kernel}
-      deployRamdisk: ${deploy_ramdisk}
 pullSecret: |
   $(echo $PULL_SECRET | jq -c .)
 sshKey: |