[1LP][RFR] Continue swapping EC2 with OpenStack and fix HTTP 503 issues.

cfme/tests/cli/test_appliance_console_db_restore.py

@@ -6,7 +6,7 @@
 from wait_for import wait_for
 
 from cfme import test_requirements
-from cfme.cloud.provider.ec2 import EC2Provider
+from cfme.cloud.provider.openstack import OpenStackProvider
 from cfme.fixtures.cli import provider_app_crud
 from cfme.infrastructure.provider.virtualcenter import VMwareProvider
 from cfme.utils.appliance.console import configure_appliances_ha
@@ -48,19 +48,23 @@ def provision_vm(request, provider):
 @pytest.fixture
 def get_appliances_with_providers(temp_appliances_unconfig_funcscope_rhevm):
     """Returns two database-owning appliances, configures first appliance with providers and
-    takes a backup prior to running tests.
-
+    takes a backup on the first one prior to running tests.
     """
     appl1, appl2 = temp_appliances_unconfig_funcscope_rhevm
     # configure appliances
     appl1.configure(region=0)
-    appl1.wait_for_web_ui()
     appl2.configure(region=0)
-    appl2.wait_for_web_ui()
+
+    for app in temp_appliances_unconfig_funcscope_rhevm:
+        app.wait_for_web_ui()
+        app.wait_for_api_available()
+
     # Add infra/cloud providers and create db backup
     provider_app_crud(VMwareProvider, appl1).setup()
-    provider_app_crud(EC2Provider, appl1).setup()
+    provider_app_crud(OpenStackProvider, appl1).setup()
     appl1.db.backup()
+    appl1.wait_for_web_ui()
+    appl1.wait_for_api_available()
     return temp_appliances_unconfig_funcscope_rhevm
 
 
@@ -75,6 +79,8 @@ def get_appliance_with_ansible(temp_appliance_preconfig_funcscope):
     appl1.enable_embedded_ansible_role()
     appl1.wait_for_embedded_ansible()
     appl1.db.backup()
+    appl1.wait_for_web_ui()
+    appl1.wait_for_api_available()
     return temp_appliance_preconfig_funcscope
 
 
@@ -89,13 +95,15 @@ def get_ext_appliances_with_providers(temp_appliances_unconfig_funcscope_rhevm,
     # configure appliances
     appl1.configure(region=0)
     appl1.wait_for_web_ui()
+    appl1.wait_for_api_available()
+
     appl2.appliance_console_cli.configure_appliance_external_join(
         app_ip, app_creds_modscope['username'], app_creds_modscope['password'], 'vmdb_production',
         app_ip, app_creds_modscope['sshlogin'], app_creds_modscope['sshpass'])
     appl2.wait_for_web_ui()
     # Add infra/cloud providers and create db backup
     provider_app_crud(VMwareProvider, appl1).setup()
-    provider_app_crud(EC2Provider, appl1).setup()
+    provider_app_crud(OpenStackProvider, appl1).setup()
     appl1.db.backup()
     return temp_appliances_unconfig_funcscope_rhevm
 
@@ -164,7 +172,7 @@ def get_ha_appliances_with_providers(unconfigured_appliances, app_creds):
 
     # Add infra/cloud providers and create db backup
     provider_app_crud(VMwareProvider, appl3).setup()
-    provider_app_crud(EC2Provider, appl3).setup()
+    provider_app_crud(OpenStackProvider, appl3).setup()
     appl1.db.backup()
 
     return unconfigured_appliances
@@ -194,7 +202,7 @@ def two_appliances_one_with_providers(temp_appliances_preconfig_funcscope):
 
     # Add infra/cloud providers
     provider_app_crud(VMwareProvider, appl1).setup()
-    provider_app_crud(EC2Provider, appl1).setup()
+    provider_app_crud(OpenStackProvider, appl1).setup()
     return appl1, appl2
 
 
@@ -240,6 +248,8 @@ def test_appliance_console_dump_restore_db_local(request, get_appliances_with_pr
     restore_db(appl2)
     appl2.evmserverd.start()
     appl2.wait_for_web_ui()
+    appl2.wait_for_api_available()
+
     # Assert providers on the second appliance
     assert set(appl2.managed_provider_names) == set(appl1.managed_provider_names), (
         'Restored DB is missing some providers'
@@ -264,8 +274,9 @@ def test_appliance_console_backup_restore_db_local(request, two_appliances_one_w
         initialEstimate: 1/2h
     """
     appl1, appl2 = two_appliances_one_with_providers
-    backup_file_name = f'/tmp/backup.{fauxfactory.gen_alphanumeric()}.dump'
+    appl1_provider_names = set(appl1.managed_provider_names)
 
+    backup_file_name = f'/tmp/backup.{fauxfactory.gen_alphanumeric()}.dump'
     appl1.db.backup(backup_file_name)
 
     # Transfer v2_key and db backup from first appliance to second appliance
@@ -297,8 +308,9 @@ def test_appliance_console_backup_restore_db_local(request, two_appliances_one_w
 
     appl2.evmserverd.start()
     appl2.wait_for_web_ui()
+    appl2.wait_for_api_available()
     # Assert providers on the second appliance
-    assert set(appl2.managed_provider_names) == set(appl1.managed_provider_names), (
+    assert set(appl2.managed_provider_names) == appl1_provider_names, (
         'Restored DB is missing some providers'
     )
     # Verify that existing provider can detect new VMs on the second appliance
@@ -325,6 +337,7 @@ def test_appliance_console_restore_pg_basebackup_ansible(get_appliance_with_ansi
     manager.quit()
     appl1.evmserverd.start()
     appl1.wait_for_web_ui()
+    appl1.wait_for_api_available()
     appl1.wait_for_embedded_ansible()
     repositories = appl1.collections.ansible_repositories
     try:
@@ -374,6 +387,8 @@ def test_appliance_console_restore_pg_basebackup_replicated(
     appl2.evmserverd.start()
     appl1.wait_for_web_ui()
     appl2.wait_for_web_ui()
+    appl1.wait_for_api_available()
+    appl2.wait_for_api_available()
     # Assert providers exist after restore and replicated to second appliances
     assert providers_before_restore == set(appl1.managed_provider_names), (
         'Restored DB is missing some providers'
@@ -461,6 +476,8 @@ def test_appliance_console_restore_db_replicated(
     appl2.evmserverd.start()
     appl1.wait_for_web_ui()
     appl2.wait_for_web_ui()
+    appl1.wait_for_api_available()
+    appl2.wait_for_api_available()
 
     # reconfigure replication between appliances which switches to "disabled"
     # during restore
@@ -509,7 +526,7 @@ def test_appliance_console_restore_db_ha(request, unconfigured_appliances, app_c
 
     # Add infra/cloud providers and create db backup
     provider_app_crud(VMwareProvider, appl3).setup()
-    provider_app_crud(EC2Provider, appl3).setup()
+    provider_app_crud(OpenStackProvider, appl3).setup()
     appl1.db.backup()
 
     providers_before_restore = set(appl3.managed_provider_names)
@@ -530,6 +547,7 @@ def test_appliance_console_restore_db_ha(request, unconfigured_appliances, app_c
 
     appl3.evmserverd.start()
     appl3.wait_for_web_ui()
+    appl3.wait_for_api_available()
     # Assert providers still exist after restore
     assert providers_before_restore == set(appl3.managed_provider_names), (
         'Restored DB is missing some providers'
@@ -543,6 +561,7 @@ def test_appliance_console_restore_db_ha(request, unconfigured_appliances, app_c
 
     appl3.evmserverd.wait_for_running()
     appl3.wait_for_web_ui()
+    appl3.wait_for_api_available()
     # Assert providers still exist after ha failover
     assert providers_before_restore == set(appl3.managed_provider_names), (
         'Restored DB is missing some providers'
@@ -623,6 +642,7 @@ def test_appliance_console_restore_db_nfs(request, two_appliances_one_with_provi
 
     appl2.evmserverd.start()
     appl2.wait_for_web_ui()
+    appl2.wait_for_api_available()
     # Assert providers on the second appliance
     assert set(appl2.managed_provider_names) == appl1_provider_names, (
         'Restored DB is missing some providers'
@@ -709,6 +729,7 @@ def test_appliance_console_restore_db_samba(request, two_appliances_one_with_pro
 
     appl2.evmserverd.start()
     appl2.wait_for_web_ui()
+    appl2.wait_for_api_available()
     # Assert providers on the second appliance
     assert set(appl2.managed_provider_names) == appl1_provider_names, (
         'Restored DB is missing some providers'

cfme/utils/appliance/__init__.py

@@ -518,6 +518,7 @@ def configure(self, log_callback=None, **kwargs):
             # restarted:
             restart_evm = False
             self.wait_for_web_ui(log_callback=log_callback)
+            self.wait_for_api_available()
             if self.version < '5.11':
                 self.configure_vm_console_cert(log_callback=log_callback)
                 restart_evm = True
@@ -529,6 +530,7 @@ def configure(self, log_callback=None, **kwargs):
             if restart_evm:
                 self.evmserverd.restart(log_callback=log_callback)
                 self.wait_for_web_ui(timeout=1800, log_callback=log_callback)
+                self.wait_for_api_available()
 
     def configure_gce(self, log_callback=None):
         # Force use of IPAppliance's configure method
@@ -1510,6 +1512,40 @@ def wait_for_web_ui(self, timeout=900, running=True, log_callback=None):
             fail_condition=not running, delay=10)
         return result
 
+    def wait_for_api_available(self, num_sec=600):
+        """Waits for the web UI to be running / to not be running
+
+        Args:
+            num_sec: Number of seconds to wait until num_sec(default ``600``)
+        """
+
+        def _check_appliance_api_ready():
+            try:
+                # There are 2 hard problems in computer science: cache
+                # invalidation, naming things, and off-by-1 errors.
+                # -- Leon Bambrick
+                #
+                # Try invalidating stale cached api object if exists
+                try:
+                    del self.__dict__['rest_api']
+                except KeyError:
+                    pass
+                api = self.rest_api
+
+                # Make sure we really make a new request. Perhaps accessing the
+                # rest_api property just creates a client object but no network
+                # communicatin is done until we access some property of the
+                # client.
+                assert api.server_info['server_href']
+                self.log.info("Appliance REST API ready")
+                return api
+            except APIException as exc:
+                self.log.warning('Appliance RESTAPI not ready: %s', exc)
+                return False
+
+        api, _ = wait_for(func=_check_appliance_api_ready, num_sec=num_sec, delay=10)
+        return api
+
     @logger_wrap("Install VDDK: {}")
     def install_vddk(self, force=False, vddk_url=None, log_callback=None):
         """Install the vddk on a appliance"""
@@ -1652,6 +1688,9 @@ def update_guid(self, log_callback=None):
             assert result.success, 'Failed to generate UUID'
         log_callback('Updated UUID: {}'.format(str(result)))
         try:
+            # There are 2 hard problems in computer science: cache
+            # invalidation, naming things, and off-by-1 errors.
+            # -- Leon Bambrick
             del self.__dict__['guid']  # invalidate cached_property
         except KeyError:
             logger.exception('Exception clearing cached_property "guid"')