From 2f70ab151c0e66d20fa576a3c98d4d2787de358c Mon Sep 17 00:00:00 2001
From: Tres Seaver <tseaver@palladion.com>
Date: Thu, 11 Aug 2016 14:59:26 -0400
Subject: [PATCH] Reduce flakiness of Monitoring system tests.

- Add retries for 503 ServiceUnavailable (eventual) on creation.

- Add retries for 404 NotFound (eventual consistency) on deletion.

- Ensure that parent groups get cleaned up even if child group deletion
  fails.

Towards:  #2092, #2093.
---
 system_tests/monitoring.py | 41 ++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py
index 654a8f16e617..0673251f1769 100644
--- a/system_tests/monitoring.py
+++ b/system_tests/monitoring.py
@@ -17,10 +17,15 @@
 from gcloud import _helpers
 from gcloud.environment_vars import TESTS_PROJECT
 from gcloud.exceptions import NotFound
+from gcloud.exceptions import ServiceUnavailable
 from gcloud import monitoring
 
+from retry import RetryErrors
 from system_test_utils import unique_resource_id
 
+retry_404 = RetryErrors(NotFound)
+retry_503 = RetryErrors(ServiceUnavailable)
+
 
 def setUpModule():
     _helpers.PROJECT = TESTS_PROJECT
@@ -172,7 +177,8 @@ def test_create_and_delete_metric_descriptor(self):
         )
 
         descriptor.create()
-        descriptor.delete()
+        retry_404(descriptor.delete)()
+
         with self.assertRaises(NotFound):
             descriptor.delete()
 
@@ -187,7 +193,7 @@ def setUp(self):
 
     def tearDown(self):
         for group in self.to_delete:
-            group.delete()
+            retry_404(group.delete)()
 
     def test_create_group(self):
         client = monitoring.Client()
@@ -196,8 +202,10 @@ def test_create_group(self):
             filter_string=self.FILTER,
             is_cluster=self.IS_CLUSTER,
         )
-        group.create()
+
+        retry_503(group.create)()
         self.to_delete.append(group)
+
         self.assertTrue(group.exists())
 
     def test_list_groups(self):
@@ -209,8 +217,10 @@ def test_list_groups(self):
         )
         before_groups = client.list_groups()
         before_names = set(group.name for group in before_groups)
-        new_group.create()
+
+        retry_503(new_group.create)()
         self.to_delete.append(new_group)
+
         self.assertTrue(new_group.exists())
         after_groups = client.list_groups()
         after_names = set(group.name for group in after_groups)
@@ -224,8 +234,10 @@ def test_reload_group(self):
             filter_string=self.FILTER,
             is_cluster=self.IS_CLUSTER,
         )
-        group.create()
+
+        retry_503(group.create)()
         self.to_delete.append(group)
+
         group.filter = 'resource.type = "aws_ec2_instance"'
         group.display_name = 'locally changed name'
         group.reload()
@@ -242,7 +254,8 @@ def test_update_group(self):
             filter_string=self.FILTER,
             is_cluster=self.IS_CLUSTER,
         )
-        group.create()
+
+        retry_503(group.create)()
         self.to_delete.append(group)
 
         group.filter = NEW_FILTER
@@ -260,7 +273,8 @@ def test_list_group_members(self):
             filter_string=self.FILTER,
             is_cluster=self.IS_CLUSTER,
         )
-        group.create()
+
+        retry_503(group.create)()
         self.to_delete.append(group)
 
         for member in group.list_members():
@@ -272,22 +286,27 @@ def test_group_hierarchy(self):
             display_name='Testing: Root group',
             filter_string=self.FILTER,
         )
-        root_group.create()
+
+        retry_503(root_group.create)()
+        self.to_delete.insert(0, root_group)
 
         middle_group = client.group(
             display_name='Testing: Middle group',
             filter_string=self.FILTER,
             parent_id=root_group.id,
         )
-        middle_group.create()
+
+        retry_503(middle_group.create)()
+        self.to_delete.insert(0, middle_group)
 
         leaf_group = client.group(
             display_name='Testing: Leaf group',
             filter_string=self.FILTER,
             parent_id=middle_group.id,
         )
-        leaf_group.create()
-        self.to_delete.extend([leaf_group, middle_group, root_group])
+
+        retry_503(leaf_group.create)()
+        self.to_delete.insert(0, leaf_group)
 
         # Test for parent.
         actual_parent = middle_group.fetch_parent()