Force the removal of the machines

orbital-transfer-survey · May 20, 2019 · 55128b3 · 55128b3
1 parent 4b88b6f
commit 55128b3
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 6 deletions.
diff --git a/acceptancetests/assess_recovery.py b/acceptancetests/assess_recovery.py
@@ -77,15 +77,22 @@ def restore_ha(bs_manager, controller_client):
     will be in a down state"""
     log.info("Restoring HA")
     machines_to_remove = []
+    show_controller(controller_client)
+    controller_client.show_status()
+    # pause here, as all the machines can report back as being down, but after
+    # the pause, one will report back.
+    controller_client._backend.pause(300)
     status = controller_client.get_status(controller=True)
+    # the order of the machines are normally wrong after a restore, so iterating
+    # through the machines until you find the correct ones to remove works.
     for name, machine in status.iter_machines():
         machine_status = machine['juju-status']
         if machine_status["current"] == "down":
             machines_to_remove.append(name)
     if len(machines_to_remove) > 0:
+        controller_client.show_status()
         condition = controller_client.remove_machine(machines_to_remove, force=True, controller=True)
         controller_client.wait_for(condition)
-        show_controller(controller_client)
     return enable_ha(bs_manager, controller_client)
 
 def show_controller(client):

diff --git a/acceptancetests/jujupy/client.py b/acceptancetests/jujupy/client.py
@@ -857,9 +857,9 @@ def remove_machine(self, machine_ids, force=False, controller=False):
         """
         options = ()
         if force:
-            options = ('--force',)
+            options = options + ('--force',)
         if controller:
-            options = ('-m', 'controller',)
+            options = options + ('-m', 'controller',)
         self.juju('remove-machine', options + tuple(machine_ids))
         return self.make_remove_machine_condition(machine_ids)
 
@@ -1693,7 +1693,7 @@ def get_controller_member_status(info_dict):
         """Return the controller-member-status of the machine if it exists."""
         return info_dict.get('controller-member-status')
 
-    def wait_for_ha(self, timeout=1200, start=None):
+    def wait_for_ha(self, timeout=1200, start=None, quorum=3):
         """Wait for voiting to be enabled.
 
         May only be called on a controller client."""
@@ -1710,7 +1710,7 @@ def status_to_ha(status):
                     continue
                 states.setdefault(status, []).append(machine)
             if list(states.keys()) == [desired_state]:
-                if len(states.get(desired_state, [])) >= 3:
+                if len(states.get(desired_state, [])) >= quorum:
                     return None
             return states
 

diff --git a/acceptancetests/jujupy/wait_condition.py b/acceptancetests/jujupy/wait_condition.py
@@ -217,7 +217,7 @@ def iter_blocking_state(self, status):
             if machine in self.machines:
                 machines.append(machine)
         if len(machines) > 0:
-            yield machine[0], 'still-present'
+            yield machines[0], 'still-present'
 
     def do_raise(self, model_name, status):
         plural = "s"