mgr/ansible: Replace Inventory Ansible playbook

A new Ansible playbook allows now to retrieve the storage devices information produced by ceph-volume.

Signed-off-by: Juan Miguel Olmo Martínez <[email protected]>

src/pybind/mgr/ansible/module.py

@@ -21,7 +21,7 @@
 # Name of the playbook used in the "get_inventory" method.
 # This playbook is expected to provide a list of storage devices in the host
 # where the playbook is executed.
-GET_STORAGE_DEVICES_CATALOG_PLAYBOOK = "host-disks.yml"
+GET_STORAGE_DEVICES_CATALOG_PLAYBOOK = "storage-inventory.yml"
 
 # Used in the create_osd method
 ADD_OSD_PLAYBOOK = "add-osd.yml"
@@ -119,19 +119,18 @@ def update_result(self):
 
         processed_result = []
 
-        raw_result = self.pb_execution.get_result(self.event_filter)
 
-        if self.process_output:
-            processed_result = self.process_output(
-                                        raw_result,
-                                        self.ar_client,
-                                        self.pb_execution.play_uuid)
-        else:
-            processed_result = raw_result
+        if self._is_complete:
+            raw_result = self.pb_execution.get_result(self.event_filter)
 
-        #Clean objects to avoid problems between interpreters
-        self.pb_execution = None
-        self.ar_client = None
+            if self.process_output:
+                processed_result = self.process_output(
+                                            raw_result,
+                                            self.ar_client,
+                                            self.pb_execution.play_uuid,
+                                            self.log)
+            else:
+                processed_result = raw_result
 
         self._result = processed_result
 
@@ -196,6 +195,7 @@ def is_complete(self):
 class Module(MgrModule, orchestrator.Orchestrator):
     """An Orchestrator that uses <Ansible Runner Service> to perform operations
     """
+
     MODULE_OPTIONS = [
         {'name': 'server_url'},
         {'name': 'username'},
@@ -278,7 +278,7 @@ def get_inventory(self, node_filter=None):
         ansible_operation = AnsibleReadOperation(client = self.ar_client,
                                                  playbook = GET_STORAGE_DEVICES_CATALOG_PLAYBOOK,
                                                  logger = self.log,
-                                                 result_pattern = "RESULTS",
+                                                 result_pattern = "list storage inventory",
                                                  params = {})
 
         # Assign the process_output function
@@ -390,7 +390,7 @@ def verify_config(self):
 # Auxiliary functions
 #==============================================================================
 
-def process_inventory_json(inventory_events, ar_client, playbook_uuid):
+def process_inventory_json(inventory_events, ar_client, playbook_uuid, logger):
     """ Adapt the output of the playbook used in 'get_inventory'
         to the Orchestrator expected output (list of InventoryNode)
 
@@ -399,10 +399,10 @@ def process_inventory_json(inventory_events, ar_client, playbook_uuid):
         Example:
         inventory_events =
         {'37-100564f1-9fed-48c2-bd62-4ae8636dfcdb': {'host': '192.168.121.254',
-                                                    'task': 'RESULTS',
+                                                    'task': 'list storage inventory',
                                                     'event': 'runner_on_ok'},
         '36-2016b900-e38f-7dcd-a2e7-00000000000e': {'host': '192.168.121.252'
-                                                    'task': 'RESULTS',
+                                                    'task': 'list storage inventory',
                                                     'event': 'runner_on_ok'}}
     :param ar_client: Ansible Runner Service client
     :param playbook_uuid: Playbook identifier
@@ -414,29 +414,32 @@ def process_inventory_json(inventory_events, ar_client, playbook_uuid):
     inventory_nodes = []
 
     # Loop over the result events and request the event data
-    for event_key, data in inventory_events.items():
+    for event_key, dummy_data in inventory_events.items():
+
         event_response = ar_client.http_get(EVENT_DATA_URL % (playbook_uuid,
                                                               event_key))
 
         # Process the data for each event
         if event_response:
             event_data = json.loads(event_response.text)["data"]["event_data"]
 
-            free_disks = event_data["res"]["disks_catalog"]
-            for item, data in free_disks.items():
-                if item not in [host.name for host in inventory_nodes]:
-
-                    devs = []
-                    for dev_key, dev_data in data.items():
-                        if dev_key not in [device.id for device in devs]:
-                            dev = orchestrator.InventoryDevice()
-                            dev.id = dev_key
-                            dev.type = 'hdd' if dev_data["rotational"] else "sdd/nvme"
-                            dev.size = dev_data["sectorsize"] * dev_data["sectors"]
-                            devs.append(dev)
-
-                    inventory_nodes.append(
-                            orchestrator.InventoryNode(item, devs))
+            host = event_data["host"]
+            devices =  json.loads(event_data["res"]["stdout"])
+
+            devs = []
+            for storage_device in devices:
+                dev = orchestrator.InventoryDevice()
+                dev.id = storage_device["path"]
+                dev.type = 'hdd' if storage_device["sys_api"]["rotational"] == "1" else 'sdd/nvme'
+                dev.size = storage_device["sys_api"]["human_readable_size"]
+                dev.rotates = storage_device["sys_api"]["rotational"] == "1"
+                dev.available = storage_device["available"]
+                dev.dev_id = "%s/%s" % (storage_device["sys_api"]["vendor"],
+                                        storage_device["sys_api"]["model"])
+
+                devs.append(dev)
+
+            inventory_nodes.append(orchestrator.InventoryNode(host, devs))
 
 
     return inventory_nodes

src/pybind/mgr/orchestrator.py

@@ -647,15 +647,40 @@ def __init__(self):
         self.type = None  # 'ssd', 'hdd', 'nvme'
         self.id = None  # unique within a node (or globally if you like).
         self.size = None  # byte integer.
-        self.extended = None  # arbitrary JSON-serializable object
+        self.rotates = False # indicates if it is a spinning disk
+        self.available = False # can be used to create a new OSD?
+        self.dev_id = None  # vendor/model
+        self.extended = {}  # arbitrary JSON-serializable object
 
         # If this drive is not empty, but is suitable for appending
         # additional journals, wals, or bluestore dbs, then report
         # how much space is available.
         self.metadata_space_free = None
 
     def to_json(self):
-        return dict(type=self.type, blank=self.blank, id=self.id, size=self.size, **self.extended)
+        return dict(type=self.type, blank=self.blank, id=self.id,
+                    size=self.size, rotates=self.rotates,
+                    available=self.available, dev_id=self.dev_id,
+                    **self.extended)
+
+    def pretty_print(self, only_header=False):
+        """Print a human friendly line with the information of the device
+
+        :param only_header: Print only the name of the device attributes
+
+        Ex:
+        >  Device Path           Type       Size    Rotates  Available Model
+        >     /dev/sdc            hdd   50.00 GB       True       True ATA/QEMU
+
+        """
+        row_format = "  {0:<15} {1:>10} {2:>10} {3:>10} {4:>10} {5:<15}\n"
+        if only_header:
+            return row_format.format("Device Path", "Type", "Size", "Rotates",
+                                     "Available", "Model")
+        else:
+            return row_format.format(self.id, self.type, self.size,
+                                     str(self.rotates), str(self.available),
+                                     self.dev_id)
 
 
 class InventoryNode(object):

src/pybind/mgr/orchestrator_cli/module.py

@@ -13,10 +13,11 @@ class OrchestratorCli(orchestrator.OrchestratorClientMixin, MgrModule):
     MODULE_OPTIONS = [
         {'name': 'orchestrator'}
     ]
+
     COMMANDS = [
         {
             'cmd': "orchestrator device ls "
-                   "name=host,type=CephString,req=false"
+                   "name=host,type=CephString,req=false "
                    "name=format,type=CephChoices,strings=json|plain,req=false ",
             "desc": "List devices on a node",
             "perm": "r"
@@ -141,23 +142,7 @@ def _select_orchestrator(self):
 
     def _list_devices(self, cmd):
         """
-        This (all lines starting with ">") is how it is supposed to work. As of
-        now, it's not yet implemented:
-        > :returns: Either JSON:
-        >     [
-        >       {
-        >         "name": "sda",
-        >         "host": "foo",
-        >         ... lots of stuff from ceph-volume ...
-        >         "stamp": when this state was refreshed
-        >       },
-        >     ]
-        >
-        > or human readable:
-        >
-        >     HOST  DEV  SIZE  DEVID(vendor\\_model\\_serial)   IN-USE  TIMESTAMP
-        >
-        > Note: needs ceph-volume on the host.
+        Provide information about storage devices present in cluster hosts
 
         Note: this does not have to be completely synchronous. Slightly out of
         date hardware inventory is fine as long as hardware ultimately appears
@@ -181,11 +166,17 @@ def _list_devices(self, cmd):
         else:
             # Return a human readable version
             result = ""
+
             for inventory_node in completion.result:
-                result += "{0}:\n".format(inventory_node.name)
+                result += "Host {0}:\n".format(inventory_node.name)
+
+                if inventory_node.devices:
+                    result += inventory_node.devices[0].pretty_print(only_header=True)
+                else:
+                    result += "No storage devices found"
+
                 for d in inventory_node.devices:
-                    result += "  {0} ({1}, {2}b)\n".format(
-                        d.id, d.type, d.size)
+                    result += d.pretty_print()
                 result += "\n"
 
             return HandleCommandResult(stdout=result)