more explicit progression stream

clonedb.py

@@ -70,12 +70,22 @@ def load_settings():
         return yaml.load(f, Loader=yaml.FullLoader)
 
 
+def _print(msg, end=False):
+    msg = msg.ljust(80)
+    print(f'\r{msg}', end='' if not end else '\n', flush=True)
+
+
 class MysqldumpHandler(PipeHandler):
     """ Handle and process the stdout / stderr output from a mysqldump process
     """
     _rx_prog = re.compile(r'Retrieving table structure for table (\w+)')
     _log_all = LOG_PIPES_OUTPUT
-    _action_name = "dumping"
+
+    def __init__(self, logger_name, level, total_prog):
+        super().__init__(logger_name, level)
+        self.total_prog = total_prog
+        self.prog = 0
+        self._last_logged = ""
 
     def process(self, line):
         """ Process the last line that was read
@@ -84,25 +94,32 @@ class MysqldumpHandler(PipeHandler):
         if SHOW_PROGRESSION:
             match = self._rx_prog.search(line)
             if match:
-                logger.debug('... %s %s', self._action_name, match.group(1))
-                print('.', end="", flush=True)
+                self.log_new_table(match.group(1), "dumping")
         if self._log_all:
             logger.debug(line)
 
+    def log_new_table(self, tname, action_name=""):
+        if tname == self._last_logged:
+            return
+        self.prog += 1
+        logger.debug('... %s %s', action_name, tname)
+        _print(f'{action_name} `{tname}` [{self.prog} / {self.total_prog}]')
+        self._last_logged = tname
+
+    def log_end(self):
+        _print(f'\r-- done --', end=True)
+
     def close(self):
         """ Close the write end of the pipe.
         """
-        print('', flush=True)
         super().close()
 
-
 class MysqlHandler(MysqldumpHandler):
     """ Handle and process the stdout / stderr output from a mysql process
     """
     _rx_prog = re.compile(r'^((?:CREATE TABLE )|(?:INSERT INTO ))`(\w+)`')
     _log_all = LOG_PIPES_OUTPUT
     _action_name = "restoring"
-    _last_logged = ""
 
     def process(self, line):
         """ Process the last line that was read
@@ -111,13 +128,11 @@ class MysqlHandler(MysqldumpHandler):
         if SHOW_PROGRESSION:
             match = self._rx_prog.search(line)
             if match:
-                tname = match.group(2)
-                if tname != self._last_logged:
-                    logger.debug('... %s %s %s', self._action_name,
-                                 'structure of' if 'CREATE' in match.group(1) else 'data of',
-                                 tname)
-                    print('.', end="", flush=True)
-                    self._last_logged = tname
+                action_name = "restoring {}".format('structure of'
+                                                    if 'CREATE' in match.group(1)
+                                                    else 'data of')
+                self.log_new_table(match.group(2), action_name)
+
         if self._log_all:
             logger.debug(line)
 
@@ -329,15 +344,15 @@ class CloningOperation:
         return cmd
 
     @staticmethod
-    def _run_piped_processes(dump_cmd, restore_cmd):
+    def _run_piped_processes(dump_cmd, restore_cmd, tbl_count):
         """ Run the dump and the restore commands by piping them
         The output of the mysqldump process is piped into the input of the mysql one
         """
         logger.debug(">>> Dump command: %s", " ".join(map(str, dump_cmd)))
         logger.debug(">>> Piped into: %s", " ".join(map(str, restore_cmd)))
 
-        mysqldump_handler = MysqldumpHandler(logger.name)
-        mysql_handler = MysqlHandler(logger.name)
+        mysqldump_handler = MysqldumpHandler(logger.name, logging.INFO, tbl_count)
+        mysql_handler = MysqlHandler(logger.name, logging.INFO, tbl_count)
         try:
             # noinspection PyTypeChecker
             with Popen(restore_cmd, stdin=PIPE, stdout=mysql_handler, stderr=mysql_handler) as mysql:
@@ -350,6 +365,8 @@ class CloningOperation:
             if mysql.returncode:
                 raise RuntimeError('mysql returned a non zero code')
 
+            mysql_handler.log_end()
+
         except (OSError, RuntimeError, CalledProcessError) as e:
             logger.error("Execution failed: %s", e)
             raise RuntimeError(f"An error happened at runtime: {e}")
@@ -409,11 +426,11 @@ class CloningOperation:
             try:
                 if dump_structure_for:
                     logger.info(f"Cloning structure for {len(dump_structure_for)} tables (on {len(tables)})...")
-                    self._run_piped_processes(dump_structure_cmd, restore_cmd)
+                    self._run_piped_processes(dump_structure_cmd, restore_cmd, len(dump_structure_for))
 
                 if dump_data_for:
                     logger.info(f"Cloning data for {len(dump_data_for)} tables (on {len(tables)})...")
-                    self._run_piped_processes(dump_data_cmd, restore_cmd)
+                    self._run_piped_processes(dump_data_cmd, restore_cmd, len(dump_data_for))
 
                 logger.info(f"Cloning views...")
                 self.from_server.set_active_db(self.dbname)