v0.3

.gitignore

@@ -1,2 +1,5 @@
 /.project
 /log/clonedb.log
+/settings.yml
+/.clonedb.lock
+__pycache__/*

clonedb.py

@@ -5,7 +5,7 @@ Script de clonage des bases de données MySql
 > Configuration: settings.yml
 
 Usage:
-  clonedb.py [-v] [-y] [<dbname>]
+  clonedb.py [-v] [-y] [<opname>...]
   clonedb.py (-h | --help)
   clonedb.py --version
 
@@ -38,18 +38,22 @@ __VERSION__ = "0.2"
 HERE = Path(__file__).parent
 
 # Start logger
+LOG_DIR = HERE / 'log'
+LOG_DIR.mkdir_p()
 logger = logging.getLogger('clonedb')
-logging_.start("clonedb", filename=HERE / 'log' / 'clonedb.log', replace=True)
+logging_.start("clonedb", filename=LOG_DIR / 'clonedb.log', replace=True)
 
 # FIX the default ascii encoding on some linux dockers...
 sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
 
 # Options
 SHOW_PROGRESSION = True
-MAX_ALLOWED_PACKET = 1073741824
+LOG_PIPES_OUTPUT = False
+LOG_MYSQL_QUERIES = False
+
+MY_CNF_FILE = Path(HERE / "my.cnf")
 
-DEBUG = False
-LOG_MYSQL_QUERIES = DEBUG
+MAX_ALLOWED_PACKET = 1073741824
 
 
 # Utilities
@@ -69,18 +73,21 @@ def load_settings():
 class MysqldumpHandler(PipeHandler):
     """ Handle and process the stdout / stderr output from a mysqldump process
     """
-    _rx_newtable = re.compile(r'Retrieving table structure for table (\w+)')
+    _rx_prog = re.compile(r'Retrieving table structure for table (\w+)')
+    _log_all = LOG_PIPES_OUTPUT
+    _action_name = "dumping"
 
     def process(self, line):
         """ Process the last line that was read
         """
         line = line.strip('\n')
         if SHOW_PROGRESSION:
-            match = self._rx_newtable.search(line)
+            match = self._rx_prog.search(line)
             if match:
-                # logger.debug('... %s', match.group(1))
+                logger.debug('... %s %s', self._action_name, match.group(1))
                 print('.', end="", flush=True)
-        logger.debug(line)
+        if self._log_all:
+            logger.debug(line)
 
     def close(self):
         """ Close the write end of the pipe.
@@ -89,6 +96,28 @@ class MysqldumpHandler(PipeHandler):
         super().close()
 
 
+class MysqlHandler(MysqldumpHandler):
+    """ Handle and process the stdout / stderr output from a mysql process
+    """
+    _rx_prog = re.compile(r'^((?:CREATE TABLE )|(?:INSERT INTO ))`(\w+)`')
+    _log_all = LOG_PIPES_OUTPUT
+    _action_name = "restoring"
+
+    def process(self, line):
+        """ Process the last line that was read
+        """
+        line = line.strip('\n')
+        if SHOW_PROGRESSION:
+            match = self._rx_prog.search(line)
+            if match:
+                logger.debug('... %s %s %s', self._action_name,
+                             'structure of' if 'CREATE' in match.group(1) else 'data of',
+                             match.group(2))
+                print('.', end="", flush=True)
+        if self._log_all:
+            logger.debug(line)
+
+
 class MySqlServer:
     """ A server hosting a Mysql instance
     """
@@ -200,7 +229,12 @@ class MySqlServer:
         return definition
 
 
-# Beahviors for the tables cloning
+# Operation status
+UNKNOWN = 0
+SUCCESS = 1
+FAILURE = 2
+
+# Behaviors for the tables cloning
 IGNORE = 0
 STRUCTURE_ONLY = 1
 STRUCTURE_AND_DATA = 2  # -> default behavior
@@ -210,8 +244,9 @@ class CloningOperation:
     """ A database cloning operation between two Mysql servers
     """
 
-    def __init__(self, dbname, from_server, to_server, is_default=True, ignore_tables=None, structure_only=None,
+    def __init__(self, name, dbname, from_server, to_server, is_default=True, ignore_tables=None, structure_only=None,
                  filter_tables=None, ignore_views=None, compress=True):
+        self.name = name
         self.dbname = dbname
         self.from_server = from_server
         self.to_server = to_server
@@ -223,6 +258,8 @@ class CloningOperation:
         self.filter_tables = [re.compile(r) for r in filter_tables] if filter_tables else []
         self.ignore_views = [re.compile(r) for r in ignore_views] if ignore_views else []
 
+        self.status = UNKNOWN
+
     def __repr__(self):
         return f"Cloning {self.dbname} from {self.from_server} to {self.to_server}"
 
@@ -238,6 +275,7 @@ class CloningOperation:
                     "--single-transaction",
                     "-u", self.from_server.username,
                     f"--password={self.from_server.password}",
+                    f"--max-allowed-packet={MAX_ALLOWED_PACKET}",
                     "--skip-add-drop-table",
                     "--skip-add-locks",
                     "--skip-comments",
@@ -261,19 +299,26 @@ class CloningOperation:
         ready-to-consume list for Popen
         @see https://dev.mysql.com/doc/refman/8.0/en/mysql-command-options.html#option_mysql_quick
         """
+        init_command = "set global max_allowed_packet=1073741824;" \
+                       "set global wait_timeout=28800;" \
+                       "set global interactive_timeout=28800;"
+
         cmd = ["mysql",
                "-h", self.to_server.host,
                "-P", str(self.to_server.port),
                "-u", self.to_server.username,
                f"--password={self.to_server.password}",
-               f"--max-allowed-packet={MAX_ALLOWED_PACKET}",
+               f"--init-command={init_command}",
                "--reconnect",
                "--quick",
+               "--unbuffered",
+               "--wait",
+               "--verbose",
                "-D", self.dbname
                ]
 
-        if LOG_MYSQL_QUERIES:
-            cmd.append("--verbose")
+        # if LOG_PIPES_OUTPUT:
+        #   cmd.append("--verbose")
 
         if self.compress:
             cmd.append("--compress")
@@ -287,12 +332,13 @@ class CloningOperation:
         logger.debug(">>> Dump command: %s", " ".join(map(str, dump_cmd)))
         logger.debug(">>> Piped into: %s", " ".join(map(str, restore_cmd)))
 
-        pipe_handler = MysqldumpHandler(logger.name)
+        mysqldump_handler = MysqldumpHandler(logger.name)
+        mysql_handler = MysqlHandler(logger.name)
         try:
             # noinspection PyTypeChecker
-            with Popen(restore_cmd, stdin=PIPE, stdout=pipe_handler, stderr=pipe_handler) as mysql:
+            with Popen(restore_cmd, stdin=PIPE, stdout=mysql_handler, stderr=mysql_handler) as mysql:
                 # noinspection PyTypeChecker
-                with Popen(dump_cmd, stdout=PIPE, stderr=pipe_handler) as mysqldump:
+                with Popen(dump_cmd, stdout=PIPE, stderr=mysqldump_handler) as mysqldump:
                     mysql.stdin.write(mysqldump.stdout.read())
 
             if mysqldump.returncode:
@@ -305,7 +351,8 @@ class CloningOperation:
             raise RuntimeError(f"An error happened at runtime: {e}")
 
         finally:
-            pipe_handler.close()
+            mysqldump_handler.close()
+            mysql_handler.close()
 
     def run(self):
         """ Run the cloning op
@@ -337,7 +384,7 @@ class CloningOperation:
 
             # Dump structure: --single-transaction --no-data --routines {dbname} tbname1 tname2 ...
             dump_structure_for = [t for t, s in tables.items() if s != IGNORE]
-            dump_structure_cmd = self._build_dump_command(["--single-transaction", "--no-data", "--routines"],
+            dump_structure_cmd = self._build_dump_command(["--no-data", "--routines"],
                                                           dump_structure_for)
 
             # Dump data: --no-create-info --skip-triggers {dbname} tbname1 tname2 ...
@@ -374,11 +421,13 @@ class CloningOperation:
                     try:
                         self.to_server.exec_query(definition)
                     except (pymysql.err.ProgrammingError, pymysql.err.InternalError) as e:
-                        logger.error('Unable to create the internal view {p}: {e}')
+                        logger.error('Unable to create the internal view %s: %s', v, e)
 
+                self.status = SUCCESS
                 logger.info("> the database was successfully cloned")
             except RuntimeError:
-                logger.error(f"<!> An error happened while cloning the '{self.dbname}' database")
+                self.status = FAILURE
+                logger.error("<!> An error happened while cloning the '%s' database", self.dbname)
 
         finally:
             self.from_server.close()
@@ -424,12 +473,17 @@ def main(settings, arguments):
         to_server = servers[args['to_server']]
         kwargs = {k: v for k, v in args.items() if k not in ('dbname', 'from_server', 'to_server')}
 
-        op = CloningOperation(dbname, from_server, to_server, **kwargs)
+        op = CloningOperation(name, dbname, from_server, to_server, **kwargs)
         ops[name] = op
 
     # Operations to launch
-    if arguments.get('<dbname>', None):
-        selected_ops = [ops[arguments['<dbname>']]]
+    if arguments.get('<opname>', None):
+        selected_ops = []
+        for opname in arguments['<opname>']:
+            try:
+                selected_ops.append(ops[opname])
+            except KeyError:
+                logger.error('No operation found with name %s', opname)
     else:
         selected_ops = [op for op in ops.values() if op.is_default]
 
@@ -452,6 +506,10 @@ def main(settings, arguments):
     for op in selected_ops:
         op.run()
 
+    failures = [op.name for op in selected_ops if op.status == FAILURE]
+    if failures:
+        logger.error("WARNING! the following operations failed: %s", ', '.join(failures))
+
 
 if __name__ == '__main__':
     # load settings from settings.yml file

readme.md

@@ -4,24 +4,26 @@ Script de clonage des bases de données MySql
 
 ## Usage
 
-Usage:
-  clonedb.py [-v] [-y] [<opname>]
-  clonedb.py (-h | --help)
-  clonedb.py --version
+    Usage:
+      python3 clonedb.py [-v] [-y] [<opname>...]
+      python3 clonedb.py (-h | --help)
+      python3 clonedb.py --version
+    
+    Options:
+      -y, --yes       Do not ask for confirmation
+      -h --help       Show this screen.
+      --version       Show version.
 
-Options:
-  -y, --yes       Do not ask for confirmation
-  -h --help       Show this screen.
-  --version       Show version.
 
 ## Installation
 
-> requiert python 3.6+: `sudo apt-get install python3 python3-pip`
+> requiert python 3.6+ et mysql: `sudo apt-get install python3 python3-pip mysql-client`
 
     git clone https://gitlab.2iopenservice.com/olivier/clonedb
     cd clonedb
     pip3 install -r requirements.txt
 
+
 ## Configuration
 
 Pour configurer les opérations de clônage, ouvrez le fichier `settings.yml`.
@@ -52,7 +54,6 @@ La structure du fichier est la suivante:
         compress: True
         filter_tables: []
 
-### Configuration minimum
 
 Ajouter une entrée dans la section `servers` pour 
 le serveur d'origine et le serveur cible de l'opération de clônage.