implements the mysql user management and backward compat mysql 5/8

clonedb.py

@@ -54,9 +54,14 @@ LOG_MYSQL_QUERIES = True
 
 MAX_ALLOWED_PACKET = 1073741824
 
+CHARSET_TO_ENCODING = {
+    'utf8': 'utf-8',
+    'utf8mb4': 'utf-8',
+    'latin1': 'latin'
+}
 
-# Utilities
 
+# Utilities
 def load_settings():
     """ Load the settings from the 'settings.yml' file
     If there is no such file, the base settings.yml file is created
@@ -217,6 +222,13 @@ class MySqlServer:
         row = cursor.fetchone()
         return row is not None
 
+    def get_db_charset(self, dbname):
+        """ return the charset (encoding) of the mysql database """
+        cursor = self.exec_query(f"""SELECT default_character_set_name 
+                                     FROM information_schema.SCHEMATA S 
+                                     WHERE schema_name = '{dbname}';""")
+        return cursor.fetchone()[0]
+
     def list_tables(self, dbname=""):
         """ Return a list of tables (but not views!)
         for either the currently selected database,
@@ -236,7 +248,7 @@ class MySqlServer:
     def get_view_definition(self, view_name, set_definer=""):
         """ Return the SQL create statement for the view
         If 'set_definer' is not empty, the username in the 'SET DEFINER' part
-        of the create statement is replace by the one given
+        of the create statement is replaced by the one given
         """
         cursor = self.exec_query(f"show create view {view_name}")
         definition = cursor.fetchone()[1]
@@ -248,6 +260,13 @@ class MySqlServer:
         return definition
 
 
+class MysqlUser:
+    def __init__(self, username, pwd, host='localhost'):
+        self.username = username
+        self.pwd = pwd
+        self.host = host
+
+
 # Operation status
 UNKNOWN = 0
 SUCCESS = 1
@@ -263,12 +282,14 @@ class CloningOperation:
     """ A database cloning operation between two Mysql servers
     """
 
-    def __init__(self, name, dbname, from_server, to_server, is_default=True, ignore_tables=None, structure_only=None,
+    def __init__(self, name, dbname, from_server, to_server, grant=None,
+                 is_default=True, ignore_tables=None, structure_only=None,
                  filter_tables=None, ignore_views=None, compress=True):
         self.name = name
         self.dbname = dbname
         self.from_server = from_server
         self.to_server = to_server
+        self.grant = grant if grant is not None else []
 
         self.is_default = is_default
         self.compress = compress
@@ -298,6 +319,7 @@ class CloningOperation:
                     "--skip-add-drop-table",
                     "--skip-add-locks",
                     "--skip-comments",
+                    "--column-statistics=0"
                     ]
 
         if self.compress:
@@ -344,7 +366,22 @@ class CloningOperation:
         return cmd
 
     @staticmethod
-    def _run_piped_processes(dump_cmd, restore_cmd, tbl_count):
+    def _clean_sql(bin_cmd, encoding):
+        """ clean some old sql declaration from mysql 5 in order to preserve
+            a compatibility between servers"""
+        cmd = bin_cmd.decode('latin')
+
+        # To ensure compatibility between mysql5 and 8+
+        cmd = re.sub(",?NO_AUTO_CREATE_USER", "", cmd)
+
+        return cmd.encode('latin')
+
+    @staticmethod
+    def _run_piped_processes(
+            dump_cmd,
+            restore_cmd,
+            tbl_count,
+            encoding):
         """ Run the dump and the restore commands by piping them
         The output of the mysqldump process is piped into the input of the mysql one
         """
@@ -358,7 +395,9 @@ class CloningOperation:
             with Popen(restore_cmd, stdin=PIPE, stdout=mysql_handler, stderr=mysql_handler) as mysql:
                 # noinspection PyTypeChecker
                 with Popen(dump_cmd, stdout=PIPE, stderr=mysqldump_handler) as mysqldump:
-                    mysql.stdin.write(mysqldump.stdout.read())
+                    cmd = mysqldump.stdout.read()
+                    cmd = CloningOperation._clean_sql(cmd, encoding)
+                    mysql.stdin.write(cmd)
 
             if mysqldump.returncode:
                 raise RuntimeError('mysqldump returned a non zero code')
@@ -390,6 +429,18 @@ class CloningOperation:
             self.to_server.connect()
             logger.debug('Connected to %s', self.to_server)
 
+            # Create admin users if not exist
+            for user in self.grant:
+                exists = self.to_server.exec_query(
+                    f"SELECT count(*) FROM mysql.user WHERE User = '{user.username}' and Host='{user.host}';"
+                ).fetchone()[0] > 0
+
+                if not exists:
+                    logger.info(f'Create user %s@%s on %s', user.username, user.host, self.to_server)
+                    self.to_server.exec_query(
+                        f"CREATE USER '{user.username}'@'{user.host}' IDENTIFIED BY '{user.pwd}';")
+
+            # List tables
             tables = {}
             for tname in self.from_server.list_tables():
                 if any(rx.match(tname) for rx in self.ignore_tables):
@@ -422,15 +473,39 @@ class CloningOperation:
             self.to_server.exec_query(f"CREATE SCHEMA `{self.dbname}`;")
             self.to_server.set_active_db(self.dbname)
 
+            # Following is to avoid conflict between mysql 5 and mysql 8+
+            # (@see https://stackoverflow.com/questions/50336378/variable-sql-mode-cant-be-set-to-the-value-of-no-auto-create-user)
+            self.to_server.exec_query(f"SET GLOBAL log_bin_trust_function_creators = 1;")
+
+            # Grant admin users if any
+            for user in self.grant:
+                self.to_server.exec_query(
+                    f"GRANT ALL ON {self.dbname}.* TO '{user.username}'@'{user.host}';"
+                )
+
+            # charsets
+            charset = self.from_server.get_db_charset(self.dbname)
+            encoding = CHARSET_TO_ENCODING[charset]
+
             # Run mysqldump
             try:
                 if dump_structure_for:
                     logger.info(f"Cloning structure for {len(dump_structure_for)} tables (on {len(tables)})...")
-                    self._run_piped_processes(dump_structure_cmd, restore_cmd, len(dump_structure_for))
+                    self._run_piped_processes(
+                        dump_structure_cmd,
+                        restore_cmd,
+                        len(dump_structure_for),
+                        encoding
+                    )
 
                 if dump_data_for:
                     logger.info(f"Cloning data for {len(dump_data_for)} tables (on {len(tables)})...")
-                    self._run_piped_processes(dump_data_cmd, restore_cmd, len(dump_data_for))
+                    self._run_piped_processes(
+                        dump_data_cmd,
+                        restore_cmd,
+                        len(dump_data_for),
+                        encoding
+                    )
 
                 logger.info(f"Cloning views...")
                 self.from_server.set_active_db(self.dbname)
@@ -466,6 +541,8 @@ def main(settings, arguments):
 
     # Load the servers' configuration
     servers = {}
+    if 'servers' not in settings:
+        raise RuntimeError(f'Missing section in settings.yml: {servers}')
     for server_name, server_settings in settings['servers'].items():
         hostname = server_settings['host']
 
@@ -488,15 +565,27 @@ def main(settings, arguments):
 
         servers[server_name] = server
 
+    # Load the users' configuration
+    users = {}
+    for username, args in settings.get('users', {}).items():
+        host = args.get('host', 'localhost')
+        pwd = args.get('pwd', '')
+        users[username] = MysqlUser(username, pwd, host)
+
     # Load the cloning ops' configuration
     ops = {}
+    if 'operations' not in settings:
+        raise RuntimeError(f'Missing section in settings.yml: {servers}')
     for name, args in settings['operations'].items():
         dbname = args['dbname']
         from_server = servers[args['from_server']]
         to_server = servers[args['to_server']]
-        kwargs = {k: v for k, v in args.items() if k not in ('dbname', 'from_server', 'to_server')}
+        grant = args.get('grant', [])
+        admins = [user for username, user in users.items() if username in grant]
+        kwargs = {k: v for k, v in args.items() \
+                  if k not in ('dbname', 'from_server', 'to_server', 'grant')}
 
-        op = CloningOperation(name, dbname, from_server, to_server, **kwargs)
+        op = CloningOperation(name, dbname, from_server, to_server, admins, **kwargs)
         ops[name] = op
 
     # Operations to launch
@@ -526,6 +615,10 @@ def main(settings, arguments):
             return
         logger.debug('> User confirmed')
 
+    # Create the user if they do not exist
+    # CREATE USER IF NOT EXISTS 'user'@'localhost' IDENTIFIED BY 'password';
+    # GRANT ALL ON opentalent TO 'opentalent'@'localhost';
+
     # Run the cloning operations
     for op in selected_ops:
         op.run()

core/pipe_handler.py

@@ -37,9 +37,13 @@ class PipeHandler(threading.Thread):
     def run(self):
         """ Run the thread, logging everything.
         """
-        for line in iter(self.pipeReader.readline, ''):
-            self.process(line.strip('\n'))
-        self.pipeReader.close()
+        try:
+            for line in iter(self.pipeReader.readline, ''):
+                self.process(line.strip('\n'))
+        except UnicodeDecodeError:
+            self.process(' -- Error while decoding the incoming, unable to log --')
+        finally:
+            self.pipeReader.close()
 
     def close(self):
         """ Close the write end of the pipe.

readme.md

@@ -27,6 +27,8 @@ Script de clonage des bases de données MySql
 
 ## Configuration
 
+### Structure générale
+
 Pour configurer les opérations de clônage, ouvrez le fichier `settings.yml`.
 La structure du fichier est la suivante:
 
@@ -43,6 +45,11 @@ La structure du fichier est la suivante:
           port: 22
           user: user
 
+    users:
+      [username]:
+        hosts: localhost
+        pwd: [password]
+
     operations:
       [op_name]:
         dbname: my_db
@@ -54,7 +61,10 @@ La structure du fichier est la suivante:
         structure_only: []
         compress: True
         filter_tables: []
+        grant: []
+
 
+### Configurer les serveurs
 
 Ajouter une entrée dans la section `servers` pour 
 le serveur d'origine et le serveur cible de l'opération de clônage.
@@ -72,9 +82,22 @@ une section `ssh`, dans laquelle
 pourront apparaitre les paramètres de connexion SSH:
 `key_file` (chemin d'accès à la clé privée ssh), `port`, `user`
 
-> Les noms donnés à chaque serveur (`[server_name]`) n'ont pas d'importance 
+> Les noms donnés à chaque serveur (`[server_name]`) n'ont pas d'importance pour l'exécution de clonedb
+
+### Configurer les users (facultatif)
+
+Il est possible d'ajouter une section `users`. Les users listés dans cette section
+seront créés avant toute opération de clonage **seulement s'ils n'existent pas déjà**.
+Les users ne seront jamais remplacés.
+
+Les paramètres sont:
 
-Ensuite, ajouter une entrée dans la section `operations` et définir au moins les trois paramètres suivants: 
+* `host`: l'host correspondant ('localhost' par défaut)
+* `pwd`: le mot de passe
+
+### Configurer les opérations de clônage
+
+Enfin, ajouter une entrée dans la section `operations` et définir au moins les trois paramètres suivants: 
    * dbname: le nom de la base de données à cloner
    * from_server: le nom du serveur source tel qu'il a été défini à la section servers
    * to_server: le nom du serveur cible tel qu'il a été défini à la section servers
@@ -89,9 +112,4 @@ correspondantes sera clonée, pas les données contenues.
 * `ignore_views`: liste de noms de tables ou d'expressions régulières, les vues correspondantes seront ignorées
 * `filter_tables`: liste de noms de tables ou d'expressions régulières; si une liste est donnée, 
 seules les tables correspondantes seront traitées.
-
-
-## Workarounds
-
-* `The user specified as a definer ('username'@'%') does not exist`: add a user account 
-to the targeted mysql server named `username@%` with admin privileges
+* `grant`: une liste des users à qui seront accordés les droits d'admin sur cette base

settings.yml.dist

@@ -44,6 +44,15 @@ servers:
       port: 22
       user: exploitation
 
+users:
+  opentalent:
+    host: localhost
+    pwd: nSDC3Bm3ws88qKjV63
+
+  openassos:
+    host: localhost
+    pwd: Vq2icge7SM3P26CaC3
+
 operations:
   crm:
     dbname: crm
@@ -51,6 +60,8 @@ operations:
     to_server: docker
     is_default: True
     compress: True
+    grant:
+      - opentalent
 
   adminassos:
     dbname: adminassos
@@ -67,6 +78,8 @@ operations:
     compress: True
     ignore_tables: ['Access', 'matomo.*', 'user_piwik.*', 'zzz_.*', 'tt_content_save', 'tt_content_v59']
     structure_only: ['cf_cache_.*', 'sys_log']
+    grant:
+      - openassos
 
   opentalent:
     dbname: opentalent
@@ -77,6 +90,8 @@ operations:
     structure_only: ['Audit_.*']
     ignore_tables: []
     ignore_views: []
+    grant:
+      - opentalent
 
   metabase:
     dbname: metabase