update clonedb

clonedb/clonedb.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 """
-Script de clonage des bases de données MySql
+Script de clonage des bases de données MariaDb 10.*
 (requiert python 3.6+)
 
 > Configuration: settings.yml
@@ -54,12 +54,6 @@ LOG_MYSQL_QUERIES = True
 
 MAX_ALLOWED_PACKET = 1073741824
 
-CHARSET_TO_ENCODING = {
-    'utf8': 'utf-8',
-    'utf8mb4': 'utf-8',
-    'latin1': 'latin'
-}
-
 
 # Utilities
 def load_settings():
@@ -318,10 +312,15 @@ class CloningOperation:
                     f"--max-allowed-packet={MAX_ALLOWED_PACKET}",
                     "--skip-add-drop-table",
                     "--skip-add-locks",
-                    "--skip-comments",
-                    "--column-statistics=0"
+                    "--skip-comments"
                     ]
 
+        try:
+            self.from_server.exec_query("SELECT COLUMN_NAME FROM information_schema.COLUMN_STATISTICS;")
+        except pymysql.err.OperationalError:
+            # fix the occasional 'Unknown table 'COLUMN_STATISTICS' in information_schema' bug
+            base_cmd.append("--column-statistics=0")
+
         if self.compress:
             base_cmd.append("--compress")
 
@@ -365,23 +364,11 @@ class CloningOperation:
             cmd.append("--compress")
         return cmd
 
-    @staticmethod
-    def _clean_sql(bin_cmd, encoding):
-        """ clean some old sql declaration from mysql 5 in order to preserve
-            a compatibility between servers"""
-        cmd = bin_cmd.decode('latin')
-
-        # To ensure compatibility between mysql5 and 8+
-        cmd = re.sub(",?NO_AUTO_CREATE_USER", "", cmd)
-
-        return cmd.encode('latin')
-
     @staticmethod
     def _run_piped_processes(
             dump_cmd,
             restore_cmd,
-            tbl_count,
-            encoding):
+            tbl_count):
         """ Run the dump and the restore commands by piping them
         The output of the mysqldump process is piped into the input of the mysql one
         """
@@ -395,9 +382,7 @@ class CloningOperation:
             with Popen(restore_cmd, stdin=PIPE, stdout=mysql_handler, stderr=mysql_handler) as mysql:
                 # noinspection PyTypeChecker
                 with Popen(dump_cmd, stdout=PIPE, stderr=mysqldump_handler) as mysqldump:
-                    cmd = mysqldump.stdout.read()
-                    cmd = CloningOperation._clean_sql(cmd, encoding)
-                    mysql.stdin.write(cmd)
+                    mysql.stdin.write(mysqldump.stdout.read())
 
             if mysqldump.returncode:
                 raise RuntimeError('mysqldump returned a non zero code')
@@ -473,20 +458,12 @@ class CloningOperation:
             self.to_server.exec_query(f"CREATE SCHEMA `{self.dbname}`;")
             self.to_server.set_active_db(self.dbname)
 
-            # Following is to avoid conflict between mysql 5 and mysql 8+
-            # (@see https://stackoverflow.com/questions/50336378/variable-sql-mode-cant-be-set-to-the-value-of-no-auto-create-user)
-            self.to_server.exec_query(f"SET GLOBAL log_bin_trust_function_creators = 1;")
-
             # Grant admin users if any
             for user in self.grant:
                 self.to_server.exec_query(
                     f"GRANT ALL ON {self.dbname}.* TO '{user.username}'@'{user.host}';"
                 )
 
-            # charsets
-            charset = self.from_server.get_db_charset(self.dbname)
-            encoding = CHARSET_TO_ENCODING[charset]
-
             # Run mysqldump
             try:
                 if dump_structure_for:
@@ -494,8 +471,7 @@ class CloningOperation:
                     self._run_piped_processes(
                         dump_structure_cmd,
                         restore_cmd,
-                        len(dump_structure_for),
-                        encoding
+                        len(dump_structure_for)
                     )
 
                 if dump_data_for:
@@ -503,8 +479,7 @@ class CloningOperation:
                     self._run_piped_processes(
                         dump_data_cmd,
                         restore_cmd,
-                        len(dump_data_for),
-                        encoding
+                        len(dump_data_for)
                     )
 
                 logger.info(f"Cloning views...")
@@ -615,10 +590,6 @@ def main(settings, arguments):
             return
         logger.debug('> User confirmed')
 
-    # Create the user if they do not exist
-    # CREATE USER IF NOT EXISTS 'user'@'localhost' IDENTIFIED BY 'password';
-    # GRANT ALL ON opentalent TO 'opentalent'@'localhost';
-
     # Run the cloning operations
     for op in selected_ops:
         op.run()

clonedb/settings.yml.dist

@@ -7,6 +7,14 @@ servers:
       username: root
       password: mysql660
 
+  localhost:
+    host: localhost
+    description: Local Mysql
+    mysql:
+      port: 3305
+      username: root
+      password: mysql660
+
   preprod:
     host: preprod.2iopenservice.com
     description: Preprod
@@ -53,6 +61,10 @@ users:
     host: localhost
     pwd: Vq2icge7SM3P26CaC3
 
+  typo3:
+    host: localhost
+    pwd: Vq2icge7SM3P26CaC3
+
 operations:
   crm:
     dbname: crm
@@ -81,6 +93,16 @@ operations:
     grant:
       - openassos
 
+  typo3:
+    dbname: typo3
+    from_server: prod-front
+    to_server: docker
+    is_default: True
+    compress: True
+    structure_only: ['cf_cache_.*', 'sys_log']
+    grant:
+      - typo3
+
   opentalent:
     dbname: opentalent
     from_server: prod-back