minor fixes

clonedb.py

@@ -48,10 +48,8 @@ sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
 
 # Options
 SHOW_PROGRESSION = True
-LOG_PIPES_OUTPUT = False
-LOG_MYSQL_QUERIES = False
-
-MY_CNF_FILE = Path(HERE / "my.cnf")
+LOG_PIPES_OUTPUT = True
+LOG_MYSQL_QUERIES = True
 
 MAX_ALLOWED_PACKET = 1073741824
 
@@ -114,6 +112,7 @@ class MysqldumpHandler(PipeHandler):
         """
         super().close()
 
+
 class MysqlHandler(MysqldumpHandler):
     """ Handle and process the stdout / stderr output from a mysql process
     """
@@ -173,7 +172,7 @@ class MySqlServer:
                                    user=self.username,
                                    password=self.password,
                                    autocommit=autocommit,
-                                   max_allowed_packet="32M",
+                                   max_allowed_packet=MAX_ALLOWED_PACKET,
                                    )
 
         if not self.cnn.open:
@@ -434,6 +433,7 @@ class CloningOperation:
 
                 logger.info(f"Cloning views...")
                 self.from_server.set_active_db(self.dbname)
+                self.to_server.set_active_db(self.dbname)
                 for v in self.from_server.list_views(self.dbname):
                     if any(rx.match(v) for rx in self.ignore_views):
                         continue

core/clonedb.log

@@ -1,32 +0,0 @@
-2020-05-20 13:54:37,153 - INFO - Start db cloning utility...
-2020-05-20 13:54:37,153 - DEBUG - Settings: {'servers': {'docker': {'host': 'localhost', 'description': 'Docker mariaDb', 'mysql': {'port': 3306, 'username': 'root', 'password': 'mysql660'}}, 'localhost': {'host': 'localhost', 'description': 'Local Mysql', 'mysql': {'port': 3305, 'username': 'root', 'password': 'mysql660'}}, 'preprod': {'host': 'preprod.2iopenservice.com', 'description': 'Preprod', 'mysql': {'port': 3306, 'username': 'dbcloner', 'password': 'wWZ4hYcrmHLW2mUK'}, 'ssh': {'key_file': '~/.ssh/id_rsa_exploitation', 'port': 22, 'user': 'exploitation'}}, 'prod-back': {'host': 'prod-back.2iopenservice.com', 'description': 'Prod-Back', 'mysql': {'port': 3306, 'username': 'dbcloner', 'password': 'wWZ4hYcrmHLW2mUK'}, 'ssh': {'key_file': '~/.ssh/id_rsa_exploitation', 'port': 22, 'user': 'exploitation'}}, 'prod-front': {'host': 'prod-front.2iopenservice.com', 'description': 'Prod-Front', 'mysql': {'port': 3306, 'username': 'dbcloner', 'password': 'wWZ4hYcrmHLW2mUK'}, 'ssh': {'key_file': '~/.ssh/id_rsa_exploitation', 'port': 22, 'user': 'exploitation'}}}, 'operations': {'crm': {'dbname': 'crm', 'from_server': 'preprod', 'to_server': 'docker', 'is_default': True, 'ignore_tables': [], 'ignore_views': [], 'structure_only': [], 'compress': True}, 'adminassos': {'dbname': 'adminassos', 'from_server': 'preprod', 'to_server': 'docker'}, 'openassos': {'dbname': 'openassos', 'from_server': 'preprod', 'to_server': 'docker', 'is_default': True, 'ignore_tables': ['Access', 'matomo.*', 'user_piwik.*', 'zzz_.*', 'tt_content_save', 'tt_content_v59'], 'structure_only': ['cf_cache_.*', 'sys_log']}, 'opentalent': {'dbname': 'opentalent', 'from_server': 'preprod', 'to_server': 'docker', 'structure_only': [], 'ignore_tables': ['Audit_.*'], 'ignore_views': ['view_audit']}, 'metabase': {'dbname': 'metabase', 'from_server': 'preprod', 'to_server': 'docker'}}}
-2020-05-20 13:54:37,153 - DEBUG - Arguments: {'--help': False, '--version': False, '--yes': True, '-v': False, '<dbname>': 'crm'}
-2020-05-20 13:54:37,155 - INFO - *** Cloning crm ***
-2020-05-20 13:54:37,155 - INFO - > From Preprod (preprod.2iopenservice.com:3306 as dbcloner)
-2020-05-20 13:54:37,155 - INFO - > To Docker mariaDb (localhost:3306 as root)
-2020-05-20 13:54:37,895 - DEBUG - Connected to Preprod (preprod.2iopenservice.com:3306 as dbcloner)
-2020-05-20 13:54:37,897 - DEBUG - Connected to Docker mariaDb (localhost:3306 as root)
-2020-05-20 13:54:38,001 - INFO - (Re)create the database
-2020-05-20 13:54:38,004 - INFO - Cloning structure for 16 tables (on 16)...
-2020-05-20 13:54:38,005 - DEBUG - >>> Dump command: mysqldump --single-transaction -u dbcloner --password=wWZ4hYcrmHLW2mUK --skip-add-drop-table --skip-add-locks --skip-comments --compress --verbose --host 127.0.0.1 --port 6000 --single-transaction --no-data --routines crm cm_action cm_action_com cm_action_histo cm_asso cm_asso_trader cm_bank cm_contact cm_invoice cm_miscellaneous cm_miscellaneous_com cm_pes_assos_config cm_plugin_assos cm_plugins cm_product cm_product_net cm_trader
-2020-05-20 13:54:38,005 - DEBUG - >>> Piped into: mysql -h localhost -P 3306 -u root --password=mysql660 --max-allowed-packet=1073741824 --reconnect --quick -D crm --compress
-2020-05-20 13:54:39,663 - DEBUG - ... cm_action
-2020-05-20 13:54:40,335 - DEBUG - ... cm_action_com
-2020-05-20 13:54:41,006 - DEBUG - ... cm_action_histo
-2020-05-20 13:54:41,672 - DEBUG - ... cm_asso
-2020-05-20 13:54:42,340 - DEBUG - ... cm_asso_trader
-2020-05-20 13:54:43,013 - DEBUG - ... cm_bank
-2020-05-20 13:54:43,687 - DEBUG - ... cm_contact
-2020-05-20 13:54:44,360 - DEBUG - ... cm_invoice
-2020-05-20 13:54:45,122 - DEBUG - ... cm_miscellaneous
-2020-05-20 13:54:45,970 - DEBUG - ... cm_miscellaneous_com
-2020-05-20 13:54:46,662 - DEBUG - ... cm_pes_assos_config
-2020-05-20 13:54:47,347 - DEBUG - ... cm_plugin_assos
-2020-05-20 13:54:48,033 - DEBUG - ... cm_plugins
-2020-05-20 13:54:48,746 - DEBUG - ... cm_product
-2020-05-20 13:54:49,427 - DEBUG - ... cm_product_net
-2020-05-20 13:54:50,110 - DEBUG - ... cm_trader
-2020-05-20 13:54:51,173 - ERROR - Execution failed: 
-2020-05-20 13:54:51,174 - ERROR - <!> An error happened while cloning the 'crm' database
-2020-05-20 13:54:51,234 - DEBUG - Preprod (preprod.2iopenservice.com:3306 as dbcloner) - connection closed
-2020-05-20 13:54:51,235 - DEBUG - Docker mariaDb (localhost:3306 as root) - connection closed

core/pipe_handler.py

@@ -18,7 +18,7 @@ class PipeHandler(threading.Thread):
         threading.Thread.__init__(self)
         self.daemon = False
         self.fdRead, self.fdWrite = os.pipe()
-        self.pipeReader = os.fdopen(self.fdRead)
+        self.pipeReader = os.fdopen(self.fdRead, encoding='utf-8')
         self.start()
 
         self.logger = logging.getLogger(logger_name)

readme.md

@@ -17,12 +17,13 @@ Script de clonage des bases de données MySql
 
 ## Installation
 
-> requiert python 3.6+ et mysql: `sudo apt-get install python3 python3-pip mysql-client`
+> requiert python 3.6+ et mysql: `sudo apt-get install python3 python3-pip mariadb-client-10.3`
 
     git clone https://gitlab.2iopenservice.com/olivier/clonedb
     cd clonedb
     pip3 install -r requirements.txt
 
+> mysqldump doit être en version 10+  
 
 ## Configuration
 

settings.yml.dist

@@ -74,9 +74,9 @@ operations:
     to_server: docker
     is_default: True
     compress: True
-    structure_only: []
-    ignore_tables: ['Audit_.*']
-    ignore_views: ['view_audit']
+    structure_only: ['Audit_.*']
+    ignore_tables: []
+    ignore_views: []
 
   metabase:
     dbname: metabase