v0.1 - reproduit l'existant

.gitignore

@@ -0,0 +1 @@
+/.project

clonedb.log

@@ -1,13 +1,12 @@
-2020-05-14 15:18:48,823 - DEBUG - Arguments given: {'--help': False,
- '--verbose': True,
+2020-05-15 16:32:20,584 - INFO - Start db cloning utility...
+2020-05-15 16:32:20,585 - DEBUG - Arguments: {'--help': False,
  '--version': False,
- '--yes': True,
- 'dbname': False}
-2020-05-14 15:18:48,823 - DEBUG - Settings: {'origin': {'host': 'localhost', 'port': 3306, 'username': 'root', 'password': 'mysql660', 'description': 'Docker mariaDb'}, 'local': {'host': 'localhost', 'port': 3305, 'username': 'root', 'password': 'mysql660', 'description': 'local mysql server'}, 'databases': {'crm': None}}
-2020-05-14 15:18:48,823 - DEBUG - Mode: Verbose
-2020-05-14 15:18:48,823 - DEBUG - Try to connect to root@localhost:3306 (Docker mariaDb)
-2020-05-14 15:18:48,827 - DEBUG - Try to connect to root@localhost:3305 (local mysql server)
-2020-05-14 15:18:48,828 - INFO - *** Cloning crm ***
-2020-05-14 15:18:48,828 - DEBUG - From root@localhost:3306 (Docker mariaDb)
-2020-05-14 15:18:48,828 - DEBUG - To root@localhost:3305 (local mysql server)
-2020-05-14 15:18:48,829 - INFO - -- Clonage des bases de données terminé --
+ '--yes': False,
+ '-v': False,
+ '<dbname>': 'crm'}
+2020-05-15 16:32:20,585 - DEBUG - Settings: {'ssh': {'host': 'preprod.2iopenservice.com', 'key_file': '~/.ssh/id_rsa_exploitation', 'port': 22, 'user': 'exploitation'}, 'remote': {'host': 'preprod.2iopenservice.com', 'port': 3306, 'username': 'dbcloner', 'password': 'wWZ4hYcrmHLW2mUK', 'description': 'Preprod'}, 'local': {'host': 'localhost', 'port': 3306, 'username': 'root', 'password': 'mysql660', 'description': 'Docker mariaDb'}, 'databases': {'crm': None, 'adminassos': None, 'openassos': None, 'opentalent': None, 'metabase': None}}
+2020-05-15 16:32:20,585 - DEBUG - Ask for confirmation...
+2020-05-15 16:32:24,094 - DEBUG - > user confirmed by answering 'y'
+2020-05-15 16:32:24,095 - INFO - *** Cloning crm ***
+2020-05-15 16:32:24,095 - DEBUG - From dbcloner@preprod.2iopenservice.com:3306 (Preprod)
+2020-05-15 16:32:24,095 - DEBUG - To root@localhost:3306 (Docker mariaDb)

clonedb.py

@@ -6,12 +6,11 @@ serveur de production vers le serveur local
 > Configuration: settings.yml
 
 Usage:
-  clonedb.py [-v] [-y] [dbname]
+  clonedb.py [-v] [-y] [<dbname>]
   clonedb.py (-h | --help)
   clonedb.py --version
 
 Options:
-  -v, --verbose   Displays more informations
   -y, --yes       Do not ask for confirmation
   -h --help       Show this screen.
   --version       Show version.
@@ -19,138 +18,160 @@ Options:
 @author: olivier.massot, 05-2020
 """
 import logging
+import subprocess
 import sys
 
-import mysql.connector
 import yaml
 from docopt import docopt
-
 from path import Path
 
 import logging_
+from locker import Lockfile, AlreadyRunning
+from mysql import MySqlServer
 
 __VERSION__ = "0.1"
 
 HERE = Path(__file__).parent
-LOCKFILE = HERE / '.clonedb.lock'
 
+# Start logger
+logger = logging.getLogger('clonedb')
+logging_.start("clonedb", replace=True)
+
+# Load settings
 with open(HERE / 'settings.yml', 'r') as f:
     SETTINGS = yaml.load(f, Loader=yaml.FullLoader)
 
+# FIX the default ascii encoding on some linux dockers...
+sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
+
+# Utilities
+def _format_cmd(cmd):
+    if type(cmd) is list:
+        return " ".join(cmd)
+    return cmd
+
+
+def _run_process(cmds):
+    logger.debug("Run: %s", " ".join(map(str, cmds)))
+    logpipe = logging_.LogPipe('clonedb')
+    try:
+        # noinspection PyTypeChecker
+        p = subprocess.run(cmds, shell=False, stdin=sys.stdin, stdout=logpipe, stderr=logpipe)
+        if p.returncode != 0:
+            logger.error("Process was terminated by signal %s", p.returncode)
+            raise RuntimeError()
+
+    except (OSError, RuntimeError) as e:
+        logger.error("Execution failed: %s", e)
+        raise RuntimeError(f"An error happened at runtime: {e}")
+
+    finally:
+        logpipe.close()
+
 
-class MySqlServer:
-    def __init__(self, host, port, username, password, description=""):
+class SshTunnel:
+    def __init__(self, host, port=22, user="root", key_file="~/.ssh/id_rsa"):
         self.host = host
         self.port = port
-        self.username = username
-        self.password = password
-        self.description = description or "no description"
-
-        self.cnn = None
-
-    def __repr__(self):
-        return f"{self.username}@{self.host}:{self.port} ({self.description})"
-
-    def connect(self):
-        logger.debug(f'Try to connect to {self}')
-        self.cnn = mysql.connector.connect(
-            host=self.host,
-            port=self.port,
-            user=self.username,
-            passwd=self.password
-        )
-
-    def db_exists(self, dbname):
-        cursor = self.cnn.cursor()
-        cursor.execute(f"""SELECT SCHEMA_NAME
-                           FROM INFORMATION_SCHEMA.SCHEMATA
-                           WHERE SCHEMA_NAME = '{dbname}'""")
-        row = cursor.fetchone()
-        return row is not None
-
-
-def clonedb(from_server, to_server, dbname):
-    logger.info(f"*** Cloning {dbname} ***")
-    logger.debug(f"From {from_server}")
-    logger.debug(f"To {to_server}")
+        self.user = user
+        self.key_file = key_file
 
-    to_server.cnn.cmd_query('DROP DATABASE {dbname};')
-    to_server.cnn.cmd_query('CREATE DATABASE {dbname};')
+    def call(self, cmd):
+        ssh_call = ["ssh",
+                    "-i", self.key_file,
+                    "-p", str(self.port),
+                    "-o", "StrictHostKeyChecking=no",
+                    f"{self.user}@{self.host}",
+                    ]
 
-    dump_cmd = "mysqldump --single-transaction -u ${USERDBROOTREMOTE} --password=${PASSDBROOTREMOTE} $1"
-    ssh_dump_cmd = f"ssh -i {SSHEXPLOITATIONKEY} -p {PORT} -C exploitation@${IPPROD} {dump_cmd}"
+        ssh_cmd = ssh_call + cmd
+        _run_process(ssh_cmd)
 
-    restore_cmd = "mysql -h ${3} -P ${4} -u ${USERDBROOT} --password=${PASSDBROOT} -D $2"
 
-    cmd = f"{ssh_dump_cmd} | {restore_cmd}"
+def clonedb(ssh_tunnel, remote_mysql, local_mysql, dbname, options):
+    logger.info(f"*** Cloning {dbname} ***")
+    logger.debug(f"From {remote_mysql}")
+    logger.debug(f"To {local_mysql}")
+
+    dump_cmd = ["mysqldump",
+                "--single-transaction",
+                "-u", remote_mysql.username,
+                f"--password={remote_mysql.password}",
+                "--add-drop-database",
+                "--compress",
+                dbname]
+
+    restore_cmd = ["mysql",
+                   "-h", local_mysql.host,
+                   "-P", str(local_mysql.port),
+                   "-u", local_mysql.username,
+                   f"--password={local_mysql.password}",
+                   "-D", dbname]
+
+    cloning_cmd = dump_cmd + ['|'] + restore_cmd
 
+    try:
+        ssh_tunnel.call(cloning_cmd)
+        logger.info("> the database was successfully cloned")
+    except RuntimeError:
+        logger.error(f"<!> An error happened while cloning the '{dbname}' database")
+
+
+def ask_confirmation(msg):
+    logger.debug('Ask for confirmation...')
+    msg += "\nWould you like to continue? (yes/no)"
+    while 1:
+        answer = input(msg)
+        if answer in ('oui', 'yes', 'y', 'o'):
+            logger.debug(f"> user confirmed by answering '{answer}'")
+            return True
+        elif answer in ('non', 'no', 'n'):
+            logger.debug(f"> user cancelled by answering '{answer}'")
+            return False
+        else:
+            msg = "The answer could'nt be understood. Continue? (yes/no)"
+
+
+def main(dbnames, prompt=True):
+    ssh_tunnel = SshTunnel(**SETTINGS['ssh'])
+    remote_mysql = MySqlServer(**SETTINGS['remote'])
+    local_mysql = MySqlServer(**SETTINGS['local'])
+
+    if prompt:
+        # Ask for confirmation
+        msg = f"""The followind databases will be cloned
+from '{remote_mysql}' to '{local_mysql}':
+> {', '.join(dbnames)}
+"WARNING: the existing local databases will be replaced"""
+
+        if not ask_confirmation(msg):
+            logger.info("-- Operation cancelled by user --")
+            return
 
-    cmd = "ssh -i ${SSHEXPLOITATIONKEY} -p ${PORT} -C exploitation@${IPPROD} mysqldump --single-transaction -u ${USERDBROOTREMOTE} --password=${PASSDBROOTREMOTE} $1 | mysql -h ${3} -P ${4} -u ${USERDBROOT} --password=${PASSDBROOT} -D $2"
+    # start to clone
+    for dbname in dbnames:
+        options = SETTINGS['databases'].get(dbname, None) or {}
+        clonedb(ssh_tunnel, remote_mysql, local_mysql, dbname, options)
 
 
 if __name__ == '__main__':
-    arguments = docopt(__doc__, help=__doc__, version=__VERSION__)
-    verbose = '--verbose' in arguments
 
-    logger = logging.getLogger('clonedb')
-    logging_.start("clonedb", logging.DEBUG if verbose else logging.INFO, replace=True)
-
-    if LOCKFILE.exists():
-        logger.critical("Une opération de clonage est déjà en cours. "
-                        "veuillez patienter ou annuler le traitement existant")
-        sys.exit(1)
+    # parse CLI arguments
+    arguments = docopt(__doc__, help=__doc__, version=__VERSION__)
+    prompt = "--yes" in arguments
+    dbname = arguments.get('dbname', None)
 
-    logger.debug(f"Arguments given: {arguments}")
+    print('\n')
+    logger.info("Start db cloning utility...")
+    logger.debug(f"Arguments: {arguments}")
     logger.debug(f"Settings: {SETTINGS}")
-    if verbose:
-        logger.debug("Mode: Verbose")
-
-    remote_server = MySqlServer(**SETTINGS['remote'])
-    remote_server.connect()
 
-    local_server = MySqlServer(**SETTINGS['local'])
-    local_server.connect()
-
-    if arguments['dbname']:
-        dbnames = [arguments['dbname']]
+    # database to clone
+    if arguments.get('<dbname>', None):
+        dbnames = [arguments['<dbname>']]
     else:
-        dbnames = [db for db in SETTINGS['databases']]
-
-    # Demande confirmation
-    if not '--yes' in arguments:
-        logger.debug('Ask for confirmation...')
-        answer = ""
-        msg = f"Les bases de données suivantes vont être clonées depuis " \
-              f"'{remote_server} vers '{local_server}':\n' " \
-              f"> {', '.join(dbnames)} \n" \
-              "ATTENTION: Les bases existantes seront remplacées.\n" \
-              "Voulez vous continuer? (oui/non)"
-        while 1:
-            answer = input(msg)
-            if answer in ('oui', 'yes', 'y', 'o'):
-                logger.debug(f"> user confirmed by answering '{answer}'")
-                break
-            elif answer in ('non', 'no', 'n'):
-                logger.info("-- Opération annulée par l'utilisateur --")
-                sys.exit(1)
-            else:
-                msg = "La réponse n'a pas été comprise. Voulez vous continuer? (oui/non)"
-
-    # check for databases existence
-    missing_db = [dbname for dbname in dbnames if not remote_server.db_exists(dbname)]
-    if missing_db:
-        for missing in missing_db:
-            logger.critical(
-                f"<!> Aucune base de donnée nommée '{missing}' trouvée sur {remote_server}")
-        logger.critical("-- Opération annulée --")
-        sys.exit(1)
+        dbnames = list(SETTINGS['databases'])
 
-    # start to clone
-    try:
-        LOCKFILE.touch()
-        for dbname in dbnames:
-            clonedb(remote_server, local_server, dbname)
-
-        logger.info("-- Clonage des bases de données terminé --")
-    finally:
-        LOCKFILE.remove()
+    with Lockfile(path=HERE / '.clonedb.lock',
+                  on_error=lambda: logger.critical("A cloning process is already running, please wait...")):
+        main(dbnames, prompt)

future_settings.yml

@@ -1,18 +0,0 @@
-origin:
-  host: preprod.opentalent.fr
-  port: 3306
-  username: openassos
-  password: Vq2icge7SM3P26CaC3
-
-local:
-  host: localhost
-  port: 3306
-  username:
-  password:
-
-databases:
-  crm:
-  commercial:
-  openassos:
-  adminassos:
-  portail:

locker.py

@@ -0,0 +1,27 @@
+"""
+Lockfile utility
+
+@author: olivier.massot, 05-2020
+"""
+from path import Path
+
+
+class AlreadyRunning(RuntimeError):
+    pass
+
+def on_error():
+    raise AlreadyRunning
+
+class Lockfile:
+    def __init__(self, path, on_error=None):
+        self.path = Path(path)
+        self.on_error = on_error
+
+    def __enter__(self):
+        if self.path.exists():
+            self.on_error() if self.on_error is not None else on_error()
+        self.path.touch()
+
+    def __exit__(self, type, value, traceback):
+        self.path.remove()
+

logging.yml

@@ -13,7 +13,7 @@ formatters:
 handlers:
     console:
         class: logging.StreamHandler
-        level: DEBUG
+        level: INFO
         formatter: short
         
     file:

logging_.py

@@ -3,7 +3,9 @@
 @author: devone, 02-2020
 '''
 import logging.config
+import os
 import sys
+import threading
 import traceback
 import yaml
 
@@ -37,3 +39,37 @@ def start(name="main", level=0, filename="", replace=False):
         logger.error("{}\n{}\n{}".format(typ.__name__, value, ''.join(traceback.format_tb(trace))))
         SYS_EXCEPT_HOOK(typ, value, trace)
     sys.excepthook = _excepthook
+
+
+class LogPipe(threading.Thread):
+
+    def __init__(self, logger_name, level=logging.INFO):
+        """Setup the object with a logger and a loglevel
+        and start the thread
+        """
+        threading.Thread.__init__(self)
+        self.logger_name = logger_name
+        self.daemon = False
+        self.level = level
+        self.fdRead, self.fdWrite = os.pipe()
+        self.pipeReader = os.fdopen(self.fdRead)
+        self.start()
+
+    def fileno(self):
+        """Return the write file descriptor of the pipe
+        """
+        return self.fdWrite
+
+    def run(self):
+        """Run the thread, logging everything.
+        """
+        logger = logging.getLogger(self.logger_name)
+        for line in iter(self.pipeReader.readline, ''):
+            logger.log(self.level, line.strip('\n'))
+
+        self.pipeReader.close()
+
+    def close(self):
+        """Close the write end of the pipe.
+        """
+        os.close(self.fdWrite)

mysql.py

@@ -0,0 +1,41 @@
+"""
+Mysql utilities
+
+@author: olivier.massot, 05-2020
+"""
+import mysql
+
+
+class MySqlServer:
+    def __init__(self, host, port, username, password, description=""):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.description = description or "no description"
+        self.cnn = None
+
+    def __repr__(self):
+        return f"{self.username}@{self.host}:{self.port} ({self.description})"
+
+    def connect(self):
+        self.cnn = mysql.connector.connect(
+            host=self.host,
+            port=self.port,
+            user=self.username,
+            passwd=self.password
+        )
+
+    def exec(self, sql):
+        """ Execute the sql code and return the cursor """
+        cursor = self.cnn.cursor()
+        logger.debug(sql)
+        cursor.execute(sql)
+        return cursor
+
+    def db_exists(self, dbname):
+        cursor = self.exec(f"""SELECT SCHEMA_NAME
+                           FROM INFORMATION_SCHEMA.SCHEMATA
+                           WHERE SCHEMA_NAME = '{dbname}'""")
+        row = cursor.fetchone()
+        return row is not None

readme.md

@@ -0,0 +1,18 @@
+# CloneDB
+
+Script de clonage des bases de données mariaDb depuis le
+serveur de production vers le serveur local
+(requiert python 3.6+)
+
+> Configuration: settings.yml
+
+    Usage:
+      clonedb.py [-v] [-y] [dbname]
+      clonedb.py (-h | --help)
+      clonedb.py --version
+    
+    Options:
+      -v, --verbose   Displays more informations
+      -y, --yes       Do not ask for confirmation
+      -h --help       Show this screen.
+      --version       Show version.

settings.yml

@@ -1,17 +1,26 @@
+ssh:
+  host: preprod.2iopenservice.com
+  key_file: ~/.ssh/id_rsa_exploitation
+  port: 22
+  user: exploitation
+
 remote:
-  host: localhost
+  host: preprod.2iopenservice.com
   port: 3306
-  username: root
-  password: mysql660
-  description: Docker mariaDb
-  ssh_key: ~/.ssh/id_rsa_exploitation
+  username: dbcloner
+  password: wWZ4hYcrmHLW2mUK
+  description: Preprod
 
 local:
   host: localhost
-  port: 3305
+  port: 3306
   username: root
   password: mysql660
-  description: local mysql server
+  description: Docker mariaDb
 
 databases:
-  crm:
+  crm:
+  adminassos:
+  openassos:
+  opentalent:
+  metabase: