Init

core/BufferingSMTPHandler.py

@@ -0,0 +1,56 @@
+# Copyright 2001-2002 by Vinay Sajip. All Rights Reserved.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose and without fee is hereby granted,
+# provided that the above copyright notice appear in all copies and that
+# both that copyright notice and this permission notice appear in
+# supporting documentation, and that the name of Vinay Sajip
+# not be used in advertising or publicity pertaining to distribution
+# of the software without specific, written prior permission.
+# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
+# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
+# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
+# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+# This file is part of the Python logging distribution. See
+# http://www.red-dove.com/python_logging.html
+#
+"""Test harness for the logging module. Tests BufferingSMTPHandler, an alternative implementation
+of SMTPHandler.
+Copyright (C) 2001-2002 Vinay Sajip. All Rights Reserved.
+"""
+
+from email.mime.text import MIMEText
+import logging.handlers
+import smtplib
+
+
+class BufferingSMTPHandler(logging.handlers.BufferingHandler):
+    def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity):
+        logging.handlers.BufferingHandler.__init__(self, capacity)
+        self.mailhost = mailhost
+        self.mailport = None
+        self.fromaddr = fromaddr
+        self.toaddrs = toaddrs
+        self.subject = subject
+
+    def flush(self):
+        try:
+            if len(self.buffer) > 0:
+                port = self.mailport if self.mailport else smtplib.SMTP_PORT
+
+                msg = "\n".join([self.format(record) for record in self.buffer])
+                msg = MIMEText(msg.encode('utf-8'), _charset='utf-8')
+                msg['Subject'] = self.subject
+                msg['From'] = self.fromaddr
+                msg['To'] = ",".join(self.toaddrs)
+
+                smtp = smtplib.SMTP(self.mailhost, port)
+                smtp.sendmail(self.fromaddr, self.toaddrs, msg.as_string())
+                smtp.quit()
+                self.buffer = []
+        except Exception as e:
+            print(e)
+            raise

core/db.py

@@ -0,0 +1,135 @@
+'''
+    Convenient access to various databases
+'''
+from collections import namedtuple
+from datetime import datetime
+import logging
+
+import pypyodbc
+
+
+pypyodbc.lowercase = False
+
+logger = logging.getLogger("database")
+
+class CustomDb(pypyodbc.Connection):
+    """ Connexion to a database """
+    _cache = {}
+    default_name = ""
+    drivername = ""
+    dsn = ""
+    default_user = ""
+    default_pwd = ""
+
+    def __init__(self, **kwargs):
+        cls = self.__class__
+        if not "uid" in kwargs and cls.default_user:
+            kwargs["uid"] = cls.default_user
+        if not "pwd" in kwargs and cls.default_pwd:
+            kwargs["pwd"] = cls.default_pwd
+        super(CustomDb, self).__init__(cls.dsn, **kwargs)
+
+    def connect(self, *args, **kwargs):
+        """ Establish the connexion to the database"""
+        logger.info("Connection to %s: %s", self.__class__.__name__, self.connectString)
+        super(CustomDb, self).connect(*args, **kwargs)
+
+    def read(self, sql, *args):
+        """ yield rows as NamedTupleRow """
+        cursor = self.execute(sql)
+        row = cursor.fetchone()
+        fieldnames = [(column[0] if column[0].isidentifier() else "field_{}".format(i)) for i, column in enumerate(cursor.description)]
+
+        rowmodel = namedtuple("Row", fieldnames)
+        while row:
+            yield rowmodel(*row)
+            row = cursor.fetchone()
+        cursor.close()
+
+    def read_all(self, sql, *args):
+        """ return the selection as a list of dictionnaries """
+        cursor = self.execute(sql)
+        fieldnames = [(column[0] if column[0].isidentifier() else "field_{}".format(i)) for i, column in enumerate(cursor.description)]
+
+        rowmodel = namedtuple("Row", fieldnames)
+        data = [rowmodel(*row) for row in cursor.fetchall()]
+        cursor.close()
+        return data
+
+    def first(self, sql, *args):
+        try:
+            return next(self.read(sql, *args))
+        except StopIteration:
+            return None
+
+    def exists(self, sql, *args):
+        """ return True if the sql command retrieves records """
+        return (self.first(sql, *args) is not None)
+
+    def execute(self, sql, *args):
+        cursor = self.cursor()
+        args = [sql, tuple(args)] if args else [sql]
+        try:
+            cursor.execute(*args)
+        except:
+            logger.debug(sql)
+            print(sql)
+            raise
+        return cursor
+
+
+class AccessDb(CustomDb):
+    dsn = "DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};FIL={MS Access};"
+    default_user = "admin"
+    default_pwd = ""
+    def __init__(self, dbpath, **kwargs):
+        super(AccessDb, self).__init__(dbq=dbpath, **kwargs)
+
+    def assert_connected(self):
+        for row in self.read("SELECT TOP 1 * FROM MSysObjects;"):
+            if not row:
+                raise AssertionError("Unable to connect to: {}".format(self.connectString))
+            return
+
+    @staticmethod
+    def format_date(dat, in_format="%Y-%m-%dT%H:%M:%S", out_format="%m/%d/%Y"):
+        return datetime.strptime(str(dat), in_format).strftime(out_format)
+
+    @staticmethod
+    def nz(val, default=""):
+        return val if val else default
+
+class AccessSDb(AccessDb):
+    dsn = "DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};FIL={MS Access};"
+    default_user = ""
+    default_pwd = ""
+    def __init__(self, dbpath, mdwpath, uid, pwd, **kwargs):
+        super(AccessSDb, self).__init__(dbpath, uid=uid, pwd=pwd, systemdb=mdwpath, **kwargs)
+
+class OracleDb(CustomDb):
+    dsn = "DRIVER={Oracle dans ORA102};"
+    def __init__(self, dbname, user, pwd, **kwargs):
+        super(OracleDb, self).__init__(dbq=dbname, uid=user, pwd=pwd, **kwargs)
+
+class SqlServerDb(CustomDb):
+    dsn = "DRIVER={SQL Server};"
+    def __init__(self, server, dbname, user, pwd, **kwargs):
+        super(SqlServerDb, self).__init__(server=server, database=dbname, uid=user, pwd=pwd, **kwargs)
+
+class PostgresDb(CustomDb):
+    dsn = "DRIVER={PostgreSQL Unicode};"
+    server = ""
+    db = ""
+    user = ""
+    pwd = ""
+    def __init__(self, server, dbname, user, pwd, **kwargs):
+        super(PostgresDb, self).__init__(server=server, database=dbname, uid=user, pwd=pwd, **kwargs)
+
+# class SqliteDb(CustomDb):
+#     drivername = "QODBC"
+#     dsn = "DRIVER={{Microsoft Access Driver (*.mdb, *.accdb)}};FIL={{MS Access}}"
+#     default_user = "admin"
+#     pwd = ""
+#     def __init__(self, dbpath, **kwargs):
+#         CustomDb.__init__(self, dbq=dbpath, **kwargs)
+#

core/logconf.py

@@ -0,0 +1,40 @@
+'''
+Created on 6 juil. 2017
+
+@author: olivier.massot
+'''
+import logging.config
+import sys
+import traceback
+
+import yaml
+
+from core.pde import DEFAULT_LOG_DIR, LOG_CONFIG_FILE
+
+
+SYS_EXCEPT_HOOK = sys.excepthook
+
+def start(name="main", level=0, filename=""):
+    # charge la configuration du logging depuis le fichier 'logging.yaml'
+    with open(LOG_CONFIG_FILE, 'rt') as f:
+        conf = yaml.load(f)
+
+    if level:
+        conf["loggers"][name]["level"] = level
+
+    if not filename:
+        filename = DEFAULT_LOG_DIR / r'{}.log'.format(name)
+    conf["handlers"]["file"]["filename"] = filename
+
+    conf["handlers"]["mail"]["fromaddr"] = conf["handlers"]["mail"]["fromaddr"].replace("%name%", name)
+    conf["handlers"]["mail"]["subject"] = conf["handlers"]["mail"]["subject"].replace("%name%", name)
+
+    logging.config.dictConfig(conf)
+
+    logger = logging.getLogger(name)
+    def _excepthook(typ, value, trace):
+        """ Remplace la gestion d'erreur standard, pour logger aussi les erreurs non gérées """
+        logger.error("{}\n{}\n{}".format(typ.__name__, value, ''.join(traceback.format_tb(trace))))
+        SYS_EXCEPT_HOOK(typ, value, trace)
+    sys.excepthook = _excepthook
+

core/mail.py

@@ -0,0 +1,79 @@
+'''
+
+usage:
+
+    m = Mail("mail-auto@bas-rhin.fr", ["destinataire@bas-rhin.fr"], "mon mail", "mon message")
+    m.send()
+
+'''
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import logging.handlers
+import smtplib
+
+
+class Mail():
+    HOST = ""
+
+    def __init__(self, fromaddr, toaddrs, subject, msg):
+        self.mailport = None
+        self.fromaddr = fromaddr
+        self.toaddrs = toaddrs
+        self.subject = subject
+        self.msg = msg
+
+        self.mail = MIMEMultipart('alternative')
+        self.mail['Subject'] = self.subject
+        self.mail['From'] = self.fromaddr
+        self.mail['To'] = ",".join(self.toaddrs)
+
+        self.mail.attach(MIMEText(self.msg, 'html', _charset='utf-8'))
+
+    def send(self):
+        if not self.HOST:
+            raise ValueError("Mail.HOST has to be defined before sending any mail")
+        
+        port = self.mailport if self.mailport else smtplib.SMTP_PORT
+
+        with smtplib.SMTP(self.HOST, port) as smtp:
+            smtp.sendmail(self.fromaddr, self.toaddrs, self.mail.as_string())
+
+
+class BufferingSMTPHandler(logging.handlers.BufferingHandler):
+    """ buffering email for logging """
+    def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity):
+        logging.handlers.BufferingHandler.__init__(self, capacity)
+        self.mailhost = mailhost
+        self.mailport = None
+        self.fromaddr = fromaddr
+        self.toaddrs = toaddrs
+        self.subject = subject
+
+    def flush(self):
+        try:
+            if len(self.buffer) > 0:
+                port = self.mailport if self.mailport else smtplib.SMTP_PORT
+
+                msg = "\n".join([self.format(record) for record in self.buffer])
+                msg = MIMEText(msg.encode('utf-8'), _charset='utf-8')
+                msg['Subject'] = self.subject
+                msg['From'] = self.fromaddr
+                msg['To'] = ",".join(self.toaddrs)
+
+                smtp = smtplib.SMTP(self.mailhost, port)
+                smtp.sendmail(self.fromaddr, self.toaddrs, msg.as_string())
+                smtp.quit()
+                self.buffer = []
+        except Exception as e:
+            print(e)
+            raise
+
+
+
+
+if __name__ == "__main__":
+    mail = Mail("",
+                ["olinox14@yahoo.fr"],
+                "test",
+                "test")
+    mail.send()

core/model.py

@@ -0,0 +1,114 @@
+'''
+
+
+    @author: olivier.massot, févr. 2018
+'''
+import csv
+from datetime import datetime
+import logging
+
+import dateutil.parser
+
+from core.sqlformatter import SqlFormatter
+
+
+logger = logging.getLogger("model")
+
+Sql = SqlFormatter()
+
+csv.register_dialect('tsv', delimiter='\t', quotechar='', quoting=csv.QUOTE_NONE)
+
+class Model():
+    _mapping = {}
+
+    def __setattr__(self, name, value):
+
+        try:
+            val, type_ = value
+            if not type_ in (int, float, bool, str, datetime):
+                raise TypeError("Type de donnée invalide pour une propriété de 'Model' ('{}')".format(type(val)))
+            self.__class__._mapping[name] = type_
+        except (TypeError, ValueError):
+            val = value
+            if val is None:
+                super().__setattr__(name, val)
+                return
+            try:
+                type_ = self.__class__._mapping[name]
+            except KeyError:
+                super().__setattr__(name, val)
+                return
+
+        val = Model._cast(val, type_)
+
+        super().__setattr__(name, val)
+
+    @staticmethod
+    def _cast(value, type_):
+        if value is None:
+            return value
+        if type_ == datetime:
+            if type(value) is str:
+                return dateutil.parser.parse(value)
+            elif type(value) is datetime:
+                return value
+            else:
+                raise ValueError("'{}' ne peut pas être converti en date".format(value))
+        if type_ is bool and value in ("True", "False"):
+            return (value == "True")
+        else:
+            return type_(value)
+
+    @property
+    def _fields(self):
+        return list(self.__dict__.keys())
+
+    @property
+    def data(self):
+        return self.__dict__
+
+    def __repr__(self):
+        return "<{} => {}>".format(self.__class__.__name__, ",".join(["{}={}".format(field, value) for field, value in self.__dict__.items()]))
+
+    @classmethod
+    def from_dict(cls, data):
+        """ Retourne un objet à partir d'un dictionnaire de données """
+        model = cls()
+        for key, value in data.items():
+            setattr(model, key, value)
+        return model
+
+    @classmethod
+    def _parse(cls, field, value):
+        if value == 'None':
+            value = None
+        try:
+            return Model._cast(value, cls._mapping[field])
+        except KeyError:
+            return value
+
+    # Fonctions CSV
+    def dump_to_csv(self, path):
+        """ Ajoute les données du modèle au format CSV dans le fichier spécifié en paramètre.
+        Créé le fichier s'il n'existe pas, avec une ligne d'en-tête """
+        if not path.exists():
+            logger.debug("Génère le fichier %s", path)
+            with open(path, 'w+', newline='') as f:
+                writer = csv.writer(f, 'tsv')
+                writer.writerow(self._fields)
+
+        with open(path, "a", newline='') as f:
+            writer = csv.writer(f, 'tsv')
+            writer.writerow([str(getattr(self, field)).replace("\t", " ") for field in self._fields])
+
+    @classmethod
+    def load_csv(cls, path):
+        """ parcourt les lignes du fichier csv et renvoie chaque ligne sous forme d'un objet Model
+        ATTENTION: chaque propriété dont le type n'est pas précisé dans _mapping aura le type 'string'
+        """
+        with open(path) as f:
+            reader = csv.reader(f, 'tsv')
+            fields = next(reader)
+            for row in csv.reader(f, 'tsv'):
+                data = {key: cls._parse(key, value) for key, value in zip(fields, row)}
+                yield(cls.from_dict(data))

core/sqlformatter.py

@@ -0,0 +1,34 @@
+'''
+
+    Formatter special pour les requetes SQL
+
+    usage:
+
+        Sql = SqlFormatter()
+
+        my_query = Sql.format("SELECT {} FROM {}", "my_field", "tblname")
+
+@author: olivier.massot
+'''
+import string
+
+
+class SqlFormatter(string.Formatter):
+
+    def format_field(self, value, format_spec):
+
+        if value is None:
+            return "Null"
+
+        if format_spec == "date":
+            return "#{:%Y-%m-%d %H:%M:%S}#".format(value)
+        elif format_spec == "text":
+            return "'{}'".format(SqlFormatter._escape(str(value)))
+        elif format_spec == "float":
+            return str(value).replace(",", ".")
+        else:
+            return SqlFormatter._escape(value.__format__(format_spec))
+
+    @staticmethod
+    def _escape(instr):
+            return instr.replace("'", "''").replace("\"", "''").replace("\t", " ")

logging.yaml

@@ -0,0 +1,98 @@
+version: 1
+disable_existing_loggers: no
+formatters:
+    simple:
+        format: "%(asctime)s - %(levelname)s - %(message)s"
+    complete:
+        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    short:
+        format: "%(levelname)s - %(message)s"
+    message_only:
+        format: "%(message)s"
+        
+handlers:
+    console:
+        class: logging.StreamHandler
+        level: INFO
+        formatter: message_only
+        stream: ext://sys.stdout
+    file:
+        class: logging.handlers.RotatingFileHandler
+        level: DEBUG
+        formatter: complete
+        filename: debug.log
+        maxBytes: 100000
+        backupCount: 1
+        encoding: utf8
+    mail:
+        class: core.BufferingSMTPHandler.BufferingSMTPHandler
+        level: INFO
+        formatter: complete
+        mailhost: smtp.bas-rhin.fr
+        fromaddr: log.%name%@bas-rhin.fr
+        toaddrs: [olivier.massot@bas-rhin.fr, jacky.klein@bas-rhin.fr]
+        subject: Rapport d'execution de %name%
+        capacity: 100000000
+
+loggers:
+    gf2analytique:
+        level: DEBUG
+        handlers: [console, file, mail]
+        propagate: no
+    wincan2ctrl:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    qgis_sync_compactage:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    qgis_sync_etancheite:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    qgis_sync_wincan:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    qgis_sync_video:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    qgis_sync_videores:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    ctrl2analytique:
+        level: DEBUG
+        handlers: [console, file, mail]
+        propagate: no
+    gf2factures:
+        level: DEBUG
+        handlers: [console, file, mail]
+        propagate: no
+    analytique2facture:
+        level: DEBUG
+        handlers: [console, file, mail]
+        propagate: no
+    pda2suiviactivite:
+        level: DEBUG
+        handlers: [console]
+        propagate: no
+    mails_rappel_ctrl:
+        level: DEBUG
+        handlers: [console, file,mail]
+        propagate: no
+    suiviactivite2pda:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+    agrhum_traitement:
+        level: DEBUG
+        handlers: [console, file]
+        propagate: no
+
+root:
+    level: DEBUG
+    handlers: [console]
+    propagate: yes

main.py

@@ -0,0 +1,6 @@
+'''
+Created on 10 sept. 2018
+
+@author: OMASSOT
+'''
+

requirements.txt

@@ -0,0 +1,10 @@
+pypyodbc
+path.py
+lxml
+python-dateutil
+pyyaml
+pyshp
+python-dateutil
+requests
+requests_ntlm
+docopt