gf2analytique: log et traitement des erreurs ok

.gitignore

@@ -3,4 +3,5 @@
 .pydevproject
 Output/
 htmlcov/
-*.log
+*.log
+work/*

core/logconf.py

@@ -0,0 +1,26 @@
+'''
+Created on 6 juil. 2017
+
+@author: olivier.massot
+'''
+from datetime import datetime
+import logging.config
+
+from path import Path
+import yaml
+
+def start(name="main", level=0, filename=""):
+    # charge la configuration du logging depuis le fichier 'logging.yaml'
+    configfile = Path(__file__).parent
+    with open(configfile / 'logging.yaml', 'rt') as f:
+        conf = yaml.load(f)
+
+    if level:
+        conf["loggers"][name]["level"] = level
+
+    if not filename:
+        filename = r'log\{}_{:%Y%m%d_%H%M}.log'.format(name, datetime.now())
+    conf["handlers"]["file"]["filename"] = filename
+
+    logging.config.dictConfig(conf)
+

core/logging.yaml

@@ -0,0 +1,63 @@
+version: 1
+disable_existing_loggers: no
+formatters:
+    simple:
+        format: "%(asctime)s - %(levelname)s - %(message)s"
+    complete:
+        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    short:
+        format: "%(levelname)s - %(message)s"
+    message_only:
+        format: "%(message)s"
+        
+handlers:
+    console:
+        class: logging.StreamHandler
+        level: INFO
+        formatter: message_only
+        stream: ext://sys.stdout
+
+    file:
+        class: logging.handlers.RotatingFileHandler
+        level: DEBUG
+        formatter: complete
+        filename: debug.log
+        maxBytes: 100000
+        backupCount: 1
+        encoding: utf8
+    mailbuffer:
+        class: logging.handlers.MemoryHandler
+        level: INFO
+        formatter: complete
+        capacity: 100000
+        flushLevel: CRITICAL
+        target: mail
+    mail:
+        class: logging.handlers.SMTPHandler
+        level: INFO
+        formatter: complete
+        mailhost: https://webmail.cg67.fr/owa/
+        fromaddr: olivier.massot@bas-rhin.fr
+        toaddrs: [olivier.massot@bas-rhin.fr]
+        subject: log
+        
+loggers:
+    gf2analytique:
+        level: INFO
+        handlers: [console, file]
+        propagate: no
+
+    gf2factures:
+        level: INFO
+        handlers: [console, file]
+        propagate: no
+    
+    test:
+        level: INFO
+        handlers: [console, mailbuffer]
+        propagate: no   
+        
+root:
+    level: INFO
+    handlers: [console, file]
+    propagate: yes

core/pde.py

@@ -13,6 +13,15 @@ MDW_PATH = r"\\h2o\local\4-transversal\BDD\mda\cg67Parc.mdw"
 UID = "olivier"
 PWD = "massot"
 
+# Dirs
+WRK = Path(r".\work")
+
+def mk_workdir(name):
+    WRK.mkdir_p()
+    workdir = WRK / name
+    workdir.mkdir_p()
+    return workdir
+
 # DB_DIRPATH = Path(r"\\h2o\local\4-transversal\BDD\mdb")
 DB_DIRPATH = Path(r"C:\wrktmp\mdb")
 

gf2analytique.py

@@ -1,24 +1,31 @@
 '''
-Created on 29 juin 2017
+    Script d'import des données de facturation depuis
+    la base de données ASTRE-GF vers les tables de la base Analytique
+    du Parc Départemental d'Erstein
+
+    En cas d'erreur avec les données importées:
+    1. une tentative d'autocorrection est effectuée
+    2. Si les données sont toujours invalides, une ligne est ajoutée au fichier .\\work\\gf2analytique\\err.csv
+       pour une correction manuelle.
+
+    IMPORTANT: Si le fichier 'err.csv' contient des lignes, le script tentera d'importer
+               ces lignes à la place de celles issues de Astre Gf.
+               Pour forcer un imprt depuis AstreGf, supprimez le fichier 'err.csv'
+
+    Info: Les données sont obtenues via le web service CG67.AstreGf
 
-@author: olivier.massot
 '''
-from datetime import datetime
 import logging
 import re
 
-from path import Path
-
-from core.pde import AnalytiqueDb
+from core import logconf
+from core.pde import AnalytiqueDb, mk_workdir
 from core.webservice import GfWebservice
 
-
-# TODO: proposer une méthode de correction des erreurs
-# TODO: configurer logging
 # TODO: envoi mail auto
-logger = logging.getLogger("factures")
-logging.basicConfig(filename=r'log\factures_{:%Y%m%d_%H%M}.log'.format(datetime.now()),
-                    level=logging.INFO)
+
+logger = logging.getLogger("gf2analytique")
+logconf.start("gf2analytique", logging.INFO)
 
 logger.info("Initialization")
 
@@ -28,11 +35,8 @@ analytique_db = AnalytiqueDb(autocommit=False)
 # Connect to the astre gf webservice
 ws = GfWebservice("GetPDEFactures")
 
-workdir = Path(r".\work")
-workdir.mkdir_p()
-workdir /= "gf2analytique"
-workdir.mkdir_p()
-
+# Make the working directory
+workdir = mk_workdir("gf2analytique")
 errfile = workdir / "err.csv"
 
 class AlreadyImported(Exception):
@@ -48,32 +52,15 @@ class InvalidAxe(Exception):
     pass
 
 class Facture():
+    WS_FIELDS = ["numExBudget", "codeColl", "codeBudg", "numEnv", "codeSection", "typeMvt", "numMandat", "numLiqMandat",
+                 "numLigneMandat", "codeAxe", "libAxe", "codeCout", "libCout", "dateMandat", "numBj", "numTiers",
+                 "libRai", "refIntMandat", "codePeriode", "dateDepDelai", "typeNomencMarche", "mntTtcMandat",
+                 "mntTvaMandat", "mntVent"]
     def __init__(self):
         self._factureId = None
-        self.numExBudget = None
-        self.codeColl = None
-        self.codeBudg = None
-        self.numEnv = None
-        self.codeSection = None
-        self.typeMvt = None
-        self.numMandat = None
-        self.numLiqMandat = None
-        self.numLigneMandat = None
-        self.codeAxe = None
-        self.libAxe = None
-        self.codeCout = None
-        self.libCout = None
-        self.dateMandat = None
-        self.numBj = None
-        self.numTiers = None
-        self.libRai = None
-        self.refIntMandat = None
-        self.codePeriode = None
-        self.dateDepDelai = None
-        self.typeNomencMarche = None
-        self.mntTtcMandat = None
-        self.mntTvaMandat = None
-        self.mntVent = None
+
+        for fld in self.WS_FIELDS:
+            setattr(self, fld, None)
 
     @property
     def factureId(self):
@@ -88,22 +75,58 @@ class Facture():
     def from_webservice(cls, wsdata):
         facture = cls()
         for key, value in wsdata.items():
-            facture.__dict__[key] = value
+            setattr(facture, key, value)
+#             facture.__dict__[key] = value
         facture.autocorrection()
         return facture
 
+    @classmethod
+    def from_errfile(cls, line):
+        return cls.from_webservice(dict(zip(cls.WS_FIELDS, line.split("\t"))))
+
     def is_imported(self):
         try:
             return self.factureId > 0
         except NotImported:
             return False
 
+    def _init_errfile(self):
+        try:
+            with open(errfile, 'r') as f:
+                if f.read(100):
+                    # File already exists and is not empty
+                    return
+        except FileNotFoundError:
+            pass
+
+        firstline = "\t".join(self.WS_FIELDS + ["\n"])
+        with open(errfile, 'a') as f:
+            f.write(firstline)
+
+    def dump_to_err(self):
+        self._init_errfile()
+
+        line = "\t".join([str(getattr(self, field)).replace("\t", " ") for field in self.WS_FIELDS] + ["\n"])
+
+        with open(errfile, 'a') as f:
+            f.write(line)
+
     def autocorrection(self):
         # correction auto des codes chantiers
-        if re.match(r"\d{2}5\d{3}", self.codeCout):
+        if self.codeAxe == "AFFAI" and re.match(r"\d{2}5\d{3}", self.codeCout):
             self.codeCout += "/1"
+
+        # echappe les apostrophes
         self.libRai = self.libRai.replace("'", "''")
 
+        # renomme automatiquement les noms de materiels
+        if self.codeAxe == "ENGIN":
+            row = analytique_db.first("""SELECT txtMateriel FROM tbl_materiel
+                                            WHERE txtMateriel='{codeCout}' or txtMateriel='ZZ {codeCout}'
+                                            """.format(codeCout=self.codeCout))
+            if row:
+                self.codeCout = row["txtmateriel"]
+
     def is_valid(self):
         """ controle la validité des données d'une facture """
         if not int(self.numExBudget) > 2000:
@@ -282,44 +305,56 @@ class Facture():
         logger.debug("> %s", sql)
         analytique_db.execute(sql)
 
-    def dump_to_err(self):
-
-        fields = ["numExBudget", "numEnv", "numLiqMandat", "codeAxe", "codeCout"]
-
+    @staticmethod
+    def load_errfile_data():
+        factures = []
         try:
-            with open(errfile, 'r') as f:
-                content = f.read()
+            firstline = True
+            with open(errfile, "r") as f:
+                for line in f:
+                    if firstline:
+                        firstline = False
+                        continue
+                    facture = Facture.from_errfile(line)
+                    factures.append(facture)
+
         except FileNotFoundError:
-            content = ""
-        if not content:
-            line = "\t".join(fields)
-            with open(errfile, 'a') as f:
-                f.writelines([line])
+            pass
+        return factures
 
-        line = "\t".join([getattr(self, field) for field in fields])
+    @staticmethod
+    def process(factures):
+        analysed, updated, errors = 0, 0, 0
+
+        for facture in factures:
+            analysed += 1
+            try:
+                facture.send_to_db()
+                updated += 1
+            except AlreadyImported:
+                pass
+            except InvalidData:
+                facture.dump_to_err()
+                errors += 1
+        return analysed, updated, errors
 
-        with open(errfile, 'a') as f:
-            f.writelines([line])
 
 ########
 
 if __name__ == "__main__":
 
+    to_retry = Facture.load_errfile_data()
+    errfile.remove_p()
 
-    analysed, updated, errors = 0, 0, 0
-    for wsdata in ws:
-        analysed += 1
-
-        facture = Facture.from_webservice(wsdata)
+    if to_retry:
+        logger.info("# Ré-import depuis le fichier d'erreurs")
+        logger.info("{} lignes chargées depuis {}".format(len(to_retry), errfile))
+        res = Facture.process(to_retry)
+        logger.info("> {} lignes traitées / {} importées / {} erreurs".format(res[0], res[1], res[2]))
 
-        try:
-            facture.send_to_db()
-            updated += 1
-        except AlreadyImported:
-            pass
-        except InvalidData:
-            facture.dump_to_err()
-            errors += 1
+    else:
+        logger.info("# Import depuis Astre-Gf")
+        res = Facture.process([Facture.from_webservice(wsdata) for wsdata in ws])
+        logger.info("> {} lignes traitées / {} importées / {} erreurs".format(res[0], res[1], res[2]))
 
-    logger.info("Import terminé: {} lignes traitées / {} importées / {} erreurs".format(analysed, updated, errors))
 

gf2factures.py

@@ -6,14 +6,13 @@ Created on 27 juin 2017
 from datetime import datetime
 import logging
 
+from core import logconf
 from core.pde import FacturesDb
 from core.webservice import GfWebservice
 
 
-logger = logging.getLogger("factures")
-logging.basicConfig(filename=r'log\factures_{:%Y%m%d_%H%M}.log'.format(datetime.now()),
-                    level=logging.INFO)
-
+logger = logging.getLogger("gf2factures")
+logconf.start("gf2factures", logging.INFO)
 
 logger.info("Initialization")