Script gf2analytique ok

core/db.py

@@ -1,10 +1,12 @@
 '''
     Convenient access to various databases
 '''
+from datetime import datetime
 import logging
 
 from pypyodbc import Connection
 
+
 logger = logging.getLogger("database")
 
 class CustomDb(Connection):
@@ -70,6 +72,14 @@ class AccessDb(CustomDb):
                 raise AssertionError("Unable to connect to: {}".format(self.connectString))
             return
 
+    @staticmethod
+    def format_date(dat, in_format="%Y-%m-%dT%H:%M:%S", out_format="%m/%d/%Y"):
+        return datetime.strptime(str(dat), in_format).strftime(out_format)
+
+    @staticmethod
+    def nz(val, default=""):
+        return val if val else default
+
 class AccessSDb(AccessDb):
     dsn = "DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};FIL={MS Access};"
     default_user = ""

core/pde.py

@@ -9,7 +9,6 @@ from core.db import AccessSDb
 # Web url of the WsPde web service
 WSPDE_URL = "http://localhost:2890/public/WsPDE.asmx"
 
-
 MDW_PATH = r"\\h2o\local\4-transversal\BDD\mda\cg67Parc.mdw"
 UID = "olivier"
 PWD = "massot"

core/webservice.py

@@ -29,5 +29,5 @@ class GfWebservice():
     def __iter__(self):
         if self._data is None:
             self.parse()
-        return (elt.attrib for _, elt in etree.iterparse(BytesIO(self._data)))
+        return (elt.attrib for _, elt in etree.iterparse(BytesIO(self._data)) if elt.attrib)
 

gf2analytique.py

@@ -5,24 +5,20 @@ Created on 29 juin 2017
 '''
 from datetime import datetime
 import logging
-import random
 import re
-import sys
-import time
+
+from path import Path
 
 from core.pde import AnalytiqueDb
 from core.webservice import GfWebservice
 
-# TODO: vérifier le résultat des requêtes
+
 # TODO: proposer une méthode de correction des erreurs
 # TODO: configurer logging
 # TODO: envoi mail auto
-
 logger = logging.getLogger("factures")
-logging.basicConfig(level=logging.INFO)
-# logging.basicConfig(filename=r'log\factures_{:%Y%m%d_%H%M}.log'.format(datetime.now()),
-#                     level=logging.INFO)
-
+logging.basicConfig(filename=r'log\factures_{:%Y%m%d_%H%M}.log'.format(datetime.now()),
+                    level=logging.INFO)
 
 logger.info("Initialization")
 
@@ -32,162 +28,298 @@ analytique_db = AnalytiqueDb(autocommit=False)
 # Connect to the astre gf webservice
 ws = GfWebservice("GetPDEFactures")
 
-analysed, updated, errors = 0, 0, 0
-
-def shortuid():
-    """ 15 cars decimal uuid """
-    base = int(time.time()) << 32
-    rand = random.SystemRandom().getrandbits(32)
-    return hex((base + rand))[2:-1]
-
-def format_date(dat):
-    return datetime.strptime(str(dat)[:10], "%Y-%m-%d").strftime("%m/%d/%Y")
-
-def nz(val, default="Null"):
-    return val if val else default
-
-def is_valid(facture, prompt_for_correction=False):
-    """ controle la validité des données d'une facture """
-    if facture["codeAxe"] == "ENGIN":
-        # Controle l'existence du materiel
-        if not analytique_db.first("SELECT intlMaterielID FROM tbl_materiel WHERE txtMateriel='{}'".format(facture["codeCout"])):
-            logger.warning("Le materiel n'existe pas: %s", facture["codeCout"])
-            if prompt_for_correction:
-                facture["codeCout"] = input("> txtMateriel: ")
-                return is_valid(facture, True)
-            else:
-                return False
-    elif facture["codeAxe"] == "AFFAI":
-        # Controle l'existence de l'affaire
-        if not analytique_db.first("SELECT dblAffaireId FROM tbl_Affaires WHERE strLiaisonControle='{}'".format(facture["codeCout"])):
-            logger.warning("L'affaire n'existe pas: %s", facture["codeCout"])
-            if prompt_for_correction:
-                facture["codeCout"] = input("> strLiaisonControle: ")
-                return is_valid(facture, True)
-            else:
-                return False
-    else:
-        # CodeAxe invalide
-        logger.warning("Code axe inconnu: %s", facture["codeAxe"])
-        return False
-    return True
-
-def send(facture):
-    """ ventile les données dans les tables de la base AnalytiqueDb """
-    uid = shortuid()
-
-    sql = """INSERT INTO tbl_Factures ( intExercice, strEnveloppe, strEngagement, strLiquidation, strObjet, strTiers,
-          strTiersLibelle, strMotsClefs, intOperation, dtmDeb, strNomenclature0, dblMontantTVA,
-          dblMontantTotal, strService, strUidImport, strORIGINE_DONNEES )
-          VALUES ({intExercice}, '{strEnveloppe}', '{strEngagement}', '{strLiquidation}', '{strObjet}','{strTiers}',
-          '{strTiersLibelle}', '{strMotsClefs}', {intOperation}, #{dtmDeb}#, '{strNomenclature0}',{dblMontantTVA},
-          {dblMontantTotal}, '{strService}', '{strUidImport}', '{strORIGINE_DONNEES}')
-          """.format(
-                     intExercice=facture["numExBudget"],
-                     strEnveloppe=facture["numEnv"],
-                     strEngagement=facture["numMandat"],
-                     strLiquidation=facture["numLiqMandat"],
-                     strObjet=nz(facture["libCout"]),
-                     strTiers=facture["numTiers"],
-                     strTiersLibelle=facture["libRai"],
-                     strMotsClefs=nz(facture["refIntMandat"]),
-                     intOperation=nz(facture["codePeriode"]),
-                     dtmDeb=format_date(facture["dateDepDelai"]),
-                     strNomenclature0=facture["typeNomencMarche"],
-                     dblMontantTVA=facture["mntTvaMandat"],
-                     dblMontantTotal=facture["mntVent"],
-                     strService='7710',
-                     strUidImport=uid,
-                     strORIGINE_DONNEES='ASTRE'
-                     )
-    logger.debug("> %s", sql)
-    analytique_db.execute(sql)
-
-    factureId = analytique_db.first("SELECT dblFactureId FROM tbl_Factures WHERE [strUidImport]='{}'".format(uid))["dblfactureid"]
-    logger.debug("retrieve dblFactureId: %s", factureId)
-
-    if facture["codeAxe"] == "ENGIN":
-
-        materiel = analytique_db.first("SELECT intlMaterielID FROM tbl_Materiel WHERE [txtMateriel]='{}'".format(facture["codeCout"]))
-        materielId = materiel["intlmaterielid"] if materiel else '859'
+workdir = Path(r".\work")
+workdir.mkdir_p()
+workdir /= "gf2analytique"
+workdir.mkdir_p()
 
-        logger.debug("retrieve intlMaterielID: %s", materielId)
+errfile = workdir / "err.csv"
 
-        sql = """INSERT INTO tbl_Facture_Engin ( dblFactureId, txtMateriel, strLibelle, strType, dblMontant, intlMaterielID )
-                VALUES ({}, '{}', '{}', '{}', {}, {})
-        """.format(factureId,
-                   facture["codeCout"],
-                   nz(facture["libCout"]),
-                   facture["libRai"],
-                   facture["mntVent"],
-                   materielId
-                   )
-
-    elif facture["codeAxe"] == "AFFAI":
-
-        sql = """INSERT INTO tbl_Facture_Affaire ( dblFactureId, strAffaireId, strType, strLibelle, dblMontant )
-              VALUES ({}, '{}', '{}', '{}', {})
-              """.format(factureId,
-                   facture["codeCout"],
-                   facture["libCout"],
-                   facture["libRai"],
-                   facture["mntVent"]
-                   )
-
-    logger.debug("> %s", sql)
-    analytique_db.execute(sql)
+class AlreadyImported(Exception):
+    pass
 
-    sql = """INSERT INTO tbl_Mandatement ( dblFacture, strNumMandat, dtmMandat, strBordereau )
-          VALUES ({}, '{}', #{}#, '{}')
-          """.format(factureId,
-                   facture["numMandat"],
-                   format_date(facture["dateMandat"]),
-                   facture["numBj"]
-                   )
+class NotImported(Exception):
+    pass
 
-    logger.debug("> %s", sql)
-    analytique_db.execute(sql)
-    analytique_db.commit()
+class InvalidData(Exception):
+    pass
 
-    logger.info("* imported: %s", factureId)
+class InvalidAxe(Exception):
+    pass
 
+class Facture():
+    def __init__(self):
+        self._factureId = None
+        self.numExBudget = None
+        self.codeColl = None
+        self.codeBudg = None
+        self.numEnv = None
+        self.codeSection = None
+        self.typeMvt = None
+        self.numMandat = None
+        self.numLiqMandat = None
+        self.numLigneMandat = None
+        self.codeAxe = None
+        self.libAxe = None
+        self.codeCout = None
+        self.libCout = None
+        self.dateMandat = None
+        self.numBj = None
+        self.numTiers = None
+        self.libRai = None
+        self.refIntMandat = None
+        self.codePeriode = None
+        self.dateDepDelai = None
+        self.typeNomencMarche = None
+        self.mntTtcMandat = None
+        self.mntTvaMandat = None
+        self.mntVent = None
+
+    @property
+    def factureId(self):
+        if self._factureId is None:
+            try:
+                self._factureId = self._get_facture_id()
+            except (KeyError, AttributeError, TypeError):
+                raise NotImported()
+        return self._factureId
+
+    @classmethod
+    def from_webservice(cls, wsdata):
+        facture = cls()
+        for key, value in wsdata.items():
+            facture.__dict__[key] = value
+        facture.autocorrection()
+        return facture
+
+    def is_imported(self):
+        try:
+            return self.factureId > 0
+        except NotImported:
+            return False
+
+    def autocorrection(self):
+        # correction auto des codes chantiers
+        if re.match(r"\d{2}5\d{3}", self.codeCout):
+            self.codeCout += "/1"
+        self.libRai = self.libRai.replace("'", "''")
+
+    def is_valid(self):
+        """ controle la validité des données d'une facture """
+        if not int(self.numExBudget) > 2000:
+            logger.warning("Exercice budgetaire invalide: %s", self.numExBudget)
+            return False
+        if self.codeColl != "CG67":
+            logger.warning("Code collectivité invalide: %s", self.codeColl)
+            return False
+        if self.codeBudg != "02":
+            logger.warning("Code budgetaire invalide: %s", self.codeBudg)
+            return False
+        if self.codeAxe == "ENGIN":
+            # Controle l'existence du materiel
+            if not analytique_db.first("SELECT intlMaterielID FROM tbl_materiel WHERE txtMateriel='{}'".format(self.codeCout)):
+                logger.warning("Le materiel n'existe pas: %s", self.codeCout)
+                return False
+        elif self.codeAxe == "AFFAI":
+            # Controle l'existence de l'affaire
+            if not analytique_db.first("SELECT dblAffaireId FROM tbl_Affaires WHERE strLiaisonControle='{}'".format(self.codeCout)):
+                logger.warning("L'affaire n'existe pas: %s", self.codeCout)
+                return False
+        else:
+            # CodeAxe invalide
+            logger.warning("Code axe inconnu: %s", self.codeAxe)
+            return False
+        return True
+
+    def send_to_db(self):
+
+        if self.is_imported():
+            raise AlreadyImported()
+
+        if not self.is_valid():
+            raise InvalidData()
+
+        self._insert_factures()
+
+        if self.codeAxe == "ENGIN":
+            self._insert_factures_engins()
+
+        elif self.codeAxe == "AFFAI":
+            self._insert_factures_affaires()
+
+        self._insert_mandatement()
+
+        analytique_db.commit()
+
+        logger.info("* imported: %s", self.factureId)
+
+    def _get_facture_id(self):
+        sql = """SELECT dblFactureId FROM tbl_Factures
+                    WHERE intExercice = {}
+                    AND strEnveloppe = '{}'
+                    AND strLiquidation = '{}'
+                    AND strAxe = '{}'
+                    AND
+                      ((intLiquidationLigne = {} AND strCentreCout='{}')
+                      OR (intLiquidationLigne IS NULL))
+                """.format(self.numExBudget,
+                           self.numEnv,
+                           self.numLiqMandat,
+                           self.codeAxe,
+                           self.numLigneMandat,
+                           self.codeCout)
+
+#         logger.debug("> %s", sql)
+        factureId = analytique_db.first(sql)["dblfactureid"]
+#         logger.debug("retrieve dblFactureId: %s", factureId)
+        return factureId
+
+    def _insert_factures(self):
+        sql = """INSERT INTO tbl_Factures ( intExercice,
+                                            strLiquidation,
+                                            intLiquidationLigne,
+                                            strEngagement,
+                                            strEnveloppe,
+                                            strService,
+                                            strTiers,
+                                            strTiersLibelle,
+                                            strMotsClefs,
+                                            dtmDeb,
+                                            intOperation,
+                                            strNomenclature0,
+                                            strAXE,
+                                            strCentreCout,
+                                            strObjet,
+                                            dblMontantTotal,
+                                            dblMontantTVA,
+                                            strORIGINE_DONNEES
+                                        )
+                  VALUES ({intExercice},
+                          '{strLiquidation}',
+                          {intLiquidationLigne},
+                          '{strEngagement}',
+                          '{strEnveloppe}',
+                          '{strService}',
+                          '{strTiers}',
+                          '{strTiersLibelle}',
+                          '{strMotsClefs}',
+                          #{dtmDeb}#,
+                          {intOperation},
+                          '{strNomenclature0}',
+                          '{strAxe}',
+                          '{strCentreCout}',
+                          '{strObjet}',
+                          {dblMontantTotal},
+                          {dblMontantTVA},
+                          '{strORIGINE_DONNEES}'
+                          )
+              """.format(
+                         intExercice=self.numExBudget,
+                         strLiquidation=self.numLiqMandat,
+                         intLiquidationLigne=self.numLigneMandat,
+                         strEngagement=self.numMandat,
+                         strEnveloppe=self.numEnv,
+                         strService='7710',
+                         strTiers=self.numTiers,
+                         strTiersLibelle=self.libRai,
+                         strMotsClefs=AnalytiqueDb.nz(self.refIntMandat),
+                         dtmDeb=AnalytiqueDb.format_date(self.dateDepDelai),
+                         intOperation=AnalytiqueDb.nz(self.codePeriode, "Null"),
+                         strNomenclature0=self.typeNomencMarche,
+                         strAxe=self.codeAxe,
+                         strCentreCout=self.codeCout,
+                         strObjet=AnalytiqueDb.format_date(self.dateMandat, out_format="%d/%m/%Y"),
+                         dblMontantTVA=self.mntTvaMandat,
+                         dblMontantTotal=self.mntVent,
+                         strORIGINE_DONNEES='ASTRE'
+                         )
+        logger.debug("> %s", sql)
+        analytique_db.execute(sql)
+
+    def _insert_factures_engins(self):
+        if self.codeAxe != "ENGIN":
+            raise InvalidAxe()
+
+        materiel = analytique_db.first("SELECT intlMaterielID FROM tbl_Materiel WHERE [txtMateriel]='{}'".format(self.codeCout))
+        materielId = materiel["intlmaterielid"] if materiel else '859'
+        logger.debug("retrieve intlMaterielID: %s", materielId)
 
-def freeze(facture):
-    pass
+        sql = """INSERT INTO tbl_Facture_Engin ( intlMaterielID, txtMateriel, dblFactureId, strLibelle, dblMontant, strType )
+                VALUES ({}, '{}', {}, '{}', {}, '{}')
+        """.format(materielId,
+                   self.codeCout,
+                   self.factureId,
+                   AnalytiqueDb.nz(self.libCout),
+                   self.mntVent,
+                   self.libRai
+                   )
+        logger.debug("> %s", sql)
+        analytique_db.execute(sql)
+
+    def _insert_factures_affaires(self):
+        if self.codeAxe != "AFFAI":
+            raise InvalidAxe()
+
+        sql = """INSERT INTO tbl_Facture_Affaire ( strAffaireId, dblFactureId, strLibelle, dblMontant, strType )
+              VALUES ('{}', {}, '{}', {}, '{}')
+              """.format(self.codeCout,
+                         self.factureId,
+                         self.libRai ,
+                         self.mntVent,
+                         AnalytiqueDb.nz(self.libCout),
+                         )
+        logger.debug("> %s", sql)
+        analytique_db.execute(sql)
+
+    def _insert_mandatement(self):
+        sql = """INSERT INTO tbl_Mandatement ( dblFacture, strNumMandat, dtmMandat, strBordereau )
+              VALUES ({}, '{}', #{}#, '{}')
+              """.format(self.factureId,
+                         self.numMandat,
+                         AnalytiqueDb.format_date(self.dateMandat),
+                         self.numBj
+                       )
+        logger.debug("> %s", sql)
+        analytique_db.execute(sql)
+
+    def dump_to_err(self):
+
+        fields = ["numExBudget", "numEnv", "numLiqMandat", "codeAxe", "codeCout"]
+
+        try:
+            with open(errfile, 'r') as f:
+                content = f.read()
+        except FileNotFoundError:
+            content = ""
+        if not content:
+            line = "\t".join(fields)
+            with open(errfile, 'a') as f:
+                f.writelines([line])
+
+        line = "\t".join([getattr(self, field) for field in fields])
+
+        with open(errfile, 'a') as f:
+            f.writelines([line])
 
 ########
 
 if __name__ == "__main__":
 
-    prompt_for_correction = "-c" in sys.argv
-
-    for facture in ws:
-        if not facture:
-            continue
 
+    analysed, updated, errors = 0, 0, 0
+    for wsdata in ws:
         analysed += 1
 
-        record = analytique_db.first("""SELECT dblFactureId
-                                        FROM tbl_Factures
-                                        WHERE strService='7710'
-                                        AND intExercice={}
-                                        AND strLiquidation='{}'
-                                        """.format(facture["numExBudget"], facture["numLiqMandat"]))
-        if record:
-            # already imported
-            continue
+        facture = Facture.from_webservice(wsdata)
 
-        # correction auto des codes chantiers
-        if re.match(r"\d{2}5\d{3}", facture["codeCout"]):
-            facture["codeCout"] += "/1"
-
-        if not is_valid(facture, prompt_for_correction):
-            logger.warning("INVALID DATA: numEnv= %s", facture["numEnv"])
+        try:
+            facture.send_to_db()
+            updated += 1
+        except AlreadyImported:
+            pass
+        except InvalidData:
+            facture.dump_to_err()
             errors += 1
-            continue
-
-        send(facture)
-        updated += 1
 
     logger.info("Import terminé: {} lignes traitées / {} importées / {} erreurs".format(analysed, updated, errors))