Refonte (suite)

core/db.py

@@ -122,6 +122,7 @@ class PostgresDb(CustomDb):
     db = ""
     user = ""
     pwd = ""
+    port = ""
     def __init__(self, server, dbname, user, pwd, **kwargs):
         super(PostgresDb, self).__init__(server=server, database=dbname, uid=user, pwd=pwd, **kwargs)
 

core/gis.py

@@ -4,34 +4,40 @@
 '''
 import shapefile
 
-SHAPES = {1:"Point", 3:"Polyligne", 5:"Polygone"}
+POINT = 1
+POLYLINE = 3
+POLYGON = 5
+SHAPE_NAMES = {1: "POINT", 3:"POLYLIGNE", 5:"POLYGONE"}
 
 class ShapeError(IOError): pass
 
 class ShapeFile():
-    shapes = {1: "POINT", 3:"POLYLIGNE", 5:"POLYGONE"}
+    def __init__(self, path_):
+        self.path_ = path_
     
-    def __enter__(self, path_):
-        if not path_.isfile():
+    def __enter__(self):
+        if not self.path_.isfile():
             raise FileNotFoundError("Fichier introuvable")
      
         try:
-            self.sf = shapefile.Reader(path_)
+            self.sf = shapefile.Reader(self.path_)
         except shapefile.ShapefileException:
             raise ShapeError("Fichier Shape illisible")
         
-        try:
-            _ = self.shapes[self.sf.shapeType]
-        except KeyError:
+        if not self.sf.shapeType in (POINT, POLYLINE, POLYGON):
             raise ShapeError("Type de géométrie non reconnue")
 
         return self
     
+    def shape(self):
+        return self.sf.shapeType
+    
     def fields(self):
         return [f[0] for f in self.sf.fields if f[0] != 'DeletionFlag']
         
     def records(self):
-        return self.sf.shapeRecords()
+        for record in self.sf.shapeRecords():
+            yield record
         
-    def __exit__(self):
+    def __exit__(self, exc_type, exc_val, exc_tb):
         del self.sf

core/mn.py

@@ -0,0 +1,14 @@
+'''
+
+@author: olivier.massot, 2018
+'''
+from core.db import PostgresDb
+
+class ReferentielDb(PostgresDb):
+    server = "clusterpg.linux.infra.cloud.local"
+    port="5432"
+    db = "sig50"
+    user = "sigr"
+    pwd = "T38Msh2R4q"
+    def __init__(self, **kwargs):
+        super(ReferentielDb, self).__init__(server=self.server, dbname=self.db, user=self.user, pwd=self.pwd, **kwargs)

core/validation.py

@@ -3,18 +3,57 @@
 
     @author: olivier.massot, sept. 2018
 '''
+from datetime import datetime
 import zipfile
 
+from cerberus.validator import Validator
 from path import Path, TempDir
 
+from core.validation_errors import MissingFile, FormatError
+
+
+class BaseModel():
+    index = {}
+    pk = ""
+    schema = {}
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+        
+    @classmethod
+    def indexer(cls, instance):
+        if instance.getitem(cls.pk) in cls.index:
+            raise ValueError("Duplicate PK")
+        cls.index[instance.getitem(cls.pk)]
+        
+class BaseGeoModel(BaseModel):
+    def __init__(self, geom, **kwargs):
+        super(BaseGeoModel, self).__init__(**kwargs)
+        self.geom = geom
+
+
+
 class ValidationReport():
     def __init__(self, title = ""):
         self.title = title
         self.errors = {}
-        self.valid = True
-        
+    
+    @property
+    def valid(self):
+        return len(self.error) == 0
+
+
+
+
+def is_french_date(field, value, error):
+    try:
+        datetime.strptime(value, '%d/%m/%Y')
+    except:
+        error(field, 'Doit être une date au format jj/mm/aaaa')
+
 class BaseValidator():
     
+    FILES = {}
+    
     @classmethod
     def submit(cls, subject):    
         """ prends un dossier ou une archive en entrée et vérifie son contenu  """
@@ -36,4 +75,72 @@ class BaseValidator():
 
     @classmethod
     def _submit_folder(cls, folder):
-        raise NotImplementedError()
+        
+        dataset = {}
+        report = ValidationReport("Contrôle des données de {} au format {}".format(folder.name, cls.name))
+        
+        # Charge les données en mémoire
+        for filename, model in cls.files.items():
+            path_ = Path(folder) / filename
+            
+            if not path_.isfile():
+                report.errors[MissingFile] = MissingFile("Le fichier '{}' est manquant".format(filename))
+                continue
+            
+            dataset[model] = cls._load_file(model, path_)
+        
+        # Controle la structure des données (champs, formats et types)
+        cls._structure_validation(dataset)
+        
+        # Contrôle la géométrie (optionnel)
+        cls._geometry_validation(dataset)
+        
+        # Validation technique
+        cls._technical_validation(dataset)
+    
+        return report
+    
+    @classmethod
+    def _load_file(cls, model, filename):
+        """ Charge les données du fichier et les associe à un modèle.
+        Attention: pas de contrôle de format o de validité à ce niveau!  """
+        raise NotImplementedError()
+    
+    @classmethod        
+    def _structure_validation(cls, dataset):
+        
+        errors = {}
+        errors[FormatError] = []
+        
+        for model in dataset:
+            v = Validator(model.schema)
+            
+            for item in dataset[model]:
+                v.validate(item.__dict__)
+                
+                for fieldname, verrors in v.errors.items():
+                    for err in verrors:
+                        errors[FormatError].append(FormatError("{}: {}".format(fieldname, err)))
+        
+        return errors
+    
+    @classmethod
+    def _geometry_validation(cls, dataset):
+        pass
+    
+    @classmethod
+    def _organize_dataset(cls, dataset):
+        raise NotImplementedError()
+    
+    @classmethod
+    def _technical_validation(cls, dataset):
+        raise NotImplementedError()
+    
+    
+    
+        
+    
+    
+    
+    
+    

core/validation_errors.py

@@ -10,6 +10,34 @@ WARNING = 20
 ERROR = 30
 CRITICAL = 40
 
+### Vérifications générales
+
+# Fichiers présents
+# Fichiers lisibles
+# SRID
+
+#### Schéma de données
+
+# Présence des champs
+# Type de données
+# Données obligatoires
+# Valeurs autorisées
+# Emprise géographique
+# Type de géométrie
+
+####
+
+# Unicité des codes
+# Vérification de la présence dans la base des code existants, et si exsitence, vérifie s'il s'agit du même
+# Vérification des cardinalités
+
+
+
+
+
+
+
+
 
 
 class BaseValidationError():
@@ -18,40 +46,39 @@ class BaseValidationError():
     def __init__(self, message):
         self.message = message
         
-        
 #### Erreurs Critiques
 
-class MissingFile():
+class MissingFile(BaseValidationError):
     level = CRITICAL
     name = "Fichier Manquant"
     
-class UnreadableFile():
+class UnreadableFile(BaseValidationError):
     level = CRITICAL
     name = "Fichier Illisible"
 
-class WrongSrid():
+class WrongSrid(BaseValidationError):
     level = CRITICAL
     name = "Mauvais SRID"
 
-class FieldTypeError():
+class FieldTypeError(BaseValidationError):
     level = CRITICAL
     name = "Le champs n'a pas le bon type de donnée"
     
 #### Erreurs
 
-class OutOfBoxCoordinates():
+class OutOfBoxCoordinates(BaseValidationError):
     level = ERROR
     name = "Coordonnées hors de la zone autorisée"
 
-class FormatError():
+class FormatError(BaseValidationError):
     level = ERROR
     name = "Le champs n'a pas le bon format"
 
-class MissingValueError():
+class MissingValueError(BaseValidationError):
     level = ERROR
     name = "Le champs doit être renseigné"
     
-class FieldValueError():
+class FieldValueError(BaseValidationError):
     level = ERROR
     name = "La valeur du champs est invalide"
 

requirements.txt

@@ -9,4 +9,5 @@ requests
 requests_ntlm
 docopt
 Jinja2
-Flask
+Flask
+cerberus

validators/netgeo_2_2_doe.py

@@ -6,19 +6,222 @@
 
     @author: olivier.massot, sept. 2018
 '''
-from core.validation import BaseValidator
 
-NAME = "Netgeo v2.2 DOE"
+from core import gis
+from core.mn import ReferentielDb
+from core.validation import BaseValidator, BaseGeoModel, is_french_date
 
-INPUT_FILES = ["artere_geo.shp", "cable_geo.shp", "equipement_passif.shp", "noeud_geo.shp", "tranchee_geo.shp", "zapbo_geo.shp"]
+xmin, xmax, ymin, ymax = 1341999, 1429750, 8147750, 8294000
 
+with ReferentielDb() as ref_db:
+    INSEE_VALIDES = [row[0] for row in ref_db.read("SELECT code_insee FROM sig_referentiel.admn_cd50_com;")]
 
+class Artere(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
+              'AR_CODE': {'type': 'string', 'maxlength': 26}, 
+              'AR_NOM': {'type': 'string', 'maxlength': 26}, 
+              'AR_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
+              'AR_LONG': {'type': 'float'}, 
+              'AR_ETAT': {'type': 'string', 'allowed': ['0', '1', '2', '3', '4']}, 
+              'AR_OCCP': {'type': 'string', 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'AR_NOEUD_A': {'type': 'string', 'maxlength': 20}, 
+              'AR_NOEUD_B': {'type': 'string', 'maxlength': 20}, 
+              'AR_NB_FOUR': {'type': 'string', 'maxlength': 20}, 
+              'AR_FOU_DIS': {'type': 'integer'}, 
+              'AR_TYPE_FO': {'type': 'string', 'allowed': ['PVC', 'PEHD', 'SOUS-TUBAGE PEHD', 'SOUS-TUBAGE  SOUPLE', 'FACADE', 'AERIEN', 'ENCORBELLEMENT', 'AUTRE']}, 
+              'AR_TYFO_AI': {'type': 'string', 'allowed': ['PVC', 'PEH', 'TUB', 'FAC', 'ENC', 'APP']}, 
+              'AR_DIAM_FO': {'type': 'string', 'allowed': ['10', '14', '18', '25', '28', '32', '40', '45', '60', 'NUL']}, 
+              'AR_PRO_FOU': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)']}, 
+              'AR_FAB': {'type': 'string', 'maxlength': 100}, 'AR_REFFAB': {'type': 'string', 'maxlength': 100}, 
+              'AR_COULEUR': {'type': 'string', 'maxlength': 20}, 'AR_AIGUIL': {'type': 'string', 'allowed': ['OUI', 'NON']}, 
+              'AR_NBCABL': {'type': 'integer'}, 
+              'AR_PRO_CAB': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE']}, 
+              'AR_GEST_FO': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
+              'AR_UTIL_FO': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'AR_DATE_IN': {'validator': is_french_date}, 
+              'AR_DATE_RE': {'validator': is_french_date}, 
+              'AR_REF_PLA': {'type': 'string', 'maxlength': 100}, 
+              'AR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'AR_QLT_GEO': {'type': 'string', 'allowed': ['A', 'B', 'C']}, 
+              'AR_PRO_MD': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE']}, 
+              'AR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'AR_STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
     
+class Cable(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
+              'CA_CODE': {'type': 'string', 'maxlength': 18}, 
+              'CA_NOM': {'type': 'string', 'maxlength': 18}, 
+              'CA_NUMERO': {'type': 'string', 'maxlength': 17}, 
+              'CA_EMPRISE': {'type': 'string', 'maxlength': 10}, 
+              'CA_FAB': {'type': 'string', 'maxlength': 100}, 
+              'CA_REFFAB': {'type': 'string', 'maxlength': 100}, 
+              'CA_MODELE': {'type': 'string', 'maxlength': 10}, 
+              'CA_TYPE': {'type': 'string', 'maxlength': 10, 'allowed': ['AERIEN', 'IMMEUBLE', 'FACADE', 'MIXTE', 'SOUTERRAIN']}, 
+              'CA_TYPFCT': {'type': 'string', 'maxlength': 3, 'allowed': ['CDI', 'CTR', 'CBM', 'RAC', 'CBO']}, 
+              'CA_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'CA_LONG': {'type': 'float'}, 'CA_EQ_A': {'type': 'string', 'maxlength': 18}, 
+              'CA_EQ_B': {'type': 'string', 'maxlength': 18}, 
+              'CA_DIAMETR': {'type': 'integer'}, 
+              'CA_COULEUR': {'type': 'string', 'maxlength': 20, 'allowed': ['NOIR', 'BLEU', 'BLANC']}, 
+              'CA_TECHNOL': {'type': 'string', 'maxlength': 17, 'allowed': ['G657A2_M6', 'G657A2_M12']}, 
+              'A_NB_FO_U': {'type': 'integer'}, 'CA_NB_FO_D': {'type': 'integer'}, 
+              'CA_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'CA_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['NUL']}, 
+              'CA_DATE_IN': {'validator': is_french_date}, 
+              'CA_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'CA_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Equipement(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
+              'EQ_CODE': {'type': 'string', 'maxlength': 18}, 
+              'EQ_NOM': {'type': 'string', 'maxlength': 18}, 
+              'EQ_NOM_NOE': {'type': 'string', 'maxlength': 14}, 
+              'EQ_REF': {'type': 'string', 'maxlength': 100}, 
+              'EQ_EMPRISE': {'type': 'string', 'maxlength': 7}, 
+              'EQ_FABR': {'type': 'string', 'maxlength': 100}, 
+              'EQ_CAPFO': {'type': 'integer'}, 
+              'EQ_NBMXEQ': {'type': 'integer'}, 
+              'EQ_NBCAB': {'type': 'integer'}, 
+              'EQ_DIMENS': {'type': 'string', 'maxlength': 50}, 
+              'EQ_TYPEQ': {'type': 'string', 'maxlength': 100}, 
+              'EQ_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'EQ_OCCP': {'type': 'string', 'maxlength': 3, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'EQ_TYPE': {'type': 'string', 'maxlength': 3, 'allowed': ['PBO', 'PEC', 'BPE', 'BAI']}, 
+              'EQ_TYPSTRC': {'type': 'string', 'maxlength': 16, 'allowed': ['CHAMBRE', 'AERIEN', 'FACADE', 'COLONNE MONTANTE', 'PIED IMMEUBLE', 'DTIO']}, 
+              'EQ_TYPE_LQ': {'type': 'string', 'maxlength': 6, 'allowed': ['PBO', 'BPE JB', 'BPE JD', 'BAIDC', 'BAIOP']}, 
+              'EQ_TYPE_PH': {'type': 'string', 'maxlength': 24, 'allowed': ['PBO 6', 'PBO 12', 'BPE 12EP', 'BPE 24EP', 'BPE 48EP', 'BPE 72EP', 'BPE 96EP', 'BPE 144EP', 'BPE 288EP', 'BPE 576EP', 'BPE 720EP']}, 
+              'EQ_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
+              'EQ_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'EQ_HAUT': {'type': 'float'}, 'EQ_DATE_IN': {'type': 'datetime'}, 
+              'EQ_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'EQ_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Noeud(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POINT]}}, 
+              'NO_CODE': {'type': 'string', 'maxlength': 18}, 
+              'NO_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
+              'NO_NOM': {'type': 'string', 'maxlength': 20}, 
+              'NO_VOIE': {'type': 'string', 'maxlength': 100}, 
+              'NO_EMPRISE': {'type': 'string', 'maxlength': 10}, 
+              'NO_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'NO_OCCP': {'type': 'string', 'maxlength': 3, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'NO_TYPE': {'type': 'string', 'maxlength': 3, 'allowed': ['CHA', 'POT', 'LTE', 'SEM', 'FAC', 'OUV', 'IMM']}, 
+              'NO_TYPFCT': {'type': 'string', 'maxlength': 14, 'allowed': ['INTERCONNEXION', 'SATELLITE', 'PASSAGE', 'REGARD', 'INDETERMINE']}, 
+              'NO_TYPE_LQ': {'type': 'string', 'maxlength': 10, 'allowed': ['CHTIR', 'CHRACC', 'POT', 'NRO', 'PM', 'MIMO', 'FAC', 'OUV', 'IMM']}, 
+              'NO_TYPE_PH': {'type': 'string', 'maxlength': 20, 'allowed': ['CHAMBRE', 'POTEAU', 'ARMOIRE', 'SHELTER', 'BATIMENT', 'SITE MIMO', 'FACADE', 'OUVRAGE', 'IMMEUBLE']}, 
+              'NO_CODE_PH': {'type': 'string', 'maxlength': 20}, 
+              'NO_TECH_PS': {'type': 'string', 'maxlength': 4, 'allowed': ['COAX', 'CUT', 'ECL', 'ELEC', 'VP', 'OPT', 'NC']}, 
+              'NO_AMO': {'type': 'string', 'maxlength': 20}, 
+              'NO_PLINOX': {'type': 'string', 'maxlength': 3, 'allowed': ['OUI', 'NON']}, 
+              'NO_X': {'type': 'float'}, 
+              'NO_Y': {'type': 'float'}, 
+              'NO_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'AUTRE (à préciser)', 'NUL']}, 
+              'NO_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'NO_HAUT': {'type': 'float'}, 
+              'NO_DATE_IN': {'validator': is_french_date}, 
+              'NO_REF_PLA': {'type': 'string', 'maxlength': 100}, 
+              'NO_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'NO_QLT_GEO': {'type': 'string', 'maxlength': 1, 'allowed': ['A', 'B', 'C']}, 
+              'NO_PRO_MD': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'NO_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'NO_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Tranchee(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
+              'TR_CODE': {'type': 'string', 'maxlength': 23}, 
+              'TR_NOM': {'type': 'string', 'maxlength': 23, 'tester': 'lambda t, x: x=t.TR_CODE'}, 
+              'TR_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
+              'TR_VOIE': {'type': 'string', 'maxlength': 200}, 
+              'TR_TYP_IMP': {'type': 'string', 'allowed': ['ACCOTEMENT STABILISE', 'ACCOTEMENT NON STABILISE', 'CHAUSSEE LOURDE', 'CHAUSSEE LEGERE', 'FOSSE', 'TROTTOIR', 'ESPACE VERT', 'ENCORBELLEMENT']}, 
+              'TR_MOD_POS': {'type': 'string', 'allowed': ['TRADITIONNEL', 'MICRO TRANCHEE', 'FONCAGE 60', 'FONCAGE 90', 'FONCAGE 120', 'TRANCHEUSE', 'FORAGE URBAIN', 'FORAGE RURAL', 'ENCORBELLEMENT']}, 
+              'TR_MPOS_AI': {'type': 'string', 'allowed': ['TRA', 'ALL', 'FONCAGE 60', 'FON', 'FOR', 'ENC']}, 
+              'TR_LONG': {'type': 'float'}, 
+              'TR_LARG': {'type': 'float'}, 
+              'TR_REVET': {'type': 'string', 'allowed': ['SABLE', 'BICOUCHE', 'ENROBE', 'BETON', 'PAVE', 'TERRAIN NATUREL']}, 
+              'TR_CHARGE': {'type': 'float'}, 
+              'TR_GRILLAG': {'type': 'float'}, 
+              'TR_REMBLAI': {'type': 'string'}, 
+              'TR_PLYNOX': {'type': 'string', 'allowed': ['OUI', 'NON']}, 
+              'TR_PRO_VOI': {'type': 'string', 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
+              'TR_GEST_VO': {'type': 'string', 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
+              'TR_SCHEMA': {'maxlength': 100, 'type': 'string'}, 
+              'TR_DATE_IN': {'validator': is_french_date}, 
+              'TR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'TR_QLT_GEO': {'type': 'string', 'allowed': ['A', 'B', 'C']}, 
+              'TR_PRO_MD': {'type': 'string', 'maxlength': 20}, 
+              'TR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'TR_STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Zapbo(BaseGeoModel):
+    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
+                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
+                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
+                       'parts': {}, 
+                       'points': {}, 
+                       'shapeType': {'allowed': [gis.POLYGON]}}, 
+              'ID_ZAPBO': {'type': 'string', 'maxlength': 10}, 
+              'COMMENTAIR': {'type': 'string', 'maxlength': 254, 'empty': True}, 
+              'STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+
 class Netgeo22Validator(BaseValidator):
+    name = "Netgeo v2.2 DOE"
+    files = {"artere_geo.shp": Artere, "cable_geo.shp": Cable, "equipement_passif.shp": Equipement, "noeud_geo.shp":Noeud, "tranchee_geo.shp":Tranchee, "zapbo_geo.shp":Zapbo}
     
     @classmethod
-    def _submit_folder(cls, folder):
-        print(folder)
+    def _load_file(cls, model, file_path):
+        
+        result = []
+        
+        with gis.ShapeFile(file_path) as sf:
+            fields = sf.fields()
+            for record in sf.records():
+                
+                data = dict(zip(fields, record.record))
+            
+                item = model(record.shape, **data)
+                result.append(item)
+        
+        return result
+    
+    @classmethod        
+    def _technical_validation(cls, dataset):
+        
+        # rattacher les noeuds aux artères
+        # rattacher les equipements aux cables
+        
+        return dataset
 
     
 if __name__ == "__main__":

validators/netgeo_v2-2_doe.yaml

@@ -1,455 +0,0 @@
-xmin: 1341999.9443451899569482
-xmax: 1429750.0912875002250075
-ymin: 8147750.0839389534667134
-ymax: 8294000.0620922148227692
-srid: 3949
-
-databases:
-  sig_postgres: "DRIVER={PostgreSQL Unicode};DATABASE=sig50;SERVER=clusterpg.linux.infra.cloud.local;PORT=5432;UID=sigr;PWD=T38Msh2R4q"
-  
-files:
-  "artere_geo.shp":
-    shape_type: 3
-    fields:
-      AR_CODE:
-        type: str
-        max_len: 26
-      AR_NOM:
-        type: str
-        max_len: 26
-      AR_ID_INSE:
-        type: str
-        max_len: 5
-        in_table:
-          db: sig_postgres
-          table: sig_referentiel.admn_cd50_com
-          field: code_insee
-      AR_LONG:
-        type: float
-      AR_ETAT:
-        type: str
-        in_list: ["0", "1", "2", "3", "4"]
-      AR_OCCP:
-        type: str
-        in_list: ["0", "1.1", "1.2", "2", "3", "4"]
-      AR_NOEUD_A:
-        type: str
-        max_len: 20
-      AR_NOEUD_B:
-        type: str
-        max_len: 20
-      AR_NB_FOUR:
-        type: str
-        max_len: 20
-      AR_FOU_DIS:
-        type: int
-      AR_TYPE_FO:
-        type: str
-        in_list: ["PVC", "PEHD", "SOUS-TUBAGE PEHD", "SOUS-TUBAGE  SOUPLE", "FACADE", "AERIEN", "ENCORBELLEMENT", "AUTRE"]
-      AR_TYFO_AI:
-        type: str
-        in_list: ["PVC", "PEH", "TUB", "FAC", "ENC", "APP"]
-      AR_DIAM_FO:
-        type: str
-        in_list: ["10", "14", "18", "25", "28", "32", "40", "45", "60", "NUL"]
-      AR_PRO_FOU:
-        type: str
-        in_list: ["MANCHE NUMERIQUE", "COLLECTIVITE", "ORANGE", "AUTRE (à préciser)"]
-      AR_FAB:
-        type: str
-        max_len: 100
-      AR_REFFAB:
-        type: str
-        max_len: 100
-      AR_COULEUR:
-        type: str
-        max_len: 20
-      AR_AIGUIL:
-        type: str
-        in_list: ["OUI", "NON"]
-      AR_NBCABL:
-        type: int
-      AR_PRO_CAB:
-        type: str
-        in_list: ["MANCHE NUMERIQUE"]
-      AR_GEST_FO:
-        type: str
-        in_list: ["MANCHE NUMERIQUE", "MANCHE TELECOM", "COLLECTIVITE", "ORANGE", "AUTRE (à préciser)", "NUL"]
-      AR_UTIL_FO:
-        type: str
-        in_list: ["MANCHE NUMERIQUE", "MANCHE TELECOM", "COLLECTIVITE", "ORANGE", "MANCHE FIBRE", "AUTRE (à préciser)", "NUL"]
-      AR_DATE_IN:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      AR_DATE_RE:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      AR_REF_PLA:
-        type: str
-        max_len: 100
-      AR_SRC_GEO:
-        type: str
-        max_len: 50
-      AR_QLT_GEO:
-        type: str
-        in_list: ["A", "B", "C"]
-      AR_PRO_MD:
-        type: str
-        in_list: ["MANCHE NUMERIQUE"]
-      AR_COMMENT:
-        type: str
-        max_len: 300
-        allow_empty: True
-      AR_STATUT:
-        type: str
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-
-  "cable_geo.shp":
-    shape_type: 3
-    fields:
-      CA_CODE:
-        type: str
-        max_len: 18
-      CA_NOM:
-        type: str
-        max_len: 18
-      CA_NUMERO:
-        type: str
-        max_len: 17
-      CA_EMPRISE:
-        type: str
-        max_len: 10
-      CA_FAB:
-        type: str
-        max_len: 100
-      CA_REFFAB:
-        type: str
-        max_len: 100
-      CA_MODELE:
-        type: str
-        max_len: 10
-      CA_TYPE:
-        type: str
-        max_len: 10
-        in_list: ["AERIEN", "IMMEUBLE", "FACADE", "MIXTE", "SOUTERRAIN"]
-      CA_TYPFCT:
-        type: str
-        max_len: 3
-        in_list: ["CDI", "CTR", "CBM", "RAC", "CBO"]
-      CA_ETAT:
-        type: str
-        max_len: 1
-        in_list: ["0", "1", "2", "3", "4"]
-      CA_LONG:
-        type: float
-      CA_EQ_A:
-        type: str
-        max_len: 18
-      CA_EQ_B:
-        type: str
-        max_len: 18
-      CA_DIAMETR:
-        type: int
-      CA_COULEUR:
-        type: str
-        max_len: 20
-        in_list: ["NOIR", "BLEU", "BLANC"]
-      CA_TECHNOL:
-        type: str
-        max_len: 17
-        in_list: ["G657A2_M6", "G657A2_M12"]
-      A_NB_FO_U:
-        type: int
-      CA_NB_FO_D:
-        type: int
-      CA_PRO:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE"]
-      CA_GEST:
-        type: str
-        max_len: 20
-        in_list: ["NUL"]
-      CA_DATE_IN:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      CA_COMMENT:
-        type: str
-        max_len: 300
-        allow_empty: True
-      CA_STATUT:
-        type: str
-        max_len: 14
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-    
-  "equipement_passif.shp":
-    shape_type: 1
-    fields:
-      EQ_CODE:
-        type: str
-        max_len: 18
-      EQ_NOM:
-        type: str
-        max_len: 18
-      EQ_NOM_NOE:
-        type: str
-        max_len: 14
-      EQ_REF:
-        type: str
-        max_len: 100
-      EQ_EMPRISE:
-        type: str
-        max_len: 7
-      EQ_FABR:
-        type: str
-        max_len: 100
-      EQ_CAPFO:
-        type: int
-      EQ_NBMXEQ:
-        type: int
-      EQ_NBCAB:
-        type: int
-      EQ_DIMENS:
-        type: str
-        max_len: 50
-      EQ_TYPEQ:
-        type: str
-        max_len: 100
-      EQ_ETAT:
-        type: str
-        max_len: 1
-        in_list: ["0", "1", "2", "3", "4"]
-      EQ_OCCP:
-        type: str
-        max_len: 3
-        in_list: ["0", "1.1", "1.2", "2", "3", "4"]
-      EQ_TYPE:
-        type: str
-        max_len: 3
-        in_list: ["PBO", "PEC", "BPE", "BAI"]
-      EQ_TYPSTRC:
-        type: str
-        max_len: 16
-        in_list: ["CHAMBRE", "AERIEN", "FACADE", "COLONNE MONTANTE", "PIED IMMEUBLE", "DTIO"]
-      EQ_TYPE_LQ:
-        type: str
-        max_len: 6
-        in_list: ["PBO", "BPE JB", "BPE JD", "BAIDC", "BAIOP"]
-      EQ_TYPE_PH:
-        type: str
-        max_len: 24
-        in_list: ["PBO 6", "PBO 12", "BPE 12EP", "BPE 24EP", "BPE 48EP", "BPE 72EP", "BPE 96EP", "BPE 144EP", "BPE 288EP", "BPE 576EP", "BPE 720EP"]
-      EQ_PRO:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE", "COLLECTIVITE", "ORANGE", "AUTRE (à préciser)", "NUL"]
-      EQ_GEST:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE", "MANCHE TELECOM", "COLLECTIVITE", "ORANGE", "MANCHE FIBRE", "AUTRE (à préciser)", "NUL"]
-      EQ_HAUT:
-        type: float
-      EQ_DATE_IN:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      EQ_COMMENT:
-        type: str
-        max_len: 300
-        allow_empty: True
-      EQ_STATUT:
-        type: str
-        max_len: 14
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-    
-  "noeud_geo.shp":
-    shape_type: 1
-    fields:
-      NO_CODE:
-        type: str
-        max_len: 18
-      NO_ID_INSE:
-        type: str
-        max_len: 5
-        in_table:
-          db: sig_postgres
-          table: sig_referentiel.admn_cd50_com
-          field: code_insee
-      NO_NOM:
-        type: str
-        max_len: 20
-      NO_VOIE:
-        type: str
-        max_len: 100
-      NO_EMPRISE:
-        type: str
-        max_len: 10
-      NO_ETAT:
-        type: str
-        max_len: 1
-        in_list: ["0", "1", "2", "3", "4"]
-      NO_OCCP:
-        type: str
-        max_len: 3
-        in_list: ["0", "1.1", "1.2", "2", "3", "4"]
-      NO_TYPE:
-        type: str
-        max_len: 3
-        in_list: ["CHA", "POT", "LTE", "SEM", "FAC", "OUV", "IMM"]
-      NO_TYPFCT:
-        type: str
-        max_len: 14
-        in_list: ["INTERCONNEXION", "SATELLITE", "PASSAGE", "REGARD", "INDETERMINE"]
-      NO_TYPE_LQ:
-        type: str
-        max_len: 10
-        in_list: ["CHTIR", "CHRACC", "POT", "NRO", "PM", "MIMO", "FAC", "OUV", "IMM"]
-      NO_TYPE_PH:
-        type: str
-        max_len: 20
-        in_list: ["CHAMBRE", "POTEAU", "ARMOIRE", "SHELTER", "BATIMENT", "SITE MIMO", "FACADE", "OUVRAGE", "IMMEUBLE"]
-      NO_CODE_PH:
-        type: str
-        max_len: 20
-      NO_TECH_PS:
-        type: str
-        max_len: 4
-        in_list: ["COAX", "CUT", "ECL", "ELEC", "VP", "OPT", "NC"]
-      NO_AMO:
-        type: str
-        max_len: 20
-      NO_PLINOX:
-        type: str
-        max_len: 3
-        in_list: ["OUI", "NON"]
-      NO_X:
-        type: float
-      NO_Y:
-        type: float
-      NO_PRO:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE", "COLLECTIVITE", "ORANGE", "ERDF", "AUTRE (à préciser)", "NUL"]
-      NO_GEST:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE", "MANCHE TELECOM", "COLLECTIVITE", "ORANGE", "ERDF", "MANCHE FIBRE", "AUTRE (à préciser)", "NUL"]
-      NO_HAUT:
-        type: float
-      NO_DATE_IN:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      NO_REF_PLA:
-        type: str
-        max_len: 100
-      NO_SRC_GEO:
-        type: str
-        max_len: 50
-      NO_QLT_GEO:
-        type: str
-        max_len: 1
-        in_list: ["A", "B", "C"]
-      NO_PRO_MD:
-        type: str
-        max_len: 20
-        in_list: ["MANCHE NUMERIQUE"]
-      NO_COMMENT:
-        type: str
-        max_len: 300
-        allow_empty: True
-      NO_STATUT:
-        type: str
-        max_len: 14
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-      
-  "site_geo.shp":
-    shape_type: 1
-    
-  "tranchee_geo.shp":
-    shape_type: 3
-    can_be_empty: False
-    fields:
-      TR_CODE:
-        type: str
-        max_len: 23
-      TR_NOM:
-        type: str
-        max_len: 23
-        tester: "lambda t, x: x=t.TR_CODE"
-      TR_ID_INSE:
-        type: str
-        max_len: 5
-        in_table:
-          db: sig_postgres
-          table: sig_referentiel.admn_cd50_com
-          field: code_insee
-      TR_VOIE:
-        type: str
-        max_len: 200
-      TR_TYP_IMP:
-        type: str
-        in_list: ["ACCOTEMENT STABILISE", "ACCOTEMENT NON STABILISE", "CHAUSSEE LOURDE", "CHAUSSEE LEGERE", "FOSSE", "TROTTOIR", "ESPACE VERT", "ENCORBELLEMENT"]
-      TR_MOD_POS:
-        type: str
-        in_list: ["TRADITIONNEL", "MICRO TRANCHEE", "FONCAGE 60", "FONCAGE 90", "FONCAGE 120", "TRANCHEUSE", "FORAGE URBAIN", "FORAGE RURAL", "ENCORBELLEMENT"]
-      TR_MPOS_AI:
-        type: str
-        in_list: ["TRA", "ALL", "FONCAGE 60", "FON", "FOR", "ENC"]
-      TR_LONG:
-        type: float
-      TR_LARG:
-        type: float
-      TR_REVET:
-        type: str
-        in_list: ["SABLE", "BICOUCHE", "ENROBE", "BETON", "PAVE", "TERRAIN NATUREL"]
-      TR_CHARGE:
-        type: float
-      TR_GRILLAG:
-        type: float
-      TR_REMBLAI:
-        type: str
-      TR_PLYNOX:
-        type: str
-        in_list: ["OUI", "NON"]
-      TR_PRO_VOI:
-        type: str
-        in_list: ["COMMUNE", "COMMUNAUTE DE COMMUNES", "DEPARTEMENT", "ETAT", "PRIVE"]
-      TR_GEST_VO:
-        type: str
-        in_list: ["COMMUNE", "COMMUNAUTE DE COMMUNES", "DEPARTEMENT", "ETAT", "PRIVE"]
-      TR_SCHEMA:
-        max_len: 100
-        type: str
-      TR_DATE_IN:
-        type: datetime
-        date_format: "%d/%m/%Y"
-      TR_SRC_GEO:
-        type: str
-        max_len: 50
-      TR_QLT_GEO:
-        type: str
-        in_list: ["A", "B", "C"]
-      TR_PRO_MD:
-        type: str
-        max_len: 20
-      TR_COMMENT:
-        type: str
-        max_len: 300
-        allow_empty: True
-      TR_STATUT:
-        type: str
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-        
-  "zapbo_geo.shp":
-    shape_type: 5
-    fields:
-      ID_ZAPBO:
-        type: str
-        max_len: 10
-      COMMENTAIR:
-        type: str
-        max_len: 254
-        allow_empty: True
-      STATUT:
-        type: str
-        in_list: ["EN ETUDE", "EN REALISATION", "EN SERVICE", "HORS SERVICE"]
-