Refonte ok

core/cerberus_extend.py

@@ -0,0 +1,104 @@
+'''
+
+@author: olivier.massot, 2018
+'''
+from datetime import datetime
+
+import cerberus
+
+from core import gis
+
+
+def is_french_date(field, value, error):
+    try:
+        datetime.strptime(value, '%d/%m/%Y')
+    except:
+        error(field, 'Doit être une date au format jj/mm/aaaa')
+
+
+# Ref: http://docs.python-cerberus.org/en/stable/api.html#error-codes
+
+class GeoValidator(cerberus.validator.Validator):
+    def _validate_geom(self, geom, field, value):
+        shapeType, bbox = geom
+        
+        if value.shapeType != shapeType:
+            self._error(field, "Le type de géométrie est {} (attendu: {})".format(gis.SHAPE_NAMES[value.shapeType], gis.SHAPE_NAMES[shapeType]))
+               
+        xmin, ymin, xmax, ymax = bbox
+        
+        try:
+            x1, y1, x2, y2 = value.bbox
+            
+            if any(x < xmin or x > xmax for x in (x1, x2)) or \
+               any(y < ymin or y > ymax for y in (y1, y2)):
+                self._error(field, "Certaines coordonnées hors de l'emprise autorisée")
+        except AttributeError:
+            pass
+            
+    def _validate_shapeType(self, shapeType, field, value):
+        """ Test the oddity of a value.
+
+        The rule's arguments are validated against this schema:
+        {'type': 'boolean'}
+        """
+        if value != shapeType:
+            self._error(field, "Le type de géométrie est incorrect")
+
+    def _validate_bbox(self, bbox, field, value):
+        """ Test the oddity of a value.
+
+        The rule's arguments are validated against this schema:
+        {'type': 'boolean'}
+        """
+        xmin, ymin, xmax, ymax = bbox
+        x1, y1, x2, y2 = value
+        
+        if any(x < xmin or x > xmax for x in (x1, x2)) or \
+           any(y < ymin or y > ymax for y in (y1, y2)):
+            self._error(field, "Certaines coordonnées hors de l'emprise autorisée")
+
+class CerberusErrorHandler(cerberus.errors.BasicErrorHandler):
+    messages = {0x00: "{0}",
+
+                0x01: "Le document est manquant",
+                0x02: "Le champs est obligatoire",
+                0x03: "Champs inconnu",
+                0x04: "Le champs '{0}' est obligatoire",
+                0x05: "Dépends de ces valeurs: {constraint}",
+                0x06: "{0} ne doit pas être présent avec '{field}'",
+
+                0x21: "'{0}' is not a document, must be a dict",
+                0x22: "Le champs ne peut pas être vide",
+                0x23: "Les valeurs NULL ne sont pas autorisées",
+                0x24: "Doit être du type: {constraint}",
+                0x25: "Doit être de type 'dictionnaire'",
+                0x26: "La longueur de la liste doit être de {constraint}, elle est de {0}",
+                0x27: "La longueur minimum est de {constraint}",
+                0x28: "La longueur maximum est de {constraint}",
+
+                0x41: "La valeur n'est pas au bon format ('{constraint}')",
+                0x42: "La valeur minimum autorisée est {constraint}",
+                0x43: "La valeur maximum autorisée est {constraint}",
+                0x44: "Valeur non-autorisée: {value}",
+                0x45: "Valeurs non-autorisées: {0}",
+                0x46: "Valeur non-autorisée: {value}",
+                0x47: "Valeurs non-autorisées: {0}",
+
+                0x61: "Le champs '{field}' ne peut pas être converti: {0}",
+                0x62: "Le champs '{field}' ne peut pas être renommé: {0}",
+                0x63: "Le champs est en lecture seule",
+                0x64: "La valeur par défaut du champs '{field}' ne peut pas être appliquée: {0}",
+
+                0x81: "mapping doesn't validate subschema: {0}",
+                0x82: "one or more sequence-items don't validate: {0}",
+                0x83: "one or more keys of a mapping  don't validate: {0}",
+                0x84: "one or more values in a mapping don't validate: {0}",
+                0x85: "one or more sequence-items don't validate: {0}",
+
+                0x91: "one or more definitions validate",
+                0x92: "none or more than one rule validate",
+                0x93: "no definitions validate",
+                0x94: "one or more definitions don't validate"
+                }
+

core/gis.py

@@ -10,10 +10,12 @@ POLYGON = 5
 SHAPE_NAMES = {1: "POINT", 3:"POLYLIGNE", 5:"POLYGONE"}
 
 class ShapeError(IOError): pass
+class SridError(IOError): pass
 
 class ShapeFile():
-    def __init__(self, path_):
+    def __init__(self, path_, srid=None):
         self.path_ = path_
+        self.srid = srid
     
     def __enter__(self):
         if not self.path_.isfile():
@@ -27,6 +29,10 @@ class ShapeFile():
         if not self.sf.shapeType in (POINT, POLYLINE, POLYGON):
             raise ShapeError("Type de géométrie non reconnue")
 
+#         if self.srid is not None:
+#             if not self.sf.srid == self.srid:
+#                 raise SridError("Le SRID du fichier doit être '{}'".format(self.srid))
+
         return self
     
     def shape(self):

core/validation.py

@@ -3,57 +3,66 @@
 
     @author: olivier.massot, sept. 2018
 '''
-from datetime import datetime
+import time
 import zipfile
 
 from cerberus.validator import Validator
 from path import Path, TempDir
 
-from core.validation_errors import MissingFile, FormatError
+from core import gis
+from core.cerberus_extend import CerberusErrorHandler, GeoValidator
+from core.validation_errors import MissingFile, \
+    UnreadableFile, FormatError, WrongSrid
+from schemas.common import SRID
 
 
 class BaseModel():
-    index = {}
+    filename = ""
     pk = ""
     schema = {}
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
         
     @classmethod
-    def indexer(cls, instance):
-        if instance.getitem(cls.pk) in cls.index:
+    def index_item(cls, instance):
+        if getattr(instance, cls.pk) in cls.index:
             raise ValueError("Duplicate PK")
-        cls.index[instance.getitem(cls.pk)]
+        cls.index[getattr(instance, cls.pk)] = instance
         
 class BaseGeoModel(BaseModel):
     def __init__(self, geom, **kwargs):
         super(BaseGeoModel, self).__init__(**kwargs)
         self.geom = geom
 
+class ValidatorInterruption(BaseException): 
+    pass
 
-
-class ValidationReport():
-    def __init__(self, title = ""):
-        self.title = title
-        self.errors = {}
-    
-    @property
-    def valid(self):
-        return len(self.error) == 0
-
-
-
-
-def is_french_date(field, value, error):
-    try:
-        datetime.strptime(value, '%d/%m/%Y')
-    except:
-        error(field, 'Doit être une date au format jj/mm/aaaa')
+class Checkpoint():
+    def __init__(self, name, valid=True):
+        self.name = name
+        self.valid = valid
 
 class BaseValidator():
-    
-    FILES = {}
-    
+    schema_name = ""
+    models = {}
+    dataset = {}
+    
+    def __init__(self):
+        self.valid = True
+        self.checkpoints = []
+        self.errors = []
+        self.dt = 0
+    
+    def checkpoint(self, name):
+        valid = (len(self.errors) == 0)
+        self.checkpoints.append(Checkpoint(name, valid))
+        if not valid:
+            self.valid = False
+            raise ValidatorInterruption()
+             
+    def log_error(self, validation_error):
+        self.errors.append(validation_error)
+         
     @classmethod
     def submit(cls, subject):    
         """ prends un dossier ou une archive en entrée et vérifie son contenu  """
@@ -76,70 +85,103 @@ class BaseValidator():
     @classmethod
     def _submit_folder(cls, folder):
         
-        dataset = {}
-        report = ValidationReport("Contrôle des données de {} au format {}".format(folder.name, cls.name))
+        validator = cls()
+        t0 = time.time()
+        try:
+            validator.validate(folder)
+        except ValidatorInterruption:
+            pass
+        validator.dt = time.time() - t0
         
-        # Charge les données en mémoire
-        for filename, model in cls.files.items():
-            path_ = Path(folder) / filename
-            
-            if not path_.isfile():
-                report.errors[MissingFile] = MissingFile("Le fichier '{}' est manquant".format(filename))
-                continue
-            
-            dataset[model] = cls._load_file(model, path_)
+        report = validator.build_report("{} - Validation de {}".format(validator.schema_name, folder.name))
+        return report
+    
+    def validate(self, folder):
         
-        # Controle la structure des données (champs, formats et types)
-        cls._structure_validation(dataset)
+        # Chargement des données en mémoire
+        self._load_files(folder)
+        self.checkpoint("Chargement des données")
         
-        # Contrôle la géométrie (optionnel)
-        cls._geometry_validation(dataset)
+        # Controle la structure des données (champs, formats et types)
+        self._structure_validation()
+        self.checkpoint("Contrôle de la structure des données")
         
         # Validation technique
-        cls._technical_validation(dataset)
+        self._technical_validation()
+        self.checkpoint("Validation Métier")
     
-        return report
     
-    @classmethod
-    def _load_file(cls, model, filename):
+    def _load_files(self, folder):
         """ Charge les données du fichier et les associe à un modèle.
-        Attention: pas de contrôle de format o de validité à ce niveau!  """
+        Attention: pas de contrôle de format ou de validité à ce niveau!  """
         raise NotImplementedError()
     
-    @classmethod        
-    def _structure_validation(cls, dataset):
-        
-        errors = {}
-        errors[FormatError] = []
+    def _structure_validation(self):
         
-        for model in dataset:
-            v = Validator(model.schema)
+        for model in self.models:
+            v = GeoValidator(model.schema, error_handler=CerberusErrorHandler)
             
-            for item in dataset[model]:
+            for item in self.dataset[model]:
                 v.validate(item.__dict__)
-                
-                for fieldname, verrors in v.errors.items():
+            
+                for field, verrors in v.errors.items():
                     for err in verrors:
-                        errors[FormatError].append(FormatError("{}: {}".format(fieldname, err)))
-        
-        return errors
-    
-    @classmethod
-    def _geometry_validation(cls, dataset):
-        pass
-    
-    @classmethod
-    def _organize_dataset(cls, dataset):
-        raise NotImplementedError()
+                        self.log_error(FormatError(err, filename=model.filename, field=field))
     
     @classmethod
-    def _technical_validation(cls, dataset):
+    def _technical_validation(cls):
         raise NotImplementedError()
     
-    
+    def build_report(self, title):
+        report = {}
+        report["title"] = title
+        report["exec_time"] = "{} s.".format(self.dt)
+        report["checkpoints"] = [{"name": chk.name, "valid": chk.valid} for chk in self.checkpoints]
+        
+        report["errors"] = {}
+        
+        for err in self.errors:
+            if not err.name in report["errors"]:
+                report["errors"][err.name] = {"help": err.help, "list": []}
+            
+            err_report = {"filename": err.filename or "-",
+                          "field": err.field or "-",
+                          "message": err.message}
+            if err_report not in report["errors"][err.name]["list"]:
+                report["errors"][err.name]["list"].append(err_report)
+            
+        return report
     
         
+class NetgeoValidator(BaseValidator):
     
+    def _load_files(self, folder):
+        
+        for model in self.models:
+            filename = model.filename
+            path_ = Path(folder) / filename
+            
+            if not path_.isfile():
+                self.log_error(MissingFile("Fichier manquant: '{}'".format(filename)))
+                continue
+            
+            self.dataset[model] = []
+            try:
+                with gis.ShapeFile(path_, srid=SRID) as sf:
+                    fields = sf.fields()
+                    
+                    for record in sf.records():
+                        
+                        data = dict(zip(fields, record.record))
+                    
+                        item = model(record.shape, **data)
+                        
+                        self.dataset[model].append(item)
+                        
+            except gis.ShapeError as e:
+                self.log_error(UnreadableFile(str(e)))
+            except gis.SridError:
+                self.log_error(WrongSrid(str(e)))
     
     
     

core/validation_errors.py

@@ -4,12 +4,12 @@
 '''
 
 VALIDATION_ERROR_LEVELS = {10: "MINEURE", 20: "ATTENTION", 30: "ERREUR", 40: "CRITIQUE"}
-
 MINOR = 10
 WARNING = 20
 ERROR = 30
 CRITICAL = 40
 
+
 ### Vérifications générales
 
 # Fichiers présents
@@ -37,16 +37,21 @@ CRITICAL = 40
 
 
 
-
-
+# filename, fieldname, record_id, 
 
 class BaseValidationError():
-    name = ""
-    level = 0
-    def __init__(self, message):
+    name = "Erreur"
+    level = ERROR
+    help = ""
+    def __init__(self, message, filename="", field="", index=""):
         self.message = message
+        self.filename = filename
+        self.field = field
         
-#### Erreurs Critiques
+    def __repr__(self):
+        return " - ".join(filter(None, [self.name, self.filename, self.field, self.message]))
+
+# Erreurs dans le chargement des fichiers
 
 class MissingFile(BaseValidationError):
     level = CRITICAL
@@ -60,36 +65,50 @@ class WrongSrid(BaseValidationError):
     level = CRITICAL
     name = "Mauvais SRID"
 
-class FieldTypeError(BaseValidationError):
-    level = CRITICAL
-    name = "Le champs n'a pas le bon type de donnée"
-    
-#### Erreurs
 
-class OutOfBoxCoordinates(BaseValidationError):
-    level = ERROR
-    name = "Coordonnées hors de la zone autorisée"
+### Erreurs dans la structure des données
 
-class FormatError(BaseValidationError):
+class StructureValidationError(BaseValidationError):
+    name = "Erreur dans la structure des données"
     level = ERROR
-    name = "Le champs n'a pas le bon format"
 
-class MissingValueError(BaseValidationError):
+class MissingField(BaseValidationError):
+    name = "Champs manquant"
+    level = CRITICAL
+
+class UnknownField(BaseValidationError):
+    name = "Champs inconnu"
+    level = CRITICAL
+    
+class MissingValue(BaseValidationError):
+    name = "Valeur(s) manquante(s)"
     level = ERROR
-    name = "Le champs doit être renseigné"
     
-class FieldValueError(BaseValidationError):
+class FieldTypeError(BaseValidationError):
+    name = "Erreur de type"
     level = ERROR
-    name = "La valeur du champs est invalide"
-
-#### Avertissements
-
-
 
+class TooLong(BaseValidationError):
+    name = "Dépassement de taille"
+    level = ERROR
+    
+class FormatError(BaseValidationError):
+    name = "Erreur de format"
+    level = ERROR
 
+class UnauthorizedValue(BaseValidationError):
+    name="Valeur non-autorisée"
+    level = ERROR
+    
+# Erreurs dans le contenu, erreurs métiers
 
+class TechnicalValidationError(BaseValidationError):
+    level = ERROR
 
-#### Erreurs mineures
+class DuplicatedPk(TechnicalValidationError):
+    name = "Doublons dans le champs"
 
+class RelationError(TechnicalValidationError):
+    name = "Un objet lié n'existe pas"
 
 

datachecker.py

@@ -1,342 +0,0 @@
-'''
-Python 3.7+
-
-@author: olivier.massot, sept 2018
-'''
-from datetime import datetime
-import json
-import logging
-import subprocess  # @UnusedImport
-import tempfile  # @UnusedImport
-import zipfile
-
-from path import Path, TempDir
-import pypyodbc
-import shapefile
-import yaml
-
-from core import logconf
-from core.constants import MAIN
-
-
-logger = logging.getLogger("datachecker")
-logconf.start("datachecker", logging.INFO)
-pypyodbc.lowercase = False
-logger.disabled = True
-
-# TODO: Vérifier la projection (besoin de GDAL/OGR)
-# TODO: fonctions de controle spéciales
-
-class ReportField():
-    def __init__(self, name, value = "", error=""):
-        self.name = name
-        self.value = value
-        self._error = error
-        self._valid = True
-
-    @property
-    def error(self):
-        return self._error
-    
-    @error.setter
-    def error(self, error):
-        self._valid = False
-        logger.error("%s - %s ('%s')", self.name, error, self.value)
-        self._error = error
-
-    @property
-    def valid(self):
-        return self._valid
-
-    def to_dict(self):
-        return {
-            "name": self.name,
-            "value": self.value,
-            "error": self._error,
-            "valid": self._valid
-            }
-
-class ReportRecord():
-    def __init__(self, index):
-        self.index = index
-        self._valid = True
-        self._errors = []
-        self._fields = []
-
-    @property
-    def errors(self):
-        return self._errors
-    
-    def add_error(self, error):
-        self._valid = False
-        logger.error("Ligne %s - %s", self.index, error)
-        self._errors.append(error)
-
-    @property
-    def valid(self):
-        return self._valid
-
-    @property
-    def fields(self):
-        return self._fields
-
-    def add_field(self, field):
-        if not field.valid:
-            self._valid=False
-        self._fields.append(field)
-
-    def to_dict(self):
-        return {
-            "index": self.index,
-            "valid": self._valid,
-            "errors": self._errors,
-            "fields": [f.to_dict() for f in self._fields]
-            }
-
-class ReportFile():
-    def __init__(self, file):
-        self.file = file
-        self.headers = []
-        self._valid = True
-        self._errors = []
-        self._records = []
-
-    @property
-    def valid(self):
-        return self._valid   
-
-    @property
-    def records(self):
-        return self._records
-
-    def add_record(self, row):
-        if not row.valid:
-            self._valid=False
-        self._records.append(row)
-
-    @property
-    def errors(self):
-        return self._errors
-    
-    def add_error(self, error):
-        self._valid=False
-        logger.error("Fichier %s - %s", self.file, error)
-        self._errors.append(error)
-        
-    def to_dict(self):
-        return {
-            "file": self.file,
-            "headers": self.headers,
-            "valid": self.valid,
-            "errors": self._errors,
-            "records": [r.to_dict() for r in self._records]
-            }
-
-class Report():
-    def __init__(self, title, report_files=[]):
-        self.title = title
-        self.report_files = report_files
-
-    @property
-    def valid(self):
-        return all([r.valid for r in self.report_files])
-
-    def to_dict(self):
-        return {
-            "title": self.title,
-            "report_files": [rf.to_dict() for rf in self.report_files],
-            "valid": self.valid
-            }
-    
-    def to_json(self):
-        return json.dumps(self.to_dict())
-
-def check(subject, checker):    
-    """ prends un dossier ou une archive en entier et vérifie son contenu selon les règles données par le fichier de config """
-    subject, checker = Path(subject), Path(checker)
-    
-    if subject.isfile():
-        with TempDir() as dirname:
-            zip_ref = zipfile.ZipFile(subject, 'r')
-            zip_ref.extractall(dirname)
-            zip_ref.close()
-            if Path(dirname / subject.stem).isdir(): # cas où l'archive contient un dossier qui lui-même contient les fichiers
-                dirname /= subject.stem
-            return check_folder(dirname, checker)
-        
-    elif subject.isdir():
-        return check_folder(subject, checker)
-    else:
-        raise IOError(f"Impossible de trouver le fichier ou répertoire: {subject}")
-
-def check_folder(folder, checker):
-    logger.info("*****   Traitement de '%s'   *****", folder.name)
-    
-    logger.info("> Controlleur: '%s'", checker.name)
-    
-    report = Report("Contrôle des données de {} au format {}".format(folder.name, checker.stem))
-    
-    db_cache = {}
-    
-    with open(checker, "r") as cf:
-        config = yaml.load(cf)
-    
-    databases = {}
-    for dbname, dsn in config.get("databases", {}).items():
-        cnn = pypyodbc.connect(dsn)
-        databases[dbname] = cnn
-        
-    for filename, model in config["files"].items():
-        path_ = folder / filename
-        logger.info("* Traitement de %s", path_.name)
-        
-        report_file = ReportFile(path_.name)
-        
-        if not path_.isfile():
-            report_file.add_error("Fichier introuvable")
-            continue
-        
-        try:
-            sf = shapefile.Reader(path_)
-        except shapefile.ShapefileException:
-            report_file.add_error("Fichier illisible")
-            continue
-        
-        if "srid" in config:
-            pass
-
-        xmin, xmax, ymin, ymax = (int(config.get("xmin", 0)), 
-                                  int(config.get("xmax", float("inf"))), 
-                                  int(config.get("ymin", 0)), 
-                                  int(config.get("ymax", float("inf")))
-                                  )
-            
-        if "shape_type" in model:
-            shape_names = {1:"Point", 3:"Polyligne", 5:"Polygone"}
-            if sf.shapeType != model["shape_type"]:
-                report_file.add_error("Le fichier shapefile n'est pas de type {}".format(shape_names[model["shape_type"]]))
-                del sf
-                continue
-
-        records = sf.shapeRecords()
-        if not records and not model["can_be_empty"]:
-            report_file.add_error("Le fichier shapefile ne contient aucune donnees")
-            del sf, records
-            continue
-        
-        if not "fields" in model:
-            continue
-        
-        fields = [f[0] for f in sf.fields if f[0] != 'DeletionFlag']
-        
-        report_file.headers = list(model["fields"].keys())
-        
-        # parcours et controle des enregistrements
-        for i, record in enumerate(records):
-            
-            logger.info("\n> Enregistrement n°%s\n", i)
-            report_record = ReportRecord(i)
-            
-            record_data = {field: record.record[i] for i, field in enumerate(fields)}
-            
-            try:
-                x1, y1, x2, y2 = sf.shapes()[i].bbox
-            except AttributeError:
-                x1, y1 = sf.shapes()[i].points[0]
-                x2, y2 = x1, y1
-            
-            if not xmin <= x1 <= xmax or not xmin <= x2 <= xmax or \
-               not ymin <= y1 <= ymax or not ymin <= y2 <= ymax:
-                report_record.add_error("L'élément est situé hors de la zone géographique autorisée")
-                
-            for fieldname, fieldmodel in model["fields"].items():
-                
-                report_field = ReportField(fieldname)
-                
-                try:
-                    val = record_data[fieldname]
-                except KeyError:
-                    if fieldmodel.get("required", True):
-                        report_field.error = "Champs manquant"
-                    report_record.add_field(report_field)
-                    continue
-                
-                report_field.value = val
-                
-                type_ = fieldmodel.get("type", "str")
-                if type_ == "int":
-                    try:
-                        _ = int(val)
-                    except (TypeError, ValueError):
-                        report_field.error = "Valeur Invalide, un nombre entier est attendu"
-                        report_record.add_field(report_field)
-                        continue
-                elif type_ == "float":
-                    try:
-                        _ = float(val)
-                    except (TypeError, ValueError):
-                        report_field.error = "Valeur Invalide, un nombre décimal est attendu"
-                        report_record.add_field(report_field)
-                        continue
-                elif type_ == "datetime":
-                    try:
-                        _ = datetime.strptime(val, fieldmodel.get("date_format", "%d/%m/%Y"))
-                    except ValueError:
-                        report_field.error = "Valeur Invalide, une date est attendue"
-                        report_record.add_field(report_field)
-                        continue
-                else: 
-                    if not fieldmodel.get("allow_empty", False) and not val:
-                        report_field.error = "Ce champs ne peut pas être vide"
-                        report_record.add_field(report_field)
-                        continue
-                
-                if type_ == "str" and "max_len" in fieldmodel:
-                    if len(str(val)) > fieldmodel["max_len"]:
-                        report_field.error = "Trop long, la longueur max. est de {}".format(fieldmodel["max_len"])
-                        report_record.add_field(report_field)
-                        continue
-                
-                try:
-                    if not val in fieldmodel["in_list"]:
-                        report_field.error = "Valeur invalide, pas dans la liste"
-                        report_record.add_field(report_field)
-                        continue
-                except KeyError:
-                    pass
-        
-                if "in_table" in fieldmodel:
-
-                    key = tuple([fieldmodel["in_table"]["db"], fieldmodel["in_table"]["table"], fieldmodel["in_table"]["field"]])
-                    if not key in db_cache:
-                        db = databases[fieldmodel["in_table"]["db"]]
-                        cursor = db.cursor()
-                        cursor.execute("SELECT DISTINCT {} FROM {};".format(fieldmodel["in_table"]["field"], fieldmodel["in_table"]["table"]))
-                        rows = [val[0] for val in cursor.fetchall()]
-                        db_cache[key] = rows
-                     
-                    if not val in db_cache[key]:
-                        report_field.error = "Valeur invalide, pas dans la liste"
-                        report_record.add_field(report_field)
-                        continue
-                
-                report_record.add_field(report_field)
-            report_file.add_record(report_record)
-        report.report_files.append(report_file)
-        del sf, records
-        
-    return report
-        
-
-if __name__ == "__main__":
-    
-    logger.disabled = False
-    subject = MAIN / "work" / "STURNO_192AP0_REC_COMPLEMENT_180822_OK.zip"
-    checker = MAIN / "checkers" / "netgeo_v2-2_doe.yaml"
-     
-    report = check(subject, checker)
-    
-    with open(MAIN / "report.json", "w+") as fp:
-        json.dump(report.to_dict(), fp)
-
-    logger.info("-- Fin --")

index.py

@@ -11,7 +11,7 @@ from path import Path
 from werkzeug.utils import secure_filename
 
 from core.constants import MAIN
-import datachecker
+from schemas.netgeo_2_2_doe.validator import Netgeo22DoeValidator
 
 
 app = Flask(__name__)
@@ -36,9 +36,9 @@ def report():
         with TemporaryDirectory(dir=MAIN / "upload") as d:
             filename = Path(d) / filename
             f.save(filename)
-            report = datachecker.check(filename, MAIN / "checkers" / "netgeo_v2-2_doe.yaml")
+            report = Netgeo22DoeValidator.submit(filename)
     except Exception as e:
         return render_template("index.html", validation_error=str(e))
     
-    return render_template("report.html", report=report.to_dict())
+    return render_template("report.html", report=report)
 

schemas/common.py

@@ -0,0 +1,12 @@
+'''
+
+@author: olivier.massot, 2018
+'''
+from core.mn import ReferentielDb
+
+
+XMIN, XMAX, YMIN, YMAX = 1341999, 1429750, 8147750, 8294000
+SRID = 3949
+
+with ReferentielDb() as ref_db:
+    INSEE_VALIDES = [row[0] for row in ref_db.read("SELECT code_insee FROM sig_referentiel.admn_cd50_com;")]

schemas/netgeo_2_2_doe/__init__.py

@@ -0,0 +1,2 @@
+
+from . import *

schemas/netgeo_2_2_doe/models.py

@@ -0,0 +1,164 @@
+'''
+
+@author: olivier.massot, 2018
+'''
+from core import gis
+from core.cerberus_extend import is_french_date
+from core.validation import BaseGeoModel
+from schemas.common import INSEE_VALIDES, XMIN, YMIN, XMAX, YMAX
+
+
+class Artere(BaseGeoModel):
+    filename = "artere_geo.shp"
+    pk = "AR_CODE"
+    schema = {'geom': {'geom': (gis.POLYLINE, (XMIN, YMIN, XMAX, YMAX))}, 
+              'AR_CODE': {'type': 'string', 'maxlength': 26}, 
+              'AR_NOM': {'type': 'string', 'maxlength': 26}, 
+              'AR_ID_INSE': {'type': 'string', 'empty': False, 'allowed': INSEE_VALIDES}, 
+              'AR_LONG': {'type': 'float'}, 
+              'AR_ETAT': {'type': 'string', 'empty': False, 'empty': False, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'AR_OCCP': {'type': 'string', 'empty': False, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'AR_NOEUD_A': {'type': 'string', 'maxlength': 20}, 
+              'AR_NOEUD_B': {'type': 'string', 'maxlength': 20}, 
+              'AR_NB_FOUR': {'type': 'string', 'maxlength': 20}, 
+              'AR_FOU_DIS': {'type': 'integer'}, 
+              'AR_TYPE_FO': {'type': 'string', 'empty': False, 'allowed': ['PVC', 'PEHD', 'SOUS-TUBAGE PEHD', 'SOUS-TUBAGE  SOUPLE', 'FACADE', 'AERIEN', 'ENCORBELLEMENT', 'AUTRE']}, 
+              'AR_TYFO_AI': {'type': 'string', 'empty': False, 'allowed': ['PVC', 'PEH', 'TUB', 'FAC', 'ENC', 'APP']}, 
+              'AR_DIAM_FO': {'type': 'string', 'empty': False, 'allowed': ['10', '14', '18', '25', '28', '32', '40', '45', '60', 'NUL']}, 
+              'AR_PRO_FOU': {'type': 'string', 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)']}, 
+              'AR_FAB': {'type': 'string', 'maxlength': 100}, 'AR_REFFAB': {'type': 'string', 'maxlength': 100}, 
+              'AR_COULEUR': {'type': 'string', 'maxlength': 20}, 'AR_AIGUIL': {'type': 'string', 'empty': False, 'allowed': ['OUI', 'NON']}, 
+              'AR_NBCABL': {'type': 'integer'}, 
+              'AR_PRO_CAB': {'type': 'string', 'empty': False, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'AR_GEST_FO': {'type': 'string', 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'AR_UTIL_FO': {'type': 'string', 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
+              'AR_DATE_IN': {'validator': is_french_date}, 
+              'AR_DATE_RE': {'validator': is_french_date}, 
+              'AR_REF_PLA': {'type': 'string', 'maxlength': 100}, 
+              'AR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'AR_QLT_GEO': {'type': 'string', 'empty': False, 'allowed': ['A', 'B', 'C']}, 
+              'AR_PRO_MD': {'type': 'string', 'empty': False, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'AR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'AR_STATUT': {'type': 'string', 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Cable(BaseGeoModel):
+    filename = "cable_geo.shp"
+    pk = "CA_CODE"
+    schema = {'geom': {'geom': (gis.POLYLINE, (XMIN, YMIN, XMAX, YMAX))}, 
+              'CA_CODE': {'type': 'string', 'maxlength': 18}, 
+              'CA_NOM': {'type': 'string', 'maxlength': 18}, 
+              'CA_NUMERO': {'type': 'string', 'maxlength': 17}, 
+              'CA_EMPRISE': {'type': 'string', 'maxlength': 10}, 
+              'CA_FAB': {'type': 'string', 'maxlength': 100}, 
+              'CA_REFFAB': {'type': 'string', 'maxlength': 100}, 
+              'CA_MODELE': {'type': 'string', 'maxlength': 10}, 
+              'CA_TYPE': {'type': 'string', 'maxlength': 10, 'empty': False, 'allowed': ['AERIEN', 'IMMEUBLE', 'FACADE', 'MIXTE', 'SOUTERRAIN']}, 
+              'CA_TYPFCT': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['CDI', 'CTR', 'CBM', 'RAC', 'CBO']}, 
+              'CA_ETAT': {'type': 'string', 'maxlength': 1, 'empty': False, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'CA_LONG': {'type': 'float'}, 'CA_EQ_A': {'type': 'string', 'maxlength': 18}, 
+              'CA_EQ_B': {'type': 'string', 'maxlength': 18}, 
+              'CA_DIAMETR': {'type': 'integer'}, 
+              'CA_COULEUR': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['NOIR', 'BLEU', 'BLANC']}, 
+              'CA_TECHNOL': {'type': 'string', 'maxlength': 17, 'empty': False, 'allowed': ['G657A2_M6', 'G657A2_M12']}, 
+              'A_NB_FO_U': {'type': 'integer'}, 'CA_NB_FO_D': {'type': 'integer'}, 
+              'CA_PRO': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'CA_GEST': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['NUL']}, 
+              'CA_DATE_IN': {'validator': is_french_date}, 
+              'CA_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'CA_STATUT': {'type': 'string', 'maxlength': 14, 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Equipement(BaseGeoModel):
+    filename = "equipement_passif.shp"
+    pk = "EQ_CODE"
+    schema = {'geom': {'geom': (gis.POINT, (XMIN, YMIN, XMAX, YMAX))}, 
+              'EQ_CODE': {'type': 'string', 'maxlength': 18}, 
+              'EQ_NOM': {'type': 'string', 'maxlength': 18}, 
+              'EQ_NOM_NOE': {'type': 'string', 'maxlength': 14}, 
+              'EQ_REF': {'type': 'string', 'maxlength': 100}, 
+              'EQ_EMPRISE': {'type': 'string', 'maxlength': 7}, 
+              'EQ_FABR': {'type': 'string', 'maxlength': 100}, 
+              'EQ_CAPFO': {'type': 'integer'}, 
+              'EQ_NBMXEQ': {'type': 'integer'}, 
+              'EQ_NBCAB': {'type': 'integer'}, 
+              'EQ_DIMENS': {'type': 'string', 'maxlength': 50}, 
+              'EQ_TYPEQ': {'type': 'string', 'maxlength': 100}, 
+              'EQ_ETAT': {'type': 'string', 'maxlength': 1, 'empty': False, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'EQ_OCCP': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'EQ_TYPE': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['PBO', 'PEC', 'BPE', 'BAI']}, 
+              'EQ_TYPSTRC': {'type': 'string', 'maxlength': 16, 'empty': False, 'allowed': ['CHAMBRE', 'AERIEN', 'FACADE', 'COLONNE MONTANTE', 'PIED IMMEUBLE', 'DTIO']}, 
+              'EQ_TYPE_LQ': {'type': 'string', 'maxlength': 6, 'empty': False, 'allowed': ['PBO', 'BPE JB', 'BPE JD', 'BAIDC', 'BAIOP']}, 
+              'EQ_TYPE_PH': {'type': 'string', 'maxlength': 24, 'empty': False, 'allowed': ['PBO 6', 'PBO 12', 'BPE 12EP', 'BPE 24EP', 'BPE 48EP', 'BPE 72EP', 'BPE 96EP', 'BPE 144EP', 'BPE 288EP', 'BPE 576EP', 'BPE 720EP']}, 
+              'EQ_PRO': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
+              'EQ_GEST': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'EQ_HAUT': {'type': 'float'}, 
+              'EQ_DATE_IN': {'validator': is_french_date}, 
+              'EQ_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'EQ_STATUT': {'type': 'string', 'maxlength': 14, 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Noeud(BaseGeoModel):
+    filename = "noeud_geo.shp"
+    pk = "NO_CODE"
+    schema = {'geom': {'geom': (gis.POINT, (XMIN, YMIN, XMAX, YMAX))}, 
+              'NO_CODE': {'type': 'string', 'maxlength': 18}, 
+              'NO_ID_INSE': {'type': 'string', 'empty': False, 'allowed': INSEE_VALIDES}, 
+              'NO_NOM': {'type': 'string', 'maxlength': 20}, 
+              'NO_VOIE': {'type': 'string', 'maxlength': 100}, 
+              'NO_EMPRISE': {'type': 'string', 'maxlength': 10}, 
+              'NO_ETAT': {'type': 'string', 'maxlength': 1, 'empty': False, 'allowed': ['0', '1', '2', '3', '4']}, 
+              'NO_OCCP': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
+              'NO_TYPE': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['CHA', 'POT', 'LTE', 'SEM', 'FAC', 'OUV', 'IMM']}, 
+              'NO_TYPFCT': {'type': 'string', 'maxlength': 14, 'empty': False, 'allowed': ['INTERCONNEXION', 'SATELLITE', 'PASSAGE', 'REGARD', 'INDETERMINE']}, 
+              'NO_TYPE_LQ': {'type': 'string', 'maxlength': 10, 'empty': False, 'allowed': ['CHTIR', 'CHRACC', 'POT', 'NRO', 'PM', 'MIMO', 'FAC', 'OUV', 'IMM']}, 
+              'NO_TYPE_PH': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['CHAMBRE', 'POTEAU', 'ARMOIRE', 'SHELTER', 'BATIMENT', 'SITE MIMO', 'FACADE', 'OUVRAGE', 'IMMEUBLE']}, 
+              'NO_CODE_PH': {'type': 'string', 'maxlength': 20}, 
+              'NO_TECH_PS': {'type': 'string', 'maxlength': 4, 'empty': False, 'allowed': ['COAX', 'CUT', 'ECL', 'ELEC', 'VP', 'OPT', 'NC']}, 
+              'NO_AMO': {'type': 'string', 'maxlength': 20}, 
+              'NO_PLINOX': {'type': 'string', 'maxlength': 3, 'empty': False, 'allowed': ['OUI', 'NON']}, 
+              'NO_X': {'type': 'float'}, 
+              'NO_Y': {'type': 'float'}, 
+              'NO_PRO': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'AUTRE (à préciser)', 'NUL']}, 
+              'NO_GEST': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
+              'NO_HAUT': {'type': 'float'}, 
+              'NO_DATE_IN': {'validator': is_french_date}, 
+              'NO_REF_PLA': {'type': 'string', 'maxlength': 100}, 
+              'NO_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'NO_QLT_GEO': {'type': 'string', 'maxlength': 1, 'empty': False, 'allowed': ['A', 'B', 'C']}, 
+              'NO_PRO_MD': {'type': 'string', 'maxlength': 20, 'empty': False, 'allowed': ['MANCHE NUMERIQUE']}, 
+              'NO_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'NO_STATUT': {'type': 'string', 'maxlength': 14, 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Tranchee(BaseGeoModel):
+    filename = "tranchee_geo.shp"
+    pk = "TR_CODE"
+    schema = {'geom': {'geom': (gis.POLYLINE, (XMIN, YMIN, XMAX, YMAX))}, 
+              'TR_CODE': {'type': 'string', 'maxlength': 23}, 
+              'TR_NOM': {'type': 'string', 'maxlength': 23}, 
+              'TR_ID_INSE': {'type': 'string', 'empty': False, 'allowed': INSEE_VALIDES}, 
+              'TR_VOIE': {'type': 'string', 'maxlength': 200}, 
+              'TR_TYP_IMP': {'type': 'string', 'empty': False, 'allowed': ['ACCOTEMENT STABILISE', 'ACCOTEMENT NON STABILISE', 'CHAUSSEE LOURDE', 'CHAUSSEE LEGERE', 'FOSSE', 'TROTTOIR', 'ESPACE VERT', 'ENCORBELLEMENT']}, 
+              'TR_MOD_POS': {'type': 'string', 'empty': False, 'allowed': ['TRADITIONNEL', 'MICRO TRANCHEE', 'FONCAGE 60', 'FONCAGE 90', 'FONCAGE 120', 'TRANCHEUSE', 'FORAGE URBAIN', 'FORAGE RURAL', 'ENCORBELLEMENT']}, 
+              'TR_MPOS_AI': {'type': 'string', 'empty': False, 'allowed': ['TRA', 'ALL', 'FONCAGE 60', 'FON', 'FOR', 'ENC']}, 
+              'TR_LONG': {'type': 'float'}, 
+              'TR_LARG': {'type': 'float'}, 
+              'TR_REVET': {'type': 'string', 'empty': False, 'allowed': ['SABLE', 'BICOUCHE', 'ENROBE', 'BETON', 'PAVE', 'TERRAIN NATUREL']}, 
+              'TR_CHARGE': {'type': 'float'}, 
+              'TR_GRILLAG': {'type': 'float'}, 
+              'TR_REMBLAI': {'type': 'string'}, 
+              'TR_PLYNOX': {'type': 'string', 'empty': False, 'allowed': ['OUI', 'NON']}, 
+              'TR_PRO_VOI': {'type': 'string', 'empty': False, 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
+              'TR_GEST_VO': {'type': 'string', 'empty': False, 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
+              'TR_SCHEMA': {'maxlength': 100, 'type': 'string'}, 
+              'TR_DATE_IN': {'validator': is_french_date}, 
+              'TR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
+              'TR_QLT_GEO': {'type': 'string', 'empty': False, 'allowed': ['A', 'B', 'C']}, 
+              'TR_PRO_MD': {'type': 'string', 'maxlength': 20}, 
+              'TR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
+              'TR_STATUT': {'type': 'string', 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
+
+class Zapbo(BaseGeoModel):
+    filename = "zapbo_geo.shp"
+    pk = "ID_ZAPBO"
+    schema = {'geom': {'geom': (gis.POLYGON, (XMIN, YMIN, XMAX, YMAX))}, 
+              'ID_ZAPBO': {'type': 'string', 'maxlength': 10}, 
+              'COMMENTAIR': {'type': 'string', 'maxlength': 254, 'empty': True}, 
+              'STATUT': {'type': 'string', 'empty': False, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}

schemas/netgeo_2_2_doe/validator.py

@@ -0,0 +1,72 @@
+'''
+
+@author: olivier.massot, 2018
+'''
+from core.validation import NetgeoValidator
+from core.validation_errors import DuplicatedPk, RelationError
+from schemas.netgeo_2_2_doe.models import Artere, Cable, Equipement, Noeud, \
+    Tranchee, Zapbo
+
+
+class Netgeo22DoeValidator(NetgeoValidator):
+    schema_name = "Netgeo v2.2 DOE"
+    models = [Artere, Cable, Equipement, Noeud, Tranchee, Zapbo]
+    
+    def _technical_validation(self):
+        
+        # construction de l'index
+        self.index = {}
+        for model in self.dataset:
+            self.index[model] = {}
+            
+            for item in self.dataset[model]:
+                pk_value = getattr(item, model.pk)
+                if pk_value in self.index[model]:
+                    self.log_error(DuplicatedPk("Doublons dans le champs", filename=model.filename, field=model.pk))
+                else:
+                    self.index[model][pk_value] = item
+            
+        del self.dataset
+             
+        if self.errors: 
+            return
+            
+        # rattachement les noeuds aux artères     
+        for artere in self.index[Artere]:
+            try:
+                artere.noeud_a = self.index[Artere][artere.AR_NOEUD_A]
+            except KeyError:
+                self.log_error(RelationError("Le noeud '{}' n'existe pas".format(artere.AR_NOEUD_A), filename=Artere.filename, field="AR_NOEUD_A"))
+                
+            try:
+                artere.noeud_b = self.index[Artere][artere.AR_NOEUD_B]
+            except KeyError:
+                self.log_error(RelationError("Le noeud '{}' n'existe pas".format(artere.AR_NOEUD_B), filename=Artere.filename, field="AR_NOEUD_A"))
+        
+        # rattachement des equipements aux cables
+        for cable in self.index[Cable]:
+            try:
+                cable.equipement_a = self.index[Cable][cable.CA_EQ_A]
+            except KeyError:
+                self.log_error(RelationError("L'équipement '{}' n'existe pas".format(cable.AR_NOEUD_A), filename=Cable.filename, field="CA_EQ_A"))
+                
+            try:
+                cable.equipement_b = self.index[Cable][cable.CA_EQ_B]
+            except KeyError:
+                self.log_error(RelationError("L'équipement '{}' n'existe pas".format(cable.AR_NOEUD_B), filename=Cable.filename, field="CA_EQ_B"))
+                
+        if self.errors: 
+            return
+        
+        # Contrôler dans la base si des éléments portant ces codes existent à des emplacements différents
+        
+        # Contrôler l'emprise des ZAPBO
+        
+        
+        
+    
+if __name__ == "__main__":
+    from core.constants import MAIN
+    subject = MAIN / "work" / "STURNO_192AP1_REC_171211_OK"
+    report = Netgeo22DoeValidator.submit(subject)
+    print(report)

static/datachecker.js

@@ -1,7 +1,8 @@
 
 $(document).ready( function () {
     $('.datatable').DataTable( {
-        "scrollX": true
+        "scrollX": true,
+        "pageLength": 25
     } );
     
     $('#submit-form').submit(function(e) {

templates/report.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html lang="fr">
 <head>
-    <title>{{ report.title }}</title>
+    <title>{{ report['title'] }}</title>
     <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='font-awesome-4.7.0/css/font-awesome.min.css') }}">
 	<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='jquery-ui/jquery-ui.min.css') }}"/>
 	<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='DataTables/datatables.min.css') }}"/>
@@ -17,55 +17,55 @@
 		<a id="logo" href="\" title="Retour à l'accueil">
 			<img src="{{ url_for('static', filename='mn_logo_mini.png') }}" />
 		</a>
-		<h1>Rapport du Contrôle</h1>
+		<h1>{{ report['title'] }}</h1>
 	</header>
 	
 	<section id=main>
-    {% for reportfile in report.report_files %}
+		<h3>Check-Points</h3>
+		<ul>
+	    {% for chk in report['checkpoints'] %}
+	    	{%- if chk['valid'] -%}
+	    		<li><i class="fa fa-check valid"></i> <span>{{ chk['name'] }}</span></li>
+	    	{% else %}
+	    		<li><i class="fa fa-times error"></i> <span>{{ chk['name'] }}</span></li>
+	    	{% endif %}
+	    {% endfor %}
+	    </ul>
     
-        <h2><i class="fa fa-file"></i> {{ reportfile.file }}</h2>
         
-        {%- if reportfile.errors -%}
-        	<ul>
-		        {% for err in reportfile.errors %}
-		        	<li class="fa fa-exclamation-circle error"> {{ err }}</li>
-		        {% endfor %}
-	        </ul>
-        {% endif %}
-        
-        <table class="datatable cell-border display compact stripe">
-        	<thead>
+    	{%- if report['errors'] -%}
+	    	<h3>Détail des erreurs</h3>
+	       
+	       <table class="datatable cell-border display compact stripe">
+	       	<thead>
 	       		<tr>
-		       		<th>N°</th>
-		       		<th>Valide</th>
-		       		{% for header in reportfile.headers %}
-		       			<th>{{ header }}</th>
-		       		{% endfor %}
+		       		<th>Erreur</th>
+		       		<th>Criticité</th>
+		       		<th>Fichier</th>
+		       		<th>Champs</th>
+		       		<th>Description</th>
+		       		<th>Aide</th>
 	       		</tr>
-        	</thead>
-        	<tbody>
-       		{% for record in reportfile.records %}
-       			<tr>
-	       			<td>{{ record.index }}</td>
-	       			<td>
-	       				{%- if record.valid -%}
-	       					<i class="fa fa-check valid"></i>
-	       				{% else %}
-	       					<i class="fa fa-times error"></i>
-	       				{% endif %}
-	       			</td>
-	       			{% for field in record.fields %}
-	       				{%- if field.valid -%}
-							<td>{{ field.value }}</td>
-						{% else %}
-							<td class="error" title="{{ field.error }}"><i class="fa fa-exclamation-circle"></i> <span>{{ field.value }}</span></td>
-						{% endif %}
-					{% endfor %}
-       			</tr>
-       		{% endfor %}
-        	</tbody>
-        </table>
-    {% endfor %}
+	       	</thead>
+	       	<tbody>
+	      		{% for typeerr in report['errors'] %}
+	      			{% for err in report['errors'][typeerr]['list'] %}
+		      			<tr>
+			      			<td>{{ typeerr }}</td>
+			       			<td></td>
+			       			<td>{{ err['filename'] }}</td>
+			       			<td>{{ err['field'] }}</td>
+			       			<td>{{ err['message'] }}</td>
+			       			<td>
+			       				<i class="fa fa-question-circle" style="color: blue" "title="{{ report['errors'][typeerr]['help'] }}"></i>
+			       			</td>
+		      			</tr>
+		      		{% endfor %}
+	      		{% endfor %}
+	       	</tbody>
+	       </table>
+   		{% endif %}
+    
     </section>
     
 </body>

validators/netgeo_2_2_doe.py

@@ -1,232 +0,0 @@
-'''
-
-    Controle les données d'un dossier ou d'une archive contenant des données au format Netgeo 2.2
-
-    Python 3.7+
-
-    @author: olivier.massot, sept. 2018
-'''
-
-from core import gis
-from core.mn import ReferentielDb
-from core.validation import BaseValidator, BaseGeoModel, is_french_date
-
-xmin, xmax, ymin, ymax = 1341999, 1429750, 8147750, 8294000
-
-with ReferentielDb() as ref_db:
-    INSEE_VALIDES = [row[0] for row in ref_db.read("SELECT code_insee FROM sig_referentiel.admn_cd50_com;")]
-
-class Artere(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
-              'AR_CODE': {'type': 'string', 'maxlength': 26}, 
-              'AR_NOM': {'type': 'string', 'maxlength': 26}, 
-              'AR_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
-              'AR_LONG': {'type': 'float'}, 
-              'AR_ETAT': {'type': 'string', 'allowed': ['0', '1', '2', '3', '4']}, 
-              'AR_OCCP': {'type': 'string', 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
-              'AR_NOEUD_A': {'type': 'string', 'maxlength': 20}, 
-              'AR_NOEUD_B': {'type': 'string', 'maxlength': 20}, 
-              'AR_NB_FOUR': {'type': 'string', 'maxlength': 20}, 
-              'AR_FOU_DIS': {'type': 'integer'}, 
-              'AR_TYPE_FO': {'type': 'string', 'allowed': ['PVC', 'PEHD', 'SOUS-TUBAGE PEHD', 'SOUS-TUBAGE  SOUPLE', 'FACADE', 'AERIEN', 'ENCORBELLEMENT', 'AUTRE']}, 
-              'AR_TYFO_AI': {'type': 'string', 'allowed': ['PVC', 'PEH', 'TUB', 'FAC', 'ENC', 'APP']}, 
-              'AR_DIAM_FO': {'type': 'string', 'allowed': ['10', '14', '18', '25', '28', '32', '40', '45', '60', 'NUL']}, 
-              'AR_PRO_FOU': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)']}, 
-              'AR_FAB': {'type': 'string', 'maxlength': 100}, 'AR_REFFAB': {'type': 'string', 'maxlength': 100}, 
-              'AR_COULEUR': {'type': 'string', 'maxlength': 20}, 'AR_AIGUIL': {'type': 'string', 'allowed': ['OUI', 'NON']}, 
-              'AR_NBCABL': {'type': 'integer'}, 
-              'AR_PRO_CAB': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE']}, 
-              'AR_GEST_FO': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
-              'AR_UTIL_FO': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
-              'AR_DATE_IN': {'validator': is_french_date}, 
-              'AR_DATE_RE': {'validator': is_french_date}, 
-              'AR_REF_PLA': {'type': 'string', 'maxlength': 100}, 
-              'AR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
-              'AR_QLT_GEO': {'type': 'string', 'allowed': ['A', 'B', 'C']}, 
-              'AR_PRO_MD': {'type': 'string', 'allowed': ['MANCHE NUMERIQUE']}, 
-              'AR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
-              'AR_STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-    
-class Cable(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
-              'CA_CODE': {'type': 'string', 'maxlength': 18}, 
-              'CA_NOM': {'type': 'string', 'maxlength': 18}, 
-              'CA_NUMERO': {'type': 'string', 'maxlength': 17}, 
-              'CA_EMPRISE': {'type': 'string', 'maxlength': 10}, 
-              'CA_FAB': {'type': 'string', 'maxlength': 100}, 
-              'CA_REFFAB': {'type': 'string', 'maxlength': 100}, 
-              'CA_MODELE': {'type': 'string', 'maxlength': 10}, 
-              'CA_TYPE': {'type': 'string', 'maxlength': 10, 'allowed': ['AERIEN', 'IMMEUBLE', 'FACADE', 'MIXTE', 'SOUTERRAIN']}, 
-              'CA_TYPFCT': {'type': 'string', 'maxlength': 3, 'allowed': ['CDI', 'CTR', 'CBM', 'RAC', 'CBO']}, 
-              'CA_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
-              'CA_LONG': {'type': 'float'}, 'CA_EQ_A': {'type': 'string', 'maxlength': 18}, 
-              'CA_EQ_B': {'type': 'string', 'maxlength': 18}, 
-              'CA_DIAMETR': {'type': 'integer'}, 
-              'CA_COULEUR': {'type': 'string', 'maxlength': 20, 'allowed': ['NOIR', 'BLEU', 'BLANC']}, 
-              'CA_TECHNOL': {'type': 'string', 'maxlength': 17, 'allowed': ['G657A2_M6', 'G657A2_M12']}, 
-              'A_NB_FO_U': {'type': 'integer'}, 'CA_NB_FO_D': {'type': 'integer'}, 
-              'CA_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE']}, 
-              'CA_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['NUL']}, 
-              'CA_DATE_IN': {'validator': is_french_date}, 
-              'CA_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
-              'CA_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-
-class Equipement(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
-              'EQ_CODE': {'type': 'string', 'maxlength': 18}, 
-              'EQ_NOM': {'type': 'string', 'maxlength': 18}, 
-              'EQ_NOM_NOE': {'type': 'string', 'maxlength': 14}, 
-              'EQ_REF': {'type': 'string', 'maxlength': 100}, 
-              'EQ_EMPRISE': {'type': 'string', 'maxlength': 7}, 
-              'EQ_FABR': {'type': 'string', 'maxlength': 100}, 
-              'EQ_CAPFO': {'type': 'integer'}, 
-              'EQ_NBMXEQ': {'type': 'integer'}, 
-              'EQ_NBCAB': {'type': 'integer'}, 
-              'EQ_DIMENS': {'type': 'string', 'maxlength': 50}, 
-              'EQ_TYPEQ': {'type': 'string', 'maxlength': 100}, 
-              'EQ_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
-              'EQ_OCCP': {'type': 'string', 'maxlength': 3, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
-              'EQ_TYPE': {'type': 'string', 'maxlength': 3, 'allowed': ['PBO', 'PEC', 'BPE', 'BAI']}, 
-              'EQ_TYPSTRC': {'type': 'string', 'maxlength': 16, 'allowed': ['CHAMBRE', 'AERIEN', 'FACADE', 'COLONNE MONTANTE', 'PIED IMMEUBLE', 'DTIO']}, 
-              'EQ_TYPE_LQ': {'type': 'string', 'maxlength': 6, 'allowed': ['PBO', 'BPE JB', 'BPE JD', 'BAIDC', 'BAIOP']}, 
-              'EQ_TYPE_PH': {'type': 'string', 'maxlength': 24, 'allowed': ['PBO 6', 'PBO 12', 'BPE 12EP', 'BPE 24EP', 'BPE 48EP', 'BPE 72EP', 'BPE 96EP', 'BPE 144EP', 'BPE 288EP', 'BPE 576EP', 'BPE 720EP']}, 
-              'EQ_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'AUTRE (à préciser)', 'NUL']}, 
-              'EQ_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
-              'EQ_HAUT': {'type': 'float'}, 'EQ_DATE_IN': {'type': 'datetime'}, 
-              'EQ_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
-              'EQ_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-
-class Noeud(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POINT]}}, 
-              'NO_CODE': {'type': 'string', 'maxlength': 18}, 
-              'NO_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
-              'NO_NOM': {'type': 'string', 'maxlength': 20}, 
-              'NO_VOIE': {'type': 'string', 'maxlength': 100}, 
-              'NO_EMPRISE': {'type': 'string', 'maxlength': 10}, 
-              'NO_ETAT': {'type': 'string', 'maxlength': 1, 'allowed': ['0', '1', '2', '3', '4']}, 
-              'NO_OCCP': {'type': 'string', 'maxlength': 3, 'allowed': ['0', '1.1', '1.2', '2', '3', '4']}, 
-              'NO_TYPE': {'type': 'string', 'maxlength': 3, 'allowed': ['CHA', 'POT', 'LTE', 'SEM', 'FAC', 'OUV', 'IMM']}, 
-              'NO_TYPFCT': {'type': 'string', 'maxlength': 14, 'allowed': ['INTERCONNEXION', 'SATELLITE', 'PASSAGE', 'REGARD', 'INDETERMINE']}, 
-              'NO_TYPE_LQ': {'type': 'string', 'maxlength': 10, 'allowed': ['CHTIR', 'CHRACC', 'POT', 'NRO', 'PM', 'MIMO', 'FAC', 'OUV', 'IMM']}, 
-              'NO_TYPE_PH': {'type': 'string', 'maxlength': 20, 'allowed': ['CHAMBRE', 'POTEAU', 'ARMOIRE', 'SHELTER', 'BATIMENT', 'SITE MIMO', 'FACADE', 'OUVRAGE', 'IMMEUBLE']}, 
-              'NO_CODE_PH': {'type': 'string', 'maxlength': 20}, 
-              'NO_TECH_PS': {'type': 'string', 'maxlength': 4, 'allowed': ['COAX', 'CUT', 'ECL', 'ELEC', 'VP', 'OPT', 'NC']}, 
-              'NO_AMO': {'type': 'string', 'maxlength': 20}, 
-              'NO_PLINOX': {'type': 'string', 'maxlength': 3, 'allowed': ['OUI', 'NON']}, 
-              'NO_X': {'type': 'float'}, 
-              'NO_Y': {'type': 'float'}, 
-              'NO_PRO': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'AUTRE (à préciser)', 'NUL']}, 
-              'NO_GEST': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE', 'MANCHE TELECOM', 'COLLECTIVITE', 'ORANGE', 'ERDF', 'MANCHE FIBRE', 'AUTRE (à préciser)', 'NUL']}, 
-              'NO_HAUT': {'type': 'float'}, 
-              'NO_DATE_IN': {'validator': is_french_date}, 
-              'NO_REF_PLA': {'type': 'string', 'maxlength': 100}, 
-              'NO_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
-              'NO_QLT_GEO': {'type': 'string', 'maxlength': 1, 'allowed': ['A', 'B', 'C']}, 
-              'NO_PRO_MD': {'type': 'string', 'maxlength': 20, 'allowed': ['MANCHE NUMERIQUE']}, 
-              'NO_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
-              'NO_STATUT': {'type': 'string', 'maxlength': 14, 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-
-class Tranchee(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POLYLINE]}}, 
-              'TR_CODE': {'type': 'string', 'maxlength': 23}, 
-              'TR_NOM': {'type': 'string', 'maxlength': 23, 'tester': 'lambda t, x: x=t.TR_CODE'}, 
-              'TR_ID_INSE': {'type': 'string', 'allowed': INSEE_VALIDES}, 
-              'TR_VOIE': {'type': 'string', 'maxlength': 200}, 
-              'TR_TYP_IMP': {'type': 'string', 'allowed': ['ACCOTEMENT STABILISE', 'ACCOTEMENT NON STABILISE', 'CHAUSSEE LOURDE', 'CHAUSSEE LEGERE', 'FOSSE', 'TROTTOIR', 'ESPACE VERT', 'ENCORBELLEMENT']}, 
-              'TR_MOD_POS': {'type': 'string', 'allowed': ['TRADITIONNEL', 'MICRO TRANCHEE', 'FONCAGE 60', 'FONCAGE 90', 'FONCAGE 120', 'TRANCHEUSE', 'FORAGE URBAIN', 'FORAGE RURAL', 'ENCORBELLEMENT']}, 
-              'TR_MPOS_AI': {'type': 'string', 'allowed': ['TRA', 'ALL', 'FONCAGE 60', 'FON', 'FOR', 'ENC']}, 
-              'TR_LONG': {'type': 'float'}, 
-              'TR_LARG': {'type': 'float'}, 
-              'TR_REVET': {'type': 'string', 'allowed': ['SABLE', 'BICOUCHE', 'ENROBE', 'BETON', 'PAVE', 'TERRAIN NATUREL']}, 
-              'TR_CHARGE': {'type': 'float'}, 
-              'TR_GRILLAG': {'type': 'float'}, 
-              'TR_REMBLAI': {'type': 'string'}, 
-              'TR_PLYNOX': {'type': 'string', 'allowed': ['OUI', 'NON']}, 
-              'TR_PRO_VOI': {'type': 'string', 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
-              'TR_GEST_VO': {'type': 'string', 'allowed': ['COMMUNE', 'COMMUNAUTE DE COMMUNES', 'DEPARTEMENT', 'ETAT', 'PRIVE']}, 
-              'TR_SCHEMA': {'maxlength': 100, 'type': 'string'}, 
-              'TR_DATE_IN': {'validator': is_french_date}, 
-              'TR_SRC_GEO': {'type': 'string', 'maxlength': 50}, 
-              'TR_QLT_GEO': {'type': 'string', 'allowed': ['A', 'B', 'C']}, 
-              'TR_PRO_MD': {'type': 'string', 'maxlength': 20}, 
-              'TR_COMMENT': {'type': 'string', 'maxlength': 300, 'empty': True}, 
-              'TR_STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-
-class Zapbo(BaseGeoModel):
-    schema = {'geom': {'bbox': {type: 'list', 'items': [{'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}, 
-                                                        {'type': 'int', 'min': xmin, 'max': xmax}, 
-                                                        {'type': 'int', 'min': ymin, 'max': ymax}]}, 
-                       'parts': {}, 
-                       'points': {}, 
-                       'shapeType': {'allowed': [gis.POLYGON]}}, 
-              'ID_ZAPBO': {'type': 'string', 'maxlength': 10}, 
-              'COMMENTAIR': {'type': 'string', 'maxlength': 254, 'empty': True}, 
-              'STATUT': {'type': 'string', 'allowed': ['EN ETUDE', 'EN REALISATION', 'EN SERVICE', 'HORS SERVICE']}}
-
-
-class Netgeo22Validator(BaseValidator):
-    name = "Netgeo v2.2 DOE"
-    files = {"artere_geo.shp": Artere, "cable_geo.shp": Cable, "equipement_passif.shp": Equipement, "noeud_geo.shp":Noeud, "tranchee_geo.shp":Tranchee, "zapbo_geo.shp":Zapbo}
-    
-    @classmethod
-    def _load_file(cls, model, file_path):
-        
-        result = []
-        
-        with gis.ShapeFile(file_path) as sf:
-            fields = sf.fields()
-            for record in sf.records():
-                
-                data = dict(zip(fields, record.record))
-            
-                item = model(record.shape, **data)
-                result.append(item)
-        
-        return result
-    
-    @classmethod        
-    def _technical_validation(cls, dataset):
-        
-        # rattacher les noeuds aux artères
-        # rattacher les equipements aux cables
-        
-        return dataset
-
-    
-if __name__ == "__main__":
-    from core.constants import MAIN
-    subject = MAIN / "work" / "STURNO_192AP0_REC_COMPLEMENT_180822_OK.zip"
-    report = Netgeo22Validator.submit(subject)
-
-