update indexer and various changes

core/constants.py

@@ -13,3 +13,9 @@ VLC_PATH = APP_ROOT / 'core' / 'vlc-core'
 
 # Db
 DB_PATH = APP_ROOT / 'data' / 'default' / 'db.sqlite'
+
+DATA_DIR = APP_ROOT / 'data'
+
+LOGGER_NAME = "mew"
+LOG_DIR = DATA_DIR
+LOGGER_LEVEL = 0

core/exceptions.py

@@ -2,7 +2,9 @@
 
 
 
-
+class NotSupportedFile(Exception):
+    pass
 
 class YoutubeStreamNotFound(Exception):
-    pass
+    pass
+

core/file_utilities.py

@@ -19,12 +19,12 @@ def is_media_file_ext(ext):
     return ext.lower().lstrip('.') in [e.lower().lstrip('.') for e in media_exts()]
 
 
-HASHER = hashlib.md5()
-
-
 def hash_file(filename):
-    """ return a hash for the given file """
-    with open(filename, 'rb') as f:
-        buf = f.read()
-        HASHER.update(buf)
-    return HASHER.hexdigest()
+    """ return a SHA256 hash for the given file """
+    h = hashlib.sha256()
+    b = bytearray(128*1024)
+    mv = memoryview(b)
+    with open(filename, 'rb', buffering=0) as f:
+        for n in iter(lambda: f.readinto(mv), 0):
+            h.update(mv[:n])
+    return h.hexdigest()

core/indexer.py

@@ -3,10 +3,15 @@ import time
 import vlc
 from path import Path
 
+from core import logging_
+from core.exceptions import NotSupportedFile
 from core.file_utilities import is_media_file_ext, hash_file
+from core.logging_ import Logger
 from core.models import Track
 from core.repositories import MusicFolderRepository, TrackRepository
 
+logger = Logger.get()
+
 
 class Indexation:
     def __init__(self):
@@ -18,15 +23,12 @@ class Indexation:
         self.processed = set()
         self.t0 = time.time()
 
-    def start(self):
-        self.index = {t.hash: t for t in self.track_repo.get_all()}
-        self.processed = set()
-        self.t0 = time.time()
-        self.started = True
-
-    def index_file(self, music_folder, filename):
-        if not self.started:
-            self.start()
+    def index_file(self, music_folder, filename, force_update=False):
+        filename = Path(filename)
+        if not filename.exists():
+            raise FileNotFoundError(f"File not found: {filename}")
+        if not is_media_file_ext(filename.ext):
+            raise NotSupportedFile(f"File's extension {filename.ext} is not supported")
 
         vlc_media = vlc.Media(filename)
         vlc_media.parse()
@@ -38,10 +40,13 @@ class Indexation:
             print(" ... file already indexed, ignore: ", filename)
             return
 
-        if track_hash in self.index:
+        if self.index and track_hash in self.index:
             track = self.index[track_hash]
         else:
-            track = Track()
+            track = next(iter(self.track_repo.get_by('hash', track_hash)), None) or Track()
+
+        if track.id and not force_update:
+            return track
 
         track.profile_id = 0
         track.music_folder_id = music_folder.id
@@ -60,51 +65,62 @@ class Indexation:
 
         if track.id is not None:
             self.track_repo.update(track)
-            print('updated', filename)
+            logger.debug('Index - updated: %s' % filename)
         else:
             self.track_repo.create(track)
-            print('created', filename)
+            logger.debug('Index - created: %s' % filename)
         self.track_repo.commit()
 
-        self.processed.add(track_hash)
+        return track
 
-    def index_folder(self, music_folder):
-
-        if not self.started:
-            self.start()
+    def index_all(self, filter_music_folder_id=None, force_update=False):
+        self.index = {t.hash: t for t in self.track_repo.get_all()}
+        self.processed = set()
+        self.t0 = time.time()
+        self.started = True
 
-        music_folder_path = Path(music_folder.path)
+        if filter_music_folder_id:
+            music_folders = [self.music_folder_repo.get_by_id(filter_music_folder_id)]
+        else:
+            music_folders = self.music_folder_repo.get_all()
 
-        for filename in music_folder_path.walkfiles():
-            if not is_media_file_ext(filename.ext):
-                print('   ...  ignored ... ', filename)
-                continue
-            self.index_file(music_folder, filename)
+        # add and update tracks
+        for music_folder in music_folders:
+            music_folder_path = Path(music_folder.path)
 
-    def index_all(self):
-        self.start()
+            for filename in music_folder_path.walkfiles():
+                if not is_media_file_ext(filename.ext):
+                    continue
+                track = self.index_file(music_folder, filename, force_update=force_update)
+                self.processed.add(track.hash)
 
-        music_folders = self.music_folder_repo.get_all()
-        for music_folder in music_folders:
-            self.index_folder(music_folder)
+        # mark missings
+        for hash_, track in self.index.items():
+            if hash_ in self.processed:
+                continue
+            track.status = Track.STATUS_UNAVAILABLE
+            self.track_repo.update(track, True)
+            logger.debug('Index - marked as unavailable: %s' % track.path)
 
 
 class Indexer:
 
     @staticmethod
-    def index_file(music_folder, path):
+    def index_file(music_folder, path, force_update=False):
         indexation = Indexation()
-        indexation.index_file(music_folder, path)
+        indexation.index_file(music_folder, path, force_update=force_update)
 
     @staticmethod
-    def index_folder(music_folder):
+    def index_folder(music_folder, force_update=False):
+        logger.debug('** Start indexation on folder %s' % music_folder.id)
         indexation = Indexation()
-        indexation.index_folder(music_folder)
+        indexation.index_all(filter_music_folder_id=music_folder.id, force_update=force_update)
 
     @staticmethod
-    def index_all():
+    def index_all(force_update=False):
+        logger.debug('** Start complete indexation')
         indexation = Indexation()
-        indexation.index_all()
+        indexation.index_all(force_update=force_update)
 
 
 if __name__ == '__main__':

core/logging.yaml

@@ -13,7 +13,7 @@ formatters:
 handlers:
     console:
         class: logging.StreamHandler
-        level: INFO
+        level: DEBUG
         formatter: message_only
         stream: ext://sys.stdout
     file:
@@ -32,6 +32,6 @@ loggers:
         propagate: no
 
 root:
-    level: INFO
+    level: DEBUG
     handlers: [console]
     propagate: yes

core/logging_.py

@@ -1,31 +1,43 @@
-from datetime import datetime
+import traceback
 import logging.config
 import sys
 
 from path import Path
 import yaml
 
-LOG_DIR = Path(r"%appdata%\logs").expandvars()
-LOG_DIR.makedirs_p()
+from core.constants import LOG_DIR, LOGGER_NAME, LOGGER_LEVEL
 
 SYS_EXCEPT_HOOK = sys.excepthook
 
 
-def get(name="main", level=0, filename=""):
-    # charge la configuration du logging depuis le fichier 'logging.yaml'
-    configfile = Path(__file__).parent
-    with open(configfile / 'logging.yaml', 'rt') as f:
-        conf = yaml.load(f, Loader=yaml.FullLoader)
+class Logger:
+    _current = None
 
-    if level:
-        conf["loggers"][name]["level"] = level
+    @staticmethod
+    def get():
+        if Logger._current is None:
+            configfile = Path(__file__).parent
+            with open(configfile / 'logging.yaml', 'rt') as f:
+                conf = yaml.load(f, Loader=yaml.FullLoader)
 
-    if not filename:
-        filename = LOG_DIR / r'{}_{:%Y%m%d_%H%M}.log'.format(name, datetime.now())
-    conf["handlers"]["file"]["filename"] = filename
+            conf["loggers"][LOGGER_NAME]["level"] = LOGGER_LEVEL
 
-    logging.config.dictConfig(conf)
+            filename = LOG_DIR / f'{LOGGER_NAME}.log'
+            conf["handlers"]["file"]["filename"] = filename
 
-    logger = logging.getLogger(name)
-    logger.info("Log start written at {}".format(filename))
-    return logger
+            logging.config.dictConfig(conf)
+
+            logger = logging.getLogger(LOGGER_NAME)
+
+            # Configure how errors are processed
+            sys_err = sys.excepthook
+
+            def err_handler(typ, value, trace):
+                logger.error("{}\n{}\n{}".format(typ.__name__, value, ''.join(traceback.format_tb(trace))))
+                sys_err(typ, value, trace)
+
+            sys.excepthook = err_handler
+
+            Logger._current = logger
+
+        return Logger._current

core/settings.yml

@@ -0,0 +1,3 @@
+
+current_profile: 'default'
+

core/youtube.py

@@ -8,7 +8,7 @@ from core.exceptions import YoutubeStreamNotFound
 
 MAX_ATTEMPTS = 5
 
-logger = logging_.get("mew")
+logger = logging_.get()
 
 
 class YoutubeDownloader:

main.py

@@ -13,6 +13,7 @@ from PyQt5.Qt import QApplication
 from PyQt5.QtWidgets import QMessageBox
 
 from core import logging_
+from core.logging_ import Logger
 from ui.window import MainWindow
 
 try:
@@ -22,13 +23,12 @@ try:
 except:
     pass
 
-logger = logging_.get("mew", level=10)
+logger = Logger.get()
 
 # Configure how errors are processed
 sys_err = sys.excepthook
 def err_handler(typ, value, trace):
     QApplication.restoreOverrideCursor()
-    logger.error("{}\n{}\n{}".format(typ.__name__, value, ''.join(traceback.format_tb(trace))))
     QMessageBox.critical(mainw, "Erreur: {}".format(typ.__name__), """{}""".format(value))
     sys_err(typ, value, trace)
 sys.excepthook = err_handler

ui/qt/main.ui

@@ -87,6 +87,9 @@
       </item>
       <item>
        <widget class="QStackedWidget" name="stack">
+        <property name="currentIndex">
+         <number>4</number>
+        </property>
         <widget class="QWidget" name="page_1">
          <layout class="QHBoxLayout" name="horizontalLayout_2">
           <item>
@@ -151,13 +154,149 @@
          <layout class="QHBoxLayout" name="horizontalLayout_5">
           <item>
            <layout class="QVBoxLayout" name="verticalLayout_6">
+            <property name="leftMargin">
+             <number>20</number>
+            </property>
+            <property name="topMargin">
+             <number>20</number>
+            </property>
+            <property name="rightMargin">
+             <number>20</number>
+            </property>
+            <property name="bottomMargin">
+             <number>20</number>
+            </property>
             <item>
              <widget class="QLabel" name="label_5">
               <property name="text">
-               <string>Page 5</string>
+               <string>Mes dossiers de musique</string>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QTableWidget" name="musicFoldersTable">
+              <property name="maximumSize">
+               <size>
+                <width>16777215</width>
+                <height>400</height>
+               </size>
+              </property>
+              <property name="editTriggers">
+               <set>QAbstractItemView::NoEditTriggers</set>
+              </property>
+              <property name="showDropIndicator" stdset="0">
+               <bool>false</bool>
+              </property>
+              <property name="dragDropOverwriteMode">
+               <bool>false</bool>
+              </property>
+              <property name="selectionMode">
+               <enum>QAbstractItemView::SingleSelection</enum>
+              </property>
+              <property name="selectionBehavior">
+               <enum>QAbstractItemView::SelectRows</enum>
+              </property>
+              <property name="showGrid">
+               <bool>false</bool>
+              </property>
+              <property name="cornerButtonEnabled">
+               <bool>false</bool>
               </property>
+              <attribute name="horizontalHeaderCascadingSectionResizes">
+               <bool>true</bool>
+              </attribute>
+              <attribute name="horizontalHeaderDefaultSectionSize">
+               <number>70</number>
+              </attribute>
+              <attribute name="horizontalHeaderStretchLastSection">
+               <bool>true</bool>
+              </attribute>
+              <attribute name="verticalHeaderVisible">
+               <bool>false</bool>
+              </attribute>
+              <column>
+               <property name="text">
+                <string>id</string>
+               </property>
+              </column>
+              <column>
+               <property name="text">
+                <string>Statut</string>
+               </property>
+              </column>
+              <column>
+               <property name="text">
+                <string>Emplacement</string>
+               </property>
+              </column>
              </widget>
             </item>
+            <item>
+             <layout class="QHBoxLayout" name="horizontalLayout_7">
+              <property name="sizeConstraint">
+               <enum>QLayout::SetMinimumSize</enum>
+              </property>
+              <item>
+               <widget class="QPushButton" name="musicFoldersRemoveButton">
+                <property name="minimumSize">
+                 <size>
+                  <width>0</width>
+                  <height>32</height>
+                 </size>
+                </property>
+                <property name="text">
+                 <string>Supprimer</string>
+                </property>
+               </widget>
+              </item>
+              <item>
+               <spacer name="horizontalSpacer">
+                <property name="orientation">
+                 <enum>Qt::Horizontal</enum>
+                </property>
+                <property name="sizeHint" stdset="0">
+                 <size>
+                  <width>40</width>
+                  <height>20</height>
+                 </size>
+                </property>
+               </spacer>
+              </item>
+              <item>
+               <widget class="QPushButton" name="musicFoldersAddButton">
+                <property name="minimumSize">
+                 <size>
+                  <width>128</width>
+                  <height>32</height>
+                 </size>
+                </property>
+                <property name="text">
+                 <string>Ajouter</string>
+                </property>
+               </widget>
+              </item>
+             </layout>
+            </item>
+            <item>
+             <widget class="Line" name="line">
+              <property name="orientation">
+               <enum>Qt::Horizontal</enum>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <spacer name="verticalSpacer">
+              <property name="orientation">
+               <enum>Qt::Vertical</enum>
+              </property>
+              <property name="sizeHint" stdset="0">
+               <size>
+                <width>20</width>
+                <height>40</height>
+               </size>
+              </property>
+             </spacer>
+            </item>
            </layout>
           </item>
          </layout>