update indexer (ongoing)

core/indexer.py

@@ -1,9 +1,10 @@
 import time
+from queue import Queue
+from threading import Thread, Lock
 
 import vlc
 from path import Path
 
-from core import logging_
 from core.exceptions import NotSupportedFile
 from core.file_utilities import is_media_file_ext, hash_file
 from core.logging_ import Logger
@@ -13,43 +14,68 @@ from core.repositories import MusicFolderRepository, TrackRepository
 logger = Logger.get()
 
 
-class Indexation:
+class Discoverer(Thread):
+    def __init__(self, indexer):
+        Thread.__init__(self)
+        self.indexer = indexer
+
+    def run(self):
+        music_folder_repo = MusicFolderRepository()
+        while 1:
+            for music_folder in music_folder_repo.get_all():
+                music_folder_path = Path(music_folder.path)
+
+                for filename in music_folder_path.walkfiles():
+                    if is_media_file_ext(filename.ext):
+                        self.indexer.put(filename)
+            time.sleep(1)
+
+
+class Cleaner(Thread):
+    def __init__(self, indexer):
+        Thread.__init__(self)
+        self.indexer = indexer
+
+    def run(self):
+        track_repo = TrackRepository()
+        while 1:
+            pass
+
+class Indexer(Thread):
+
     def __init__(self):
-        self.started = False
-        self.music_folder_repo = MusicFolderRepository()
-        self.track_repo = TrackRepository()
-        self.music_folders = []
-        self.index = {}
-        self.processed = set()
-        self.t0 = time.time()
-
-    def index_file(self, music_folder, filename, force_update=False):
+        Thread.__init__(self)
+        self.queue = Queue()
+        discoverer = Discoverer(self)
+
+    def run(self):
+        while 1:
+            filename = self.queue.get()
+            self.index(filename)
+
+    def put(self, filename):
+        self.queue.put(filename)
+
+    @staticmethod
+    def index(filename):
         filename = Path(filename)
         if not filename.exists():
             raise FileNotFoundError(f"File not found: {filename}")
         if not is_media_file_ext(filename.ext):
             raise NotSupportedFile(f"File's extension {filename.ext} is not supported")
 
+        track_repo = TrackRepository()
+
         vlc_media = vlc.Media(filename)
         vlc_media.parse()
         track_infos = vlc_media.get_tracks_info()
 
         track_hash = hash_file(filename)
 
-        if track_hash in self.processed:
-            print(" ... file already indexed, ignore: ", filename)
-            return
-
-        if self.index and track_hash in self.index:
-            track = self.index[track_hash]
-        else:
-            track = next(iter(self.track_repo.get_by('hash', track_hash)), None) or Track()
-
-        if track.id and not force_update:
-            return track
+        track = track_repo.get_by_hash(track_hash)
+        if not track:
+            track = Track()
 
-        track.profile_id = 0
-        track.music_folder_id = music_folder.id
         track.title = vlc_media.get_meta(vlc.Meta.Title)
         track.format = filename.ext
         track.artist = vlc_media.get_meta(vlc.Meta.AlbumArtist) or vlc_media.get_meta(vlc.Meta.Artist)
@@ -63,65 +89,12 @@ class Indexation:
         track.path = filename
         track.hash = track_hash
 
-        if track.id is not None:
-            self.track_repo.update(track)
-            logger.debug('Index - updated: %s' % filename)
-        else:
-            self.track_repo.create(track)
-            logger.debug('Index - created: %s' % filename)
-        self.track_repo.commit()
+        if track.id is None:
+            track_repo.create(track)
 
-        return track
-
-    def index_all(self, filter_music_folder_id=None, force_update=False):
-        self.index = {t.hash: t for t in self.track_repo.get_all()}
-        self.processed = set()
-        self.t0 = time.time()
-        self.started = True
-
-        if filter_music_folder_id:
-            music_folders = [self.music_folder_repo.get_by_id(filter_music_folder_id)]
-        else:
-            music_folders = self.music_folder_repo.get_all()
-
-        # add and update tracks
-        for music_folder in music_folders:
-            music_folder_path = Path(music_folder.path)
-
-            for filename in music_folder_path.walkfiles():
-                if not is_media_file_ext(filename.ext):
-                    continue
-                track = self.index_file(music_folder, filename, force_update=force_update)
-                self.processed.add(track.hash)
-
-        # mark missings
-        for hash_, track in self.index.items():
-            if hash_ in self.processed:
-                continue
-            track.status = Track.STATUS_UNAVAILABLE
-            self.track_repo.update(track, True)
-            logger.debug('Index - marked as unavailable: %s' % track.path)
-
-
-class Indexer:
-
-    @staticmethod
-    def index_file(music_folder, path, force_update=False):
-        indexation = Indexation()
-        indexation.index_file(music_folder, path, force_update=force_update)
-
-    @staticmethod
-    def index_folder(music_folder, force_update=False):
-        logger.debug('** Start indexation on folder %s' % music_folder.id)
-        indexation = Indexation()
-        indexation.index_all(filter_music_folder_id=music_folder.id, force_update=force_update)
-
-    @staticmethod
-    def index_all(force_update=False):
-        logger.debug('** Start complete indexation')
-        indexation = Indexation()
-        indexation.index_all(force_update=force_update)
+        logger.debug('Index - updated: %s' % filename)
+        track_repo.commit()
 
 
 if __name__ == '__main__':
-    Indexer.index_all()
+    pass

core/models.py

@@ -29,12 +29,6 @@ class MusicFolder(Model):
     status = Column(Integer, default=0)
 
 
-class Profile(Model):
-    __tablename__ = 'Profiles'
-
-    name = Column(String, nullable=False)
-
-
 class Tag(Model):
     __tablename__ = 'Tags'
 
@@ -50,9 +44,6 @@ class Track(Model):
     STATUS_UNAVAILABLE = 2
     STATUS_UNREADABLE = 3
 
-    profile_id = Column(Integer)
-    music_folder_id = Column(Integer)
-
     title = Column(String)
     format = Column(String)
     artist = Column(String)
@@ -65,7 +56,6 @@ class Track(Model):
     status = Column(Integer, nullable=False, default=0)
     path = Column(String, nullable=False)
     hash = Column(String, nullable=False)
-    origin = Column(String)
 
 
 class TrackTag(Model):

core/repositories.py

@@ -68,6 +68,9 @@ class TagRepository(Repository):
 class TrackRepository(Repository):
     MODEL_CLS = Track
 
+    def get_by_hash(self, hash_):
+        return self.query().filter(hash == hash_).first()
+
     def get_by_tag_id(self, tag_id):
         return self.session.query(Track)\
             .join(TrackTag, Track.id == TrackTag.track_id)\

core/scanner.py

@@ -1,17 +0,0 @@
-from core import constants
-
-
-class Scanner:
-    def __init__(self):
-        pass
-
-    def scan(self):
-        disco = {}
-        for dir_ in constants.MUSIC_FOLDERS:
-            disco[dir_] = []
-
-            for f in dir_.files():
-                if f.ext not in constants.SUPPORTED_EXTENSIONS:
-                    continue
-
-                disco[dir_].append(f)

core/settings.yml

@@ -1,3 +0,0 @@
-
-current_profile: 'default'
-