4 files changed, 80 insertions(+), 48 deletions(-)

M melon/home/new.py
M melon/models/__init__.py
M melon/servers/__init__.py
M melon/servers/utils.py

@@ 2,7 2,7 @@ import sys
 import gi
 gi.require_version('Gtk', '4.0')
 gi.require_version('Adw', '1')
-from gi.repository import Gtk, Adw, GLib
+from gi.repository import Gtk, Adw, GLib, Gio, GObject
 import threading
 from datetime import datetime
 


@@ 15,20 15,13 @@ from melon.models import get_subscribed_channels, get_app_settings
 from melon.models import get_cached_feed, clear_cached_feed, update_cached_feed, get_last_feed_refresh
 
 class NewFeed(ViewStackPage):
-    def update(self):
-        news = get_cached_feed()
-        if len(news) == 0:
-            # long task:
-            # NOTE: fetch_home_feed automatically dismisses channels
-            # if the server isn't available (i.e because it has been disabled)
-            news = fetch_home_feed()
-            update_cached_feed(news)
-        # reverse the feed, because the latest (highest uts) should be on top
-        news.reverse()
+    def update(self, news):
+        # both subscreens need the refresh button
         refresh_btn = IconButton("Refresh", "view-refresh-symbolic")
         refresh_btn.connect("clicked", lambda a: self.reload())
-        # render data
-        if len(news) == 0:
+
+        if not news:
+            # empty home feed
             status = Adw.StatusPage()
             status.set_title("*crickets chirping*")
             subs = get_subscribed_channels()


@@ 48,51 41,52 @@ class NewFeed(ViewStackPage):
             status.set_child(box)
             self.inner.set_child(status)
         else:
+            # feed not empty
             app_settings = get_app_settings()
             results = Adw.PreferencesPage()
-            title = Adw.PreferencesGroup()
             last_refresh = get_last_feed_refresh()
             if last_refresh is None:
                 last_refresh = "Never"
             else:
                 last_refresh = datetime.fromtimestamp(last_refresh).strftime("%c")
             refresh_info = f" (Last refresh: {last_refresh})"
+            title = Adw.PreferencesGroup()
             title.set_title("What's new" + refresh_info)
             title.set_description("These are the latest videos of channels you follow")
             title.set_header_suffix(refresh_btn)
-            # append box and then add results
-            # this keeps the loading animation shorter
-            # and newer videos will be visible fairly quickly
             results.add(title)
             self.inner.set_child(results)
-            for date, content in group_by_date(news).items():
-                # because we have to join the thread before reset
-                # we need a way to skip the old drawing process
-                # as it would take way longer otherwise
-                if self.stop_update:
-                    break
+            for date, content in news:
                 group = Adw.PreferencesGroup()
                 group.set_title(date)
-                added = False
+                results.add(group)
                 for resource in content:
-                    # faster thread breakout
-                    if self.stop_update:
-                        break
                     group.add(AdaptiveFeedItem(resource, app_settings["show_images_in_feed"]))
-                    # add preference group to results when first item is added to
-                    if not added:
-                        added = True
-                        results.add(group)
         # return false, because otherwise ide_add would run multiple times
         return False
-    stop_update = False
+
+    def schedule(self):
+        news = get_cached_feed()
+        if not news:
+            # clear the feed again
+            # in case this wasn't calles from reload
+            # because we do NOT want duplicate entries
+            clear_cached_feed()
+            # long task:
+            # NOTE: fetch_home_feed automatically dismisses channels
+            # if the server isn't available (i.e because it has been disabled)
+            news = fetch_home_feed()
+            update_cached_feed(news)
+        # only show last 100 videos
+        news = news[-100:]
+        # reverse the feed, because the latest (highest uts) should be on top
+        news.reverse()
+        # group the data by date
+        group = group_by_date(news)
+        GLib.idle_add(self.update, group.items())
+
     thread = None
     def do_update(self):
-        if not self.thread is None:
-            self.stop_update = True
-            # wait for old thread to finish
-            self.thread.join()
-        self.stop_update = False
         # show spinner
         spinner = Gtk.Spinner()
         spinner.set_size_request(50, 50)


@@ 101,7 95,7 @@ class NewFeed(ViewStackPage):
         cb.set_center_widget(spinner)
         self.inner.set_child(cb)
         # start thread
-        self.thread = threading.Thread(target=self.update)
+        self.thread = threading.Thread(target=self.schedule)
         self.thread.daemon = True
         self.thread.start()
 

@@ 645,9 645,8 @@ def get_cached_feed():
     """
     conn = connect_to_db()
     results = conn.execute("""
-    SELECT timestamp, server, id, url, title, description, thumbnail, channel_id, channel_name
-    FROM news,videos
-    WHERE news.video_id = id AND news.video_server = server
+    SELECT DISTINCT timestamp, server, id, url, title, description, thumbnail, channel_id, channel_name
+    FROM news join videos on news.video_id = videos.id AND news.video_server = videos.server
     """).fetchall()
     return [ (Video(d[1], d[3], d[2], d[4], (d[8], d[7]), d[5], d[6]), d[0]) for d in results ]
 def clear_cached_feed():


@@ 675,7 674,7 @@ def update_cached_feed(ls: list[(Video, int)]):
         uts = entry[1]
         ensure_video(vid)
         execute_sql(conn, """
-        INSERT INTO news
+        INSERT OR REPLACE INTO news
         VALUES (?, ?, ?)
         """,
         (uts, vid.id, vid.server))

@@ 1,5 1,10 @@
 from enum import Flag,Enum,auto
 from abc import ABC,abstractmethod
+import gi
+gi.require_version('Gtk', '4.0')
+gi.require_version('Adw', '1')
+from gi.repository import GObject
+
 from melon.servers.loader import server_finder
 from melon.import_providers import ImportProvider
 


@@ 60,7 65,7 @@ class Preference:
         self.default = default
         self.value = value
 
-class Resource:
+class Resource(GObject.Object):
     # external url of the resource
     # used for "Open in Browser"
     url: str


@@ 71,6 76,7 @@ class Resource:
     # id of server from where the resource comes
     server:str
     def __init__(self, server, url, id):
+        super().__init__()
         self.server = server
         self.id = id
         self.url = url

@@ 47,6 47,37 @@ def get_server_instance(server) -> Server:
             instance.settings[key].value = dt
     return instance
 
+import threading
+class NewsWorkerPool:
+    threads=[]
+    tasks=[]
+    results=[]
+    def __init__(self, size, tasks):
+        self.tasks = tasks
+        for i in range(size):
+            thread = threading.Thread(target=self.run, name=str(i))
+            thread.daemon = True
+            thread.start()
+            self.threads.append(thread)
+    def run(self):
+        name = threading.currentThread().getName()
+        try:
+            task = self.tasks.pop()
+            instance = task[0]
+            channel = task[1]
+            print(f"{name} obtained task")
+            dts = instance.get_timeline(channel.id)
+            for entry in dts:
+                self.results.append(entry)
+            self.run()
+        except Exception as e:
+            print(f"{name} done")
+            print(e)
+            return
+    def wait(self):
+        for thread in self.threads:
+            thread.join()
+        print("all done")
 
 def fetch_home_feed() -> list[Resource]:
     subs = get_subscribed_channels()


@@ 58,17 89,19 @@ def fetch_home_feed() -> list[Resource]:
         else:
             db[sub.server] = [ sub ]
     servers = get_allowed_servers_list(get_app_settings())
-    feed = []
+    # generate list of tasks
+    tasks = []
     for server in servers:
         if not server["id"] in db:
             continue
         instance = get_server_instance(server)
         channels = db[server["id"]]
-        for channel in channels:
-            dts = instance.get_timeline(channel.id)
-            for entry in dts:
-                feed.append(entry)
+        tasks = tasks + [ (instance, channel) for channel in channels ]
+    pool = NewsWorkerPool(10, tasks)
+    pool.wait()
+    feed = pool.results
     feed.sort(key=lambda dts: dts[1])
+    print("returning")
     return feed
 def group_by_date(dataset):
     db = {}