From 46e84e77bb23ee48fe311a9febc117b53d20b2ef Mon Sep 17 00:00:00 2001 From: Jakob Meier Date: Thu, 29 Feb 2024 14:59:59 +0100 Subject: [PATCH] render new-uploads list on main thread + only show last 100 videos FIXES: duplicates in news feed --- melon/home/new.py | 70 ++++++++++++++++++--------------------- melon/models/__init__.py | 7 ++-- melon/servers/__init__.py | 8 ++++- melon/servers/utils.py | 43 +++++++++++++++++++++--- 4 files changed, 80 insertions(+), 48 deletions(-) diff --git a/melon/home/new.py b/melon/home/new.py index 2e5d520..d7262f2 100644 --- a/melon/home/new.py +++ b/melon/home/new.py @@ -2,7 +2,7 @@ import sys import gi gi.require_version('Gtk', '4.0') gi.require_version('Adw', '1') -from gi.repository import Gtk, Adw, GLib +from gi.repository import Gtk, Adw, GLib, Gio, GObject import threading from datetime import datetime @@ -15,20 +15,13 @@ from melon.models import get_subscribed_channels, get_app_settings from melon.models import get_cached_feed, clear_cached_feed, update_cached_feed, get_last_feed_refresh class NewFeed(ViewStackPage): - def update(self): - news = get_cached_feed() - if len(news) == 0: - # long task: - # NOTE: fetch_home_feed automatically dismisses channels - # if the server isn't available (i.e because it has been disabled) - news = fetch_home_feed() - update_cached_feed(news) - # reverse the feed, because the latest (highest uts) should be on top - news.reverse() + def update(self, news): + # both subscreens need the refresh button refresh_btn = IconButton("Refresh", "view-refresh-symbolic") refresh_btn.connect("clicked", lambda a: self.reload()) - # render data - if len(news) == 0: + + if not news: + # empty home feed status = Adw.StatusPage() status.set_title("*crickets chirping*") subs = get_subscribed_channels() @@ -48,51 +41,52 @@ class NewFeed(ViewStackPage): status.set_child(box) self.inner.set_child(status) else: + # feed not empty app_settings = get_app_settings() results = Adw.PreferencesPage() - title = Adw.PreferencesGroup() last_refresh = get_last_feed_refresh() if last_refresh is None: last_refresh = "Never" else: last_refresh = datetime.fromtimestamp(last_refresh).strftime("%c") refresh_info = f" (Last refresh: {last_refresh})" + title = Adw.PreferencesGroup() title.set_title("What's new" + refresh_info) title.set_description("These are the latest videos of channels you follow") title.set_header_suffix(refresh_btn) - # append box and then add results - # this keeps the loading animation shorter - # and newer videos will be visible fairly quickly results.add(title) self.inner.set_child(results) - for date, content in group_by_date(news).items(): - # because we have to join the thread before reset - # we need a way to skip the old drawing process - # as it would take way longer otherwise - if self.stop_update: - break + for date, content in news: group = Adw.PreferencesGroup() group.set_title(date) - added = False + results.add(group) for resource in content: - # faster thread breakout - if self.stop_update: - break group.add(AdaptiveFeedItem(resource, app_settings["show_images_in_feed"])) - # add preference group to results when first item is added to - if not added: - added = True - results.add(group) # return false, because otherwise ide_add would run multiple times return False - stop_update = False + + def schedule(self): + news = get_cached_feed() + if not news: + # clear the feed again + # in case this wasn't calles from reload + # because we do NOT want duplicate entries + clear_cached_feed() + # long task: + # NOTE: fetch_home_feed automatically dismisses channels + # if the server isn't available (i.e because it has been disabled) + news = fetch_home_feed() + update_cached_feed(news) + # only show last 100 videos + news = news[-100:] + # reverse the feed, because the latest (highest uts) should be on top + news.reverse() + # group the data by date + group = group_by_date(news) + GLib.idle_add(self.update, group.items()) + thread = None def do_update(self): - if not self.thread is None: - self.stop_update = True - # wait for old thread to finish - self.thread.join() - self.stop_update = False # show spinner spinner = Gtk.Spinner() spinner.set_size_request(50, 50) @@ -101,7 +95,7 @@ class NewFeed(ViewStackPage): cb.set_center_widget(spinner) self.inner.set_child(cb) # start thread - self.thread = threading.Thread(target=self.update) + self.thread = threading.Thread(target=self.schedule) self.thread.daemon = True self.thread.start() diff --git a/melon/models/__init__.py b/melon/models/__init__.py index 73b0054..8220b45 100644 --- a/melon/models/__init__.py +++ b/melon/models/__init__.py @@ -645,9 +645,8 @@ def get_cached_feed(): """ conn = connect_to_db() results = conn.execute(""" - SELECT timestamp, server, id, url, title, description, thumbnail, channel_id, channel_name - FROM news,videos - WHERE news.video_id = id AND news.video_server = server + SELECT DISTINCT timestamp, server, id, url, title, description, thumbnail, channel_id, channel_name + FROM news join videos on news.video_id = videos.id AND news.video_server = videos.server """).fetchall() return [ (Video(d[1], d[3], d[2], d[4], (d[8], d[7]), d[5], d[6]), d[0]) for d in results ] def clear_cached_feed(): @@ -675,7 +674,7 @@ def update_cached_feed(ls: list[(Video, int)]): uts = entry[1] ensure_video(vid) execute_sql(conn, """ - INSERT INTO news + INSERT OR REPLACE INTO news VALUES (?, ?, ?) """, (uts, vid.id, vid.server)) diff --git a/melon/servers/__init__.py b/melon/servers/__init__.py index 1b9f42e..9d7515b 100644 --- a/melon/servers/__init__.py +++ b/melon/servers/__init__.py @@ -1,5 +1,10 @@ from enum import Flag,Enum,auto from abc import ABC,abstractmethod +import gi +gi.require_version('Gtk', '4.0') +gi.require_version('Adw', '1') +from gi.repository import GObject + from melon.servers.loader import server_finder from melon.import_providers import ImportProvider @@ -60,7 +65,7 @@ class Preference: self.default = default self.value = value -class Resource: +class Resource(GObject.Object): # external url of the resource # used for "Open in Browser" url: str @@ -71,6 +76,7 @@ class Resource: # id of server from where the resource comes server:str def __init__(self, server, url, id): + super().__init__() self.server = server self.id = id self.url = url diff --git a/melon/servers/utils.py b/melon/servers/utils.py index 2ba72c4..7a1a171 100644 --- a/melon/servers/utils.py +++ b/melon/servers/utils.py @@ -47,6 +47,37 @@ def get_server_instance(server) -> Server: instance.settings[key].value = dt return instance +import threading +class NewsWorkerPool: + threads=[] + tasks=[] + results=[] + def __init__(self, size, tasks): + self.tasks = tasks + for i in range(size): + thread = threading.Thread(target=self.run, name=str(i)) + thread.daemon = True + thread.start() + self.threads.append(thread) + def run(self): + name = threading.currentThread().getName() + try: + task = self.tasks.pop() + instance = task[0] + channel = task[1] + print(f"{name} obtained task") + dts = instance.get_timeline(channel.id) + for entry in dts: + self.results.append(entry) + self.run() + except Exception as e: + print(f"{name} done") + print(e) + return + def wait(self): + for thread in self.threads: + thread.join() + print("all done") def fetch_home_feed() -> list[Resource]: subs = get_subscribed_channels() @@ -58,17 +89,19 @@ def fetch_home_feed() -> list[Resource]: else: db[sub.server] = [ sub ] servers = get_allowed_servers_list(get_app_settings()) - feed = [] + # generate list of tasks + tasks = [] for server in servers: if not server["id"] in db: continue instance = get_server_instance(server) channels = db[server["id"]] - for channel in channels: - dts = instance.get_timeline(channel.id) - for entry in dts: - feed.append(entry) + tasks = tasks + [ (instance, channel) for channel in channels ] + pool = NewsWorkerPool(10, tasks) + pool.wait() + feed = pool.results feed.sort(key=lambda dts: dts[1]) + print("returning") return feed def group_by_date(dataset): db = {} -- 2.38.5