diff --git a/cry/cli.py b/cry/cli.py index beaed84..63601ba 100644 --- a/cry/cli.py +++ b/cry/cli.py @@ -96,7 +96,7 @@ def subscribe(url, literal): result = d # Check to see if this URL is already in the database. - existing = db.load_meta(result.meta.url) + existing = db.load_feed(result.meta.url) # TODO: Replace with 'load_meta'? if existing is not None: click.echo(f"This feed already exists (as {result.meta.url})") return 1 @@ -129,7 +129,7 @@ def import_opml(opml_file): click.echo(f"{url} does not seem to be a feed, skipping...") continue - existing = db.load_meta(meta.url) + existing = db.load_feed(meta.url) # TODO: Replace with 'load_meta'? if existing is not None: LOG.info(f"{url} already exists (as {meta.url})") continue @@ -151,11 +151,11 @@ def refresh(url): db = database.Database.local() if url: - f = db.load_meta(url) + f = db.load_feed(url) # TODO: Replace with 'load_meta'? if f is None: click.echo(f"Not subscribed to {url}") return 1 - feeds = [f] + feeds = [f.meta] else: feeds = db.load_all_meta() @@ -239,13 +239,11 @@ def unsubscribe(url): `list` command.) """ db = database.Database.local() - meta = db.load_meta(url) - if meta is None: + count = db.set_feed_status(url, feed.FEED_STATUS_UNSUBSCRIBED) + if count == 0: click.echo(f"Not subscribed to feed {url}") return 1 - db.update_feed_status(meta, feed.FEED_STATUS_UNSUBSCRIBED) - @cli.command("serve") def serve(): @@ -264,11 +262,6 @@ def serve(): Subscribed Feeds -

Feeds

""" @@ -279,19 +272,17 @@ def serve(): ago = f" ({f.entries[0].time_ago()})" else: ago = "" - buffer.write(f"
") buffer.write(f'

{feed_title}{ago}

') + buffer.write(f"
") if len(f.entries) > 0: - buffer.write(f"") else: buffer.write("No entries...") - buffer.write(f"
") # feed + buffer.write(f"
") buffer.flush() text = buffer.getvalue() response = text.encode("utf-8") diff --git a/cry/database.py b/cry/database.py index a73e971..1834245 100644 --- a/cry/database.py +++ b/cry/database.py @@ -115,11 +115,23 @@ class Database: def get_property(self, prop: str, default=None) -> typing.Any: with self.db: - return self._get_property(prop, default) + cursor = self.db.execute( + "SELECT value FROM properties WHERE name=?", (prop,) + ) + result = cursor.fetchone() + if result is None: + return default + return result[0] def set_property(self, prop: str, value): with self.db: - return self._set_property(prop, value) + self.db.execute( + """ + INSERT INTO properties (name, value) VALUES (?, ?) + ON CONFLICT DO UPDATE SET value=excluded.value + """, + (prop, value), + ) def ensure_database_schema(self): with self.db: @@ -131,15 +143,15 @@ class Database: ) """ ) - version = int(self._get_property("version", 0)) + version = int(self.get_property("version", 0)) for script in SCHEMA_STATEMENTS[version:]: for statement in script.split(";"): try: self.db.execute(statement) except Exception as e: raise Exception(f"Error executing:\n{statement}") from e - self._set_property("version", len(SCHEMA_STATEMENTS)) - self._set_property("origin", self.origin) + self.set_property("version", len(SCHEMA_STATEMENTS)) + self.set_property("origin", self.origin) def load_all_meta(self) -> list[feed.FeedMeta]: with self.db: @@ -246,7 +258,7 @@ class Database: return feeds - def load_meta(self, url: str) -> feed.FeedMeta | None: + def load_feed(self, url: str) -> feed.Feed | None: with self.db: cursor = self.db.execute( """ @@ -255,7 +267,9 @@ class Database: retry_after_ts, status, etag, - modified + modified, + title, + link FROM feeds WHERE url=? """, @@ -266,8 +280,8 @@ class Database: if row is None: return None - last_fetched_ts, retry_after_ts, status, etag, modified = row - return feed.FeedMeta( + last_fetched_ts, retry_after_ts, status, etag, modified, title, link = row + meta = feed.FeedMeta( url=url, last_fetched_ts=last_fetched_ts, retry_after_ts=retry_after_ts, @@ -276,6 +290,27 @@ class Database: modified=modified, ) + cursor = self.db.execute( + """ + SELECT + id, + inserted_at, + title, + link + FROM entries + WHERE feed_url=? + """, + [url], + ) + + rows = cursor.fetchall() + entries = [ + feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link) + for id, inserted_at, title, link in rows + ] + + return feed.Feed(meta=meta, title=title, link=link, entries=entries) + def update_meta(self, f: feed.FeedMeta): with self.db: self.db.execute( @@ -304,127 +339,47 @@ class Database: Returns the number of new entries inserted. """ with self.db: - self._insert_feed(f.meta, f.title, f.link) - return self._insert_entries(f.meta.url, f.entries) - - def update_feed_status(self, meta: feed.FeedMeta, status: int) -> int: - with self.db: - return self._update_feed_status(meta, status) - - def redirect_feed(self, old_url: str, new_url: str): - with self.db: - cursor = self.db.execute( - "SELECT COUNT(1) FROM feeds WHERE url=?", [new_url] - ) - row = cursor.fetchone() - if row[0] == 0: - self.db.execute( - "UPDATE feeds SET url = ? WHERE url = ?", [new_url, old_url] - ) - else: - # First update all the entries that you can with the old url. - self.db.execute( - """ - UPDATE OR IGNORE entries - SET feed_url = ? - WHERE feed_url = ? - """, - [new_url, old_url], - ) - - # TODO: It is expensive and not worth it to try to load and - # re-insert all the old stuff so I'm not going to - # bother. - - # Mark the old feed unsubscribed. - # TODO: Rebuild with helpers - self.db.execute( - """ - UPDATE feeds - SET status = ?, - last_fetched_ts = ? - WHERE url = ? - """, - [feed.FEED_STATUS_UNSUBSCRIBED, int(time.time()), old_url], - ) - - def _get_property(self, prop: str, default=None) -> typing.Any: - cursor = self.db.execute("SELECT value FROM properties WHERE name=?", (prop,)) - result = cursor.fetchone() - if result is None: - return default - return result[0] - - def _set_property(self, prop: str, value): - self.db.execute( - """ - INSERT INTO properties (name, value) VALUES (?, ?) - ON CONFLICT DO UPDATE SET value=excluded.value + self.db.execute( + """ + INSERT INTO feeds ( + url, + last_fetched_ts, + retry_after_ts, + status, + etag, + modified, + title, + link + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT DO UPDATE + SET + last_fetched_ts=excluded.last_fetched_ts, + retry_after_ts=excluded.retry_after_ts, + status=excluded.status, + etag=excluded.etag, + modified=excluded.modified, + title=excluded.title, + link=excluded.link """, - (prop, value), - ) + [ + f.meta.url, + f.meta.last_fetched_ts, + f.meta.retry_after_ts, + f.meta.status, + f.meta.etag, + f.meta.modified, + f.title, + f.link, + ], + ) - def _insert_feed(self, meta: feed.FeedMeta, title: str, link: str): - """Insert into the feeds table, handling collisions with UPSERT.""" - self.db.execute( - """ - INSERT INTO feeds ( - url, - last_fetched_ts, - retry_after_ts, - status, - etag, - modified, - title, - link - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT DO UPDATE - SET - last_fetched_ts=MAX(last_fetched_ts, excluded.last_fetched_ts), - retry_after_ts=MAX(retry_after_ts, excluded.retry_after_ts), - -- For all other fields, take the value that was computed by the - -- most recent fetch. - status=CASE - WHEN last_fetched_ts > excluded.last_fetched_ts THEN status - ELSE excluded.status - END, - etag=CASE - WHEN last_fetched_ts > excluded.last_fetched_ts THEN etag - ELSE excluded.etag - END, - modified=CASE - WHEN last_fetched_ts > excluded.last_fetched_ts THEN modified - ELSE excluded.modified - END, - title=CASE - WHEN last_fetched_ts > excluded.last_fetched_ts THEN title - ELSE excluded.title - END, - link=CASE - WHEN last_fetched_ts > excluded.last_fetched_ts THEN link - ELSE excluded.link - END - """, - [ - meta.url, - meta.last_fetched_ts, - meta.retry_after_ts, - meta.status, - meta.etag, - meta.modified, - title, - link, - ], - ) + cursor = self.db.execute( + "SELECT COUNT (*) FROM entries WHERE feed_url=?", [f.meta.url] + ) + start_count = cursor.fetchone()[0] - def _insert_entries(self, feed_url: str, entries: list[feed.Entry]) -> int: - cursor = self.db.execute( - "SELECT COUNT (*) FROM entries WHERE feed_url=?", [feed_url] - ) - start_count = cursor.fetchone()[0] - - self.db.executemany( - """ + self.db.executemany( + """ INSERT INTO entries ( id, inserted_at, @@ -454,24 +409,60 @@ class Database: ELSE excluded.link END """, - [(e.id, e.inserted_at, feed_url, e.title, e.link) for e in entries], - ) + [(e.id, e.inserted_at, f.meta.url, e.title, e.link) for e in f.entries], + ) - cursor = self.db.execute( - "SELECT COUNT (*) FROM entries WHERE feed_url=?", [feed_url] - ) - end_count = cursor.fetchone()[0] - return end_count - start_count + cursor = self.db.execute( + "SELECT COUNT (*) FROM entries WHERE feed_url=?", [f.meta.url] + ) + end_count = cursor.fetchone()[0] + return end_count - start_count - def _update_feed_status(self, meta: feed.FeedMeta, status: int) -> int: - new_ts = max(int(time.time()), meta.last_fetched_ts + 1) - cursor = self.db.execute( - """ - UPDATE feeds - SET status = ?, - last_fetched_ts = ? - WHERE url = ? - """, - [status, new_ts, meta.url], - ) - return cursor.rowcount + def set_feed_status(self, url: str, status: int) -> int: + with self.db: + cursor = self.db.execute( + """ + UPDATE feeds + SET status = ?, + last_fetched_ts = ? + WHERE url = ? + """, + [status, int(time.time()), url], + ) + return cursor.rowcount + + def redirect_feed(self, old_url: str, new_url: str): + with self.db: + cursor = self.db.execute( + "SELECT COUNT(1) FROM feeds WHERE url=?", [new_url] + ) + row = cursor.fetchone() + if row[0] == 0: + self.db.execute( + "UPDATE feeds SET url = ? WHERE url = ?", [new_url, old_url] + ) + else: + # First update all the entries that you can with the old url. + self.db.execute( + """ + UPDATE OR IGNORE entries + SET feed_url = ? + WHERE feed_url = ? + """, + [new_url, old_url], + ) + + # TODO: It is expensive and not worth it to try to load and + # re-insert all the old stuff so I'm not going to + # bother. + + # Mark the old feed unsubscribed. + self.db.execute( + """ + UPDATE feeds + SET status = ?, + last_fetched_ts = ? + WHERE url = ? + """, + [feed.FEED_STATUS_UNSUBSCRIBED, int(time.time()), old_url], + ) diff --git a/tests/test_database.py b/tests/test_database.py index 511b760..a6b1d1e 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -106,8 +106,8 @@ def test_database_store_feed(): db.ensure_database_schema() db.store_feed(FEED) - loaded_meta = db.load_meta(FEED.meta.url) - assert loaded_meta == FEED.meta + loaded = db.load_feed(FEED.meta.url) + assert loaded == FEED def test_database_store_feed_dups(): @@ -252,18 +252,16 @@ def test_database_store_update_meta(): assert db.load_all_meta()[0] == new_meta -def test_database_update_feed_status(): +def test_database_set_feed_status(): db = database.Database(":memory:", random_slug()) db.ensure_database_schema() db.store_feed(FEED) assert db.load_all_meta()[0].status != feed.FEED_STATUS_UNSUBSCRIBED - db.update_feed_status( - FEED.meta, - feed.FEED_STATUS_UNSUBSCRIBED, - ) + db.set_feed_status(FEED.meta.url, feed.FEED_STATUS_UNSUBSCRIBED) + # TODO: Ensure that the updated time is touched too. assert db.load_all_meta()[0].status == feed.FEED_STATUS_UNSUBSCRIBED