DeCarabas/cry
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: DeCarabas:463abcb92377ebce57c092ee79f96164f0dbc5c7
Branches Tags
DeCarabas:main
..
compare: DeCarabas:f9b14a7622f5598c511006231400163ea7c28272
Branches Tags
DeCarabas:main

No commits in common. "463abcb92377ebce57c092ee79f96164f0dbc5c7" and "f9b14a7622f5598c511006231400163ea7c28272" have entirely different histories.
463abcb923 ... f9b14a7622

3 changed files with 151 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

27
cry/cli.py
View file

@ -96,7 +96,7 @@ def subscribe(url, literal):
result = d result = d
# Check to see if this URL is already in the database. # Check to see if this URL is already in the database.
existing = db.load_meta(result.meta.url) existing = db.load_feed(result.meta.url) # TODO: Replace with 'load_meta'?
if existing is not None: if existing is not None:
click.echo(f"This feed already exists (as {result.meta.url})") click.echo(f"This feed already exists (as {result.meta.url})")
return 1 return 1
@ -129,7 +129,7 @@ def import_opml(opml_file):
click.echo(f"{url} does not seem to be a feed, skipping...") click.echo(f"{url} does not seem to be a feed, skipping...")
continue continue
existing = db.load_meta(meta.url) existing = db.load_feed(meta.url) # TODO: Replace with 'load_meta'?
if existing is not None: if existing is not None:
LOG.info(f"{url} already exists (as {meta.url})") LOG.info(f"{url} already exists (as {meta.url})")
continue continue
@ -151,11 +151,11 @@ def refresh(url):
db = database.Database.local() db = database.Database.local()
if url: if url:
f = db.load_meta(url) f = db.load_feed(url) # TODO: Replace with 'load_meta'?
if f is None: if f is None:
click.echo(f"Not subscribed to {url}") click.echo(f"Not subscribed to {url}")
return 1 return 1
feeds = [f] feeds = [f.meta]
else: else:
feeds = db.load_all_meta() feeds = db.load_all_meta()
@ -239,13 +239,11 @@ def unsubscribe(url):
`list` command.) `list` command.)
""" """
db = database.Database.local() db = database.Database.local()
meta = db.load_meta(url) count = db.set_feed_status(url, feed.FEED_STATUS_UNSUBSCRIBED)
if meta is None: if count == 0:
click.echo(f"Not subscribed to feed {url}") click.echo(f"Not subscribed to feed {url}")
return 1 return 1
db.update_feed_status(meta, feed.FEED_STATUS_UNSUBSCRIBED)
@cli.command("serve") @cli.command("serve")
def serve(): def serve():
@ -264,11 +262,6 @@ def serve():
<head> <head>
<meta charset="utf8"> <meta charset="utf8">
<title>Subscribed Feeds</title> <title>Subscribed Feeds</title>
<style>
body { margin-left: 4rem; }
li.entry { display: inline; padding-right: 1rem; }
li.entry:before { content: '\\2022'; padding-right: 0.5rem; }
</style>
</head> </head>
<h1>Feeds</h1> <h1>Feeds</h1>
""" """
@ -279,19 +272,17 @@ def serve():
ago = f" ({f.entries[0].time_ago()})" ago = f" ({f.entries[0].time_ago()})"
else: else:
ago = "" ago = ""
buffer.write(f"<div class='feed'>")
buffer.write(f'<h2><a href="{f.link}">{feed_title}</a>{ago}</h2>') buffer.write(f'<h2><a href="{f.link}">{feed_title}</a>{ago}</h2>')
buffer.write(f"<div>")
if len(f.entries) > 0: if len(f.entries) > 0:
buffer.write(f"<ul>")
for entry in f.entries: for entry in f.entries:
title = html.escape(entry.title) title = html.escape(entry.title)
buffer.write( buffer.write(
f'<li class="entry"><a href="{entry.link}">{title}</a> ({entry.time_ago()})</li>' f'<span class="entry">&bull; <a href="{entry.link}">{title}</a> ({entry.time_ago()})</span> '
) )
buffer.write(f"</ul>")
else: else:
buffer.write("<i>No entries...</i>") buffer.write("<i>No entries...</i>")
buffer.write(f"</div>") # feed buffer.write(f"</div>")
buffer.flush() buffer.flush()
text = buffer.getvalue() text = buffer.getvalue()
response = text.encode("utf-8") response = text.encode("utf-8")

283
cry/database.py
View file

@ -115,11 +115,23 @@ class Database:
def get_property(self, prop: str, default=None) -> typing.Any: def get_property(self, prop: str, default=None) -> typing.Any:
with self.db: with self.db:
return self._get_property(prop, default) cursor = self.db.execute(
"SELECT value FROM properties WHERE name=?", (prop,)
)
result = cursor.fetchone()
if result is None:
return default
return result[0]
def set_property(self, prop: str, value): def set_property(self, prop: str, value):
with self.db: with self.db:
return self._set_property(prop, value) self.db.execute(
"""
INSERT INTO properties (name, value) VALUES (?, ?)
ON CONFLICT DO UPDATE SET value=excluded.value
""",
(prop, value),
)
def ensure_database_schema(self): def ensure_database_schema(self):
with self.db: with self.db:
@ -131,15 +143,15 @@ class Database:
) )
""" """
) )
version = int(self._get_property("version", 0)) version = int(self.get_property("version", 0))
for script in SCHEMA_STATEMENTS[version:]: for script in SCHEMA_STATEMENTS[version:]:
for statement in script.split(";"): for statement in script.split(";"):
try: try:
self.db.execute(statement) self.db.execute(statement)
except Exception as e: except Exception as e:
raise Exception(f"Error executing:\n{statement}") from e raise Exception(f"Error executing:\n{statement}") from e
self._set_property("version", len(SCHEMA_STATEMENTS)) self.set_property("version", len(SCHEMA_STATEMENTS))
self._set_property("origin", self.origin) self.set_property("origin", self.origin)
def load_all_meta(self) -> list[feed.FeedMeta]: def load_all_meta(self) -> list[feed.FeedMeta]:
with self.db: with self.db:
@ -246,7 +258,7 @@ class Database:
return feeds return feeds
def load_meta(self, url: str) -> feed.FeedMeta | None: def load_feed(self, url: str) -> feed.Feed | None:
with self.db: with self.db:
cursor = self.db.execute( cursor = self.db.execute(
""" """
@ -255,7 +267,9 @@ class Database:
retry_after_ts, retry_after_ts,
status, status,
etag, etag,
modified modified,
title,
link
FROM feeds FROM feeds
WHERE url=? WHERE url=?
""", """,
@ -266,8 +280,8 @@ class Database:
if row is None: if row is None:
return None return None
last_fetched_ts, retry_after_ts, status, etag, modified = row last_fetched_ts, retry_after_ts, status, etag, modified, title, link = row
return feed.FeedMeta( meta = feed.FeedMeta(
url=url, url=url,
last_fetched_ts=last_fetched_ts, last_fetched_ts=last_fetched_ts,
retry_after_ts=retry_after_ts, retry_after_ts=retry_after_ts,
@ -276,6 +290,27 @@ class Database:
modified=modified, modified=modified,
) )
cursor = self.db.execute(
"""
SELECT
id,
inserted_at,
title,
link
FROM entries
WHERE feed_url=?
""",
[url],
)
rows = cursor.fetchall()
entries = [
feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link)
for id, inserted_at, title, link in rows
]
return feed.Feed(meta=meta, title=title, link=link, entries=entries)
def update_meta(self, f: feed.FeedMeta): def update_meta(self, f: feed.FeedMeta):
with self.db: with self.db:
self.db.execute( self.db.execute(
@ -304,127 +339,47 @@ class Database:
Returns the number of new entries inserted. Returns the number of new entries inserted.
""" """
with self.db: with self.db:
self._insert_feed(f.meta, f.title, f.link) self.db.execute(
return self._insert_entries(f.meta.url, f.entries) """
INSERT INTO feeds (
def update_feed_status(self, meta: feed.FeedMeta, status: int) -> int: url,
with self.db: last_fetched_ts,
return self._update_feed_status(meta, status) retry_after_ts,
status,
def redirect_feed(self, old_url: str, new_url: str): etag,
with self.db: modified,
cursor = self.db.execute( title,
"SELECT COUNT(1) FROM feeds WHERE url=?", [new_url] link
) ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
row = cursor.fetchone() ON CONFLICT DO UPDATE
if row[0] == 0: SET
self.db.execute( last_fetched_ts=excluded.last_fetched_ts,
"UPDATE feeds SET url = ? WHERE url = ?", [new_url, old_url] retry_after_ts=excluded.retry_after_ts,
) status=excluded.status,
else: etag=excluded.etag,
# First update all the entries that you can with the old url. modified=excluded.modified,
self.db.execute( title=excluded.title,
""" link=excluded.link
UPDATE OR IGNORE entries
SET feed_url = ?
WHERE feed_url = ?
""",
[new_url, old_url],
)
# TODO: It is expensive and not worth it to try to load and
# re-insert all the old stuff so I'm not going to
# bother.
# Mark the old feed unsubscribed.
# TODO: Rebuild with helpers
self.db.execute(
"""
UPDATE feeds
SET status = ?,
last_fetched_ts = ?
WHERE url = ?
""",
[feed.FEED_STATUS_UNSUBSCRIBED, int(time.time()), old_url],
)
def _get_property(self, prop: str, default=None) -> typing.Any:
cursor = self.db.execute("SELECT value FROM properties WHERE name=?", (prop,))
result = cursor.fetchone()
if result is None:
return default
return result[0]
def _set_property(self, prop: str, value):
self.db.execute(
"""
INSERT INTO properties (name, value) VALUES (?, ?)
ON CONFLICT DO UPDATE SET value=excluded.value
""", """,
(prop, value), [
) f.meta.url,
f.meta.last_fetched_ts,
f.meta.retry_after_ts,
f.meta.status,
f.meta.etag,
f.meta.modified,
f.title,
f.link,
],
)
def _insert_feed(self, meta: feed.FeedMeta, title: str, link: str): cursor = self.db.execute(
"""Insert into the feeds table, handling collisions with UPSERT.""" "SELECT COUNT (*) FROM entries WHERE feed_url=?", [f.meta.url]
self.db.execute( )
""" start_count = cursor.fetchone()[0]
INSERT INTO feeds (
url,
last_fetched_ts,
retry_after_ts,
status,
etag,
modified,
title,
link
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT DO UPDATE
SET
last_fetched_ts=MAX(last_fetched_ts, excluded.last_fetched_ts),
retry_after_ts=MAX(retry_after_ts, excluded.retry_after_ts),
-- For all other fields, take the value that was computed by the
-- most recent fetch.
status=CASE
WHEN last_fetched_ts > excluded.last_fetched_ts THEN status
ELSE excluded.status
END,
etag=CASE
WHEN last_fetched_ts > excluded.last_fetched_ts THEN etag
ELSE excluded.etag
END,
modified=CASE
WHEN last_fetched_ts > excluded.last_fetched_ts THEN modified
ELSE excluded.modified
END,
title=CASE
WHEN last_fetched_ts > excluded.last_fetched_ts THEN title
ELSE excluded.title
END,
link=CASE
WHEN last_fetched_ts > excluded.last_fetched_ts THEN link
ELSE excluded.link
END
""",
[
meta.url,
meta.last_fetched_ts,
meta.retry_after_ts,
meta.status,
meta.etag,
meta.modified,
title,
link,
],
)
def _insert_entries(self, feed_url: str, entries: list[feed.Entry]) -> int: self.db.executemany(
cursor = self.db.execute( """
"SELECT COUNT (*) FROM entries WHERE feed_url=?", [feed_url]
)
start_count = cursor.fetchone()[0]
self.db.executemany(
"""
INSERT INTO entries ( INSERT INTO entries (
id, id,
inserted_at, inserted_at,
@ -454,24 +409,60 @@ class Database:
ELSE excluded.link ELSE excluded.link
END END
""", """,
[(e.id, e.inserted_at, feed_url, e.title, e.link) for e in entries], [(e.id, e.inserted_at, f.meta.url, e.title, e.link) for e in f.entries],
) )
cursor = self.db.execute( cursor = self.db.execute(
"SELECT COUNT (*) FROM entries WHERE feed_url=?", [feed_url] "SELECT COUNT (*) FROM entries WHERE feed_url=?", [f.meta.url]
) )
end_count = cursor.fetchone()[0] end_count = cursor.fetchone()[0]
return end_count - start_count return end_count - start_count
def _update_feed_status(self, meta: feed.FeedMeta, status: int) -> int: def set_feed_status(self, url: str, status: int) -> int:
new_ts = max(int(time.time()), meta.last_fetched_ts + 1) with self.db:
cursor = self.db.execute( cursor = self.db.execute(
""" """
UPDATE feeds UPDATE feeds
SET status = ?, SET status = ?,
last_fetched_ts = ? last_fetched_ts = ?
WHERE url = ? WHERE url = ?
""", """,
[status, new_ts, meta.url], [status, int(time.time()), url],
) )
return cursor.rowcount return cursor.rowcount
def redirect_feed(self, old_url: str, new_url: str):
with self.db:
cursor = self.db.execute(
"SELECT COUNT(1) FROM feeds WHERE url=?", [new_url]
)
row = cursor.fetchone()
if row[0] == 0:
self.db.execute(
"UPDATE feeds SET url = ? WHERE url = ?", [new_url, old_url]
)
else:
# First update all the entries that you can with the old url.
self.db.execute(
"""
UPDATE OR IGNORE entries
SET feed_url = ?
WHERE feed_url = ?
""",
[new_url, old_url],
)
# TODO: It is expensive and not worth it to try to load and
# re-insert all the old stuff so I'm not going to
# bother.
# Mark the old feed unsubscribed.
self.db.execute(
"""
UPDATE feeds
SET status = ?,
last_fetched_ts = ?
WHERE url = ?
""",
[feed.FEED_STATUS_UNSUBSCRIBED, int(time.time()), old_url],
)

12
tests/test_database.py
View file

@ -106,8 +106,8 @@ def test_database_store_feed():
db.ensure_database_schema() db.ensure_database_schema()
db.store_feed(FEED) db.store_feed(FEED)
loaded_meta = db.load_meta(FEED.meta.url) loaded = db.load_feed(FEED.meta.url)
assert loaded_meta == FEED.meta assert loaded == FEED
def test_database_store_feed_dups(): def test_database_store_feed_dups():
@ -252,18 +252,16 @@ def test_database_store_update_meta():
assert db.load_all_meta()[0] == new_meta assert db.load_all_meta()[0] == new_meta
def test_database_update_feed_status(): def test_database_set_feed_status():
db = database.Database(":memory:", random_slug()) db = database.Database(":memory:", random_slug())
db.ensure_database_schema() db.ensure_database_schema()
db.store_feed(FEED) db.store_feed(FEED)
assert db.load_all_meta()[0].status != feed.FEED_STATUS_UNSUBSCRIBED assert db.load_all_meta()[0].status != feed.FEED_STATUS_UNSUBSCRIBED
db.update_feed_status( db.set_feed_status(FEED.meta.url, feed.FEED_STATUS_UNSUBSCRIBED)
FEED.meta,
feed.FEED_STATUS_UNSUBSCRIBED,
)
# TODO: Ensure that the updated time is touched too.
assert db.load_all_meta()[0].status == feed.FEED_STATUS_UNSUBSCRIBED assert db.load_all_meta()[0].status == feed.FEED_STATUS_UNSUBSCRIBED