DeCarabas/cry
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Deal with locked databases, tweaks

Browse source
This commit is contained in:
John Doty 2024-07-11 10:32:48 +09:00
parent fb9bfe0084
commit 6b02fb66bc
2 changed files with 172 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

24
cry/cli.py
View file

@ -199,22 +199,34 @@ def serve():
def do_GET(self): def do_GET(self):
db = database.Database.local() db = database.Database.local()
feeds = db.load_all(feed_limit=10) feeds = db.load_all(feed_limit=10)
del db
feeds.sort(key=feed.sort_key, reverse=True) feeds.sort(key=feed.sort_key, reverse=True)
buffer = io.StringIO() buffer = io.StringIO()
buffer.write("<head>") buffer.write(
buffer.write('<meta charset="utf-8"><title>Subscribed Feeds</title>') """
buffer.write("</head>") <!doctype html>
buffer.write("<h1>Feeds</h1>") <head>
<meta charset="utf8">
<title>Subscribed Feeds</title>
</head>
<h1>Feeds</h1>
"""
)
for f in feeds: for f in feeds:
feed_title = html.escape(f.title) feed_title = html.escape(f.title)
buffer.write(f'<h2><a href="{f.link}">{feed_title}</a></h2>') if len(f.entries) > 0:
ago = f" ({f.entries[0].time_ago()})"
else:
ago = ""
buffer.write(f'<h2><a href="{f.link}">{feed_title}</a>{ago}</h2>')
buffer.write(f"<div>") buffer.write(f"<div>")
if len(f.entries) > 0: if len(f.entries) > 0:
for entry in f.entries: for entry in f.entries:
title = html.escape(entry.title) title = html.escape(entry.title)
buffer.write( buffer.write(
f'<span>&bull; <a href="{entry.link}">{title}</a> ({entry.time_ago()})</span> ' f'<span class="entry">&bull; <a href="{entry.link}">{title}</a> ({entry.time_ago()})</span> '
) )
else: else:
buffer.write("<i>No entries...</i>") buffer.write("<i>No entries...</i>")

299
cry/database.py
View file

@ -102,20 +102,24 @@ class Database:
return db return db
def get_property(self, prop: str, default=None) -> typing.Any: def get_property(self, prop: str, default=None) -> typing.Any:
cursor = self.db.execute("SELECT value FROM properties WHERE name=?", (prop,)) with self.db:
result = cursor.fetchone() cursor = self.db.execute(
if result is None: "SELECT value FROM properties WHERE name=?", (prop,)
return default )
return result[0] result = cursor.fetchone()
if result is None:
return default
return result[0]
def set_property(self, prop: str, value): def set_property(self, prop: str, value):
self.db.execute( with self.db:
""" self.db.execute(
INSERT INTO properties (name, value) VALUES (?, ?) """
ON CONFLICT DO UPDATE SET value=excluded.value INSERT INTO properties (name, value) VALUES (?, ?)
""", ON CONFLICT DO UPDATE SET value=excluded.value
(prop, value), """,
) (prop, value),
)
def ensure_database_schema(self): def ensure_database_schema(self):
with self.db: with self.db:
@ -138,67 +142,135 @@ class Database:
self.set_property("origin", self.origin) self.set_property("origin", self.origin)
def load_all_meta(self) -> list[feed.FeedMeta]: def load_all_meta(self) -> list[feed.FeedMeta]:
cursor = self.db.execute( with self.db:
""" cursor = self.db.execute(
SELECT """
url, SELECT
last_fetched_ts, url,
retry_after_ts, last_fetched_ts,
status, retry_after_ts,
etag, status,
modified etag,
FROM feeds modified
""" FROM feeds
) """
rows = cursor.fetchall()
return [
feed.FeedMeta(
url=url,
last_fetched_ts=int(last_fetched_ts),
retry_after_ts=int(retry_after_ts),
status=int(status),
etag=etag,
modified=modified,
origin=self.origin,
) )
for url, last_fetched_ts, retry_after_ts, status, etag, modified in rows rows = cursor.fetchall()
] return [
feed.FeedMeta(
url=url,
last_fetched_ts=int(last_fetched_ts),
retry_after_ts=int(retry_after_ts),
status=int(status),
etag=etag,
modified=modified,
origin=self.origin,
)
for url, last_fetched_ts, retry_after_ts, status, etag, modified in rows
]
def load_all(self, feed_limit: int = 20, pattern: str = "") -> list[feed.Feed]: def load_all(self, feed_limit: int = 20, pattern: str = "") -> list[feed.Feed]:
pattern = pattern.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") with self.db:
sql_pattern = f"%{pattern}%" pattern = (
cursor = self.db.execute( pattern.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
""" )
SELECT sql_pattern = f"%{pattern}%"
url, cursor = self.db.execute(
last_fetched_ts, """
retry_after_ts, SELECT
status, url,
etag, last_fetched_ts,
modified, retry_after_ts,
title, status,
link etag,
FROM feeds modified,
WHERE (title LIKE :sql_pattern ESCAPE '\\' title,
OR link LIKE :sql_pattern ESCAPE '\\') link
AND status != 2 -- UNSUBSCRIBED FROM feeds
""", WHERE (title LIKE :sql_pattern ESCAPE '\\'
{"sql_pattern": sql_pattern}, OR link LIKE :sql_pattern ESCAPE '\\')
) AND status != 2 -- UNSUBSCRIBED
rows = cursor.fetchall() """,
{"sql_pattern": sql_pattern},
)
rows = cursor.fetchall()
almost_feeds = [] almost_feeds = []
for row in rows: for row in rows:
( (
url, url,
last_fetched_ts, last_fetched_ts,
retry_after_ts, retry_after_ts,
status, status,
etag, etag,
modified, modified,
title, title,
link, link,
) = row ) = row
meta = feed.FeedMeta(
url=url,
last_fetched_ts=last_fetched_ts,
retry_after_ts=retry_after_ts,
status=status,
etag=etag,
modified=modified,
origin=self.origin,
)
almost_feeds.append((meta, title, link))
feeds = []
for meta, title, link in almost_feeds:
if feed_limit > 0:
cursor = self.db.execute(
"""
SELECT
id,
inserted_at,
title,
link
FROM entries
WHERE feed_url=?
ORDER BY inserted_at DESC
LIMIT ?
""",
[meta.url, feed_limit],
)
rows = cursor.fetchall()
else:
rows = []
entries = [
feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link)
for id, inserted_at, title, link in rows
]
f = feed.Feed(meta=meta, title=title, link=link, entries=entries)
feeds.append(f)
return feeds
def load_feed(self, url: str) -> feed.Feed | None:
with self.db:
cursor = self.db.execute(
"""
SELECT
last_fetched_ts,
retry_after_ts,
status,
etag,
modified,
title,
link
FROM feeds
WHERE url=?
""",
[url],
)
row = cursor.fetchone()
if row is None:
return None
last_fetched_ts, retry_after_ts, status, etag, modified, title, link = row
meta = feed.FeedMeta( meta = feed.FeedMeta(
url=url, url=url,
last_fetched_ts=last_fetched_ts, last_fetched_ts=last_fetched_ts,
@ -208,88 +280,25 @@ class Database:
modified=modified, modified=modified,
origin=self.origin, origin=self.origin,
) )
almost_feeds.append((meta, title, link))
feeds = [] cursor = self.db.execute(
for meta, title, link in almost_feeds: """
if feed_limit > 0: SELECT
cursor = self.db.execute( id,
""" inserted_at,
SELECT title,
id, link
inserted_at, FROM entries
title, WHERE feed_url=?
link """,
FROM entries [url],
WHERE feed_url=? )
ORDER BY inserted_at DESC
LIMIT ?
""",
[meta.url, feed_limit],
)
rows = cursor.fetchall()
else:
rows = []
rows = cursor.fetchall()
entries = [ entries = [
feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link) feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link)
for id, inserted_at, title, link in rows for id, inserted_at, title, link in rows
] ]
f = feed.Feed(meta=meta, title=title, link=link, entries=entries)
feeds.append(f)
return feeds
def load_feed(self, url: str) -> feed.Feed | None:
cursor = self.db.execute(
"""
SELECT
last_fetched_ts,
retry_after_ts,
status,
etag,
modified,
title,
link
FROM feeds
WHERE url=?
""",
[url],
)
row = cursor.fetchone()
if row is None:
return None
last_fetched_ts, retry_after_ts, status, etag, modified, title, link = row
meta = feed.FeedMeta(
url=url,
last_fetched_ts=last_fetched_ts,
retry_after_ts=retry_after_ts,
status=status,
etag=etag,
modified=modified,
origin=self.origin,
)
cursor = self.db.execute(
"""
SELECT
id,
inserted_at,
title,
link
FROM entries
WHERE feed_url=?
""",
[url],
)
rows = cursor.fetchall()
entries = [
feed.Entry(id=id, inserted_at=inserted_at, title=title, link=link)
for id, inserted_at, title, link in rows
]
return feed.Feed(meta=meta, title=title, link=link, entries=entries) return feed.Feed(meta=meta, title=title, link=link, entries=entries)