DeCarabas/cry
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Sync implementation

Browse source
This commit is contained in:
John Doty 2024-07-23 07:20:20 -07:00
parent c6bd8d8556
commit 74e9091065
3 changed files with 176 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

13
cry/cli.py
View file

@ -250,8 +250,8 @@ def serve():
web.serve()
@cli.command("reconcile")
def reconcile():
@cli.command("sync")
def sync():
local_db = database.Database.local()
local_version = local_db.get_property("version", 0)
for p in database.databases_directory().glob("*.db"):
@ -263,6 +263,8 @@ def reconcile():
if local_db.origin == other_db.origin:
continue
# Ensure the schema version is compatible so that we don't run
# into trouble trying to query the other database.
other_version = other_db.get_property("version", 0)
if other_version != local_version:
click.echo(
@ -270,13 +272,16 @@ def reconcile():
)
continue
# TODO: GET CLOCK OF BOTH.
# Check to see if we've already reconciled this other database.
other_clock = other_db.get_clock()
reconciled_clock = local_db.get_reconcile_clock(other_db.origin)
reconciled_clock = local_db.get_sync_clock(other_db.origin)
if other_clock == reconciled_clock:
continue
# TODO: RECONCILE FOR REALS
local_db.sync_from(other_db)
local_db.set_sync_clock(other_db.origin, other_clock)
except Exception as e:
click.echo(f"Error loading {p}: {e}")

102
cry/database.py
View file

@ -100,7 +100,7 @@ SCHEMA_STATEMENTS = [
END;
""",
"""
CREATE TABLE reconcile_status (
CREATE TABLE sync_status (
origin VARCHAR NOT NULL PRIMARY KEY,
clock INT NOT NULL
);
@ -289,9 +289,9 @@ class Database:
) = row
meta = feed.FeedMeta(
url=url,
last_fetched_ts=last_fetched_ts,
retry_after_ts=retry_after_ts,
status=status,
last_fetched_ts=int(last_fetched_ts),
retry_after_ts=int(retry_after_ts),
status=int(status),
etag=etag,
modified=modified,
)
@ -403,6 +403,94 @@ class Database:
[feed.FEED_STATUS_UNSUBSCRIBED, int(time.time()), old_url],
)
def get_sync_clock(self, origin: str) -> int | None:
with self.db:
cursor = self.db.execute(
"SELECT clock FROM sync_status WHERE origin = ?",
[origin],
)
row = cursor.fetchone()
if row is None:
return None
return int(row[0])
def set_sync_clock(self, origin: str, clock: int):
with self.db:
self.db.execute(
"""
INSERT INTO sync_status (origin, clock)
VALUES (?, ?)
ON CONFLICT DO UPDATE SET clock=excluded.clock
""",
[origin, clock],
)
def sync_from(self, other: "Database"):
with self.db:
with other.db:
feed_cursor = other.db.execute(
"""
SELECT
url,
last_fetched_ts,
retry_after_ts,
status,
etag,
modified,
title,
link
FROM feeds
"""
)
for row in feed_cursor:
(
url,
last_fetched_ts,
retry_after_ts,
status,
etag,
modified,
title,
link,
) = row
meta = feed.FeedMeta(
url=url,
last_fetched_ts=int(last_fetched_ts),
retry_after_ts=int(retry_after_ts),
status=int(status),
etag=etag,
modified=modified,
)
self._insert_feed(meta, title, link)
entries_cursor = other.db.execute(
"""
SELECT
id,
inserted_at,
title,
link
FROM entries
WHERE feed_url=?
""",
[url],
)
entries_results = entries_cursor.fetchmany()
while len(entries_results) > 0:
self._insert_entries(
url,
[
feed.Entry(
id=id,
inserted_at=int(inserted_at),
title=title,
link=link,
)
for id, inserted_at, title, link in entries_results
],
)
entries_results = entries_cursor.fetchmany()
def _get_property(self, prop: str, default=None) -> typing.Any:
cursor = self.db.execute("SELECT value FROM properties WHERE name=?", (prop,))
result = cursor.fetchone()
@ -553,9 +641,9 @@ class Database:
last_fetched_ts, retry_after_ts, status, etag, modified = row
return feed.FeedMeta(
url=url,
last_fetched_ts=last_fetched_ts,
retry_after_ts=retry_after_ts,
status=status,
last_fetched_ts=int(last_fetched_ts),
retry_after_ts=int(retry_after_ts),
status=int(status),
etag=etag,
modified=modified,
)