Compare commits
2 commits
03d420e412
...
51049b4d30
| Author | SHA1 | Date | |
|---|---|---|---|
| 51049b4d30 | |||
| 3c278a09ba |
2 changed files with 24 additions and 5 deletions
25
cry/feed.py
25
cry/feed.py
|
|
@ -664,14 +664,14 @@ def classify_links(links, baseuri) -> typing.Tuple[list[str], list[str]]:
|
||||||
"""
|
"""
|
||||||
baseuri = baseuri.lower()
|
baseuri = baseuri.lower()
|
||||||
|
|
||||||
local, remote = [], []
|
local, remote = set(), set()
|
||||||
for link in links:
|
for link in links:
|
||||||
if link.lower().startswith(baseuri):
|
if link.lower().startswith(baseuri):
|
||||||
local.append(link)
|
local.add(link)
|
||||||
else:
|
else:
|
||||||
remote.append(link)
|
remote.add(link)
|
||||||
|
|
||||||
return local, remote
|
return list(local), list(remote)
|
||||||
|
|
||||||
|
|
||||||
def is_feed_link(link: str) -> bool:
|
def is_feed_link(link: str) -> bool:
|
||||||
|
|
@ -687,18 +687,27 @@ def is_feed_link(link: str) -> bool:
|
||||||
|
|
||||||
def is_XML_related_link(link: str) -> bool:
|
def is_XML_related_link(link: str) -> bool:
|
||||||
link = link.lower()
|
link = link.lower()
|
||||||
return "rss" in link or "rdf" in link or "xml" in link or "atom" in link
|
return (
|
||||||
|
"rss" in link
|
||||||
|
or "rdf" in link
|
||||||
|
or "xml" in link
|
||||||
|
or "atom" in link
|
||||||
|
or "feed" in link
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def check_feed(url: str) -> Feed | None:
|
async def check_feed(url: str) -> Feed | None:
|
||||||
"""Check to see if the given URL is a feed. If it is, return the feed,
|
"""Check to see if the given URL is a feed. If it is, return the feed,
|
||||||
otherwise return None.
|
otherwise return None.
|
||||||
"""
|
"""
|
||||||
|
LOG.debug(f"Checking {url}: checking...")
|
||||||
meta = FeedMeta.from_url(url)
|
meta = FeedMeta.from_url(url)
|
||||||
result, meta = await fetch_feed(meta)
|
result, meta = await fetch_feed(meta)
|
||||||
if isinstance(result, Feed):
|
if isinstance(result, Feed):
|
||||||
|
LOG.debug(f"Checking {url}: is a feed")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
LOG.debug(f"Checking {url}: is not a feed")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -738,6 +747,11 @@ async def feed_search(uri: str) -> list[Feed]:
|
||||||
|
|
||||||
LOG.debug("No links, checking A tags...")
|
LOG.debug("No links, checking A tags...")
|
||||||
local_links, remote_links = classify_links(parser.a_links, meta.url)
|
local_links, remote_links = classify_links(parser.a_links, meta.url)
|
||||||
|
for link in local_links:
|
||||||
|
LOG.debug(f" LOCAL {link}")
|
||||||
|
for link in remote_links:
|
||||||
|
LOG.debug(f" REMOTE {link}")
|
||||||
|
|
||||||
outfeeds = await check_links(filter(is_feed_link, local_links))
|
outfeeds = await check_links(filter(is_feed_link, local_links))
|
||||||
if len(outfeeds) > 0:
|
if len(outfeeds) > 0:
|
||||||
return outfeeds
|
return outfeeds
|
||||||
|
|
@ -759,6 +773,7 @@ async def feed_search(uri: str) -> list[Feed]:
|
||||||
"rss.xml", # Dave Winer/Manila
|
"rss.xml", # Dave Winer/Manila
|
||||||
"index.xml", # MT
|
"index.xml", # MT
|
||||||
"index.rss", # Slash
|
"index.rss", # Slash
|
||||||
|
"feed", # catandgirl.com and sometimes others.
|
||||||
]
|
]
|
||||||
outfeeds = await check_links([urllib.parse.urljoin(meta.url, x) for x in suffixes])
|
outfeeds = await check_links([urllib.parse.urljoin(meta.url, x) for x in suffixes])
|
||||||
return outfeeds
|
return outfeeds
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,7 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["pdm-backend"]
|
||||||
|
build-backend = "pdm.backend"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "cry"
|
name = "cry"
|
||||||
version = "0.1"
|
version = "0.1"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue