sim1222/fedi-block-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformatted

Browse Source
This commit is contained in:
Enju Aihara 2022-04-09 19:26:33 +02:00
parent 7941765b04
commit e077b77a74
1 changed files with 41 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
fetch_blocks.py
View File

@ -7,6 +7,7 @@ headers = {
"user-agent": "fedi-block-api (https://gitlab.com/EnjuAihara/fedi-block-api)" "user-agent": "fedi-block-api (https://gitlab.com/EnjuAihara/fedi-block-api)"
} }
def get_mastodon_blocks(domain: str) -> dict: def get_mastodon_blocks(domain: str) -> dict:
blocks = { blocks = {
"Suspended servers": [], "Suspended servers": [],
@ -22,7 +23,7 @@ def get_mastodon_blocks(domain: str) -> dict:
) )
except: except:
return {} return {}
for header in doc.find_all("h3"): for header in doc.find_all("h3"):
for line in header.find_next_siblings("table")[0].find_all("tr")[1:]: for line in header.find_next_siblings("table")[0].find_all("tr")[1:]:
if header.text in blocks: if header.text in blocks:
@ -36,12 +37,15 @@ def get_mastodon_blocks(domain: str) -> dict:
return { return {
"reject": blocks["Suspended servers"], "reject": blocks["Suspended servers"],
"media_removal": blocks["Filtered media"], "media_removal": blocks["Filtered media"],
"federated_timeline_removal": blocks["Limited servers"] + blocks["Silenced servers"], "federated_timeline_removal": blocks["Limited servers"]
+ blocks["Silenced servers"],
} }
def get_hash(domain: str) -> str: def get_hash(domain: str) -> str:
return sha256(domain.encode("utf-8")).hexdigest() return sha256(domain.encode("utf-8")).hexdigest()
def get_type(domain: str) -> str: def get_type(domain: str) -> str:
try: try:
res = get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5) res = get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5)
@ -56,10 +60,13 @@ def get_type(domain: str) -> str:
except: except:
return None return None
conn = sqlite3.connect("blocks.db") conn = sqlite3.connect("blocks.db")
c = conn.cursor() c = conn.cursor()
c.execute("select domain, software from instances where software in ('pleroma', 'mastodon')") c.execute(
"select domain, software from instances where software in ('pleroma', 'mastodon')"
)
for blocker, software in c.fetchall(): for blocker, software in c.fetchall():
if software == "pleroma": if software == "pleroma":
@ -67,22 +74,43 @@ for blocker, software in c.fetchall():
try: try:
# Blocks # Blocks
c.execute("delete from blocks where blocker = ?", (blocker,)) c.execute("delete from blocks where blocker = ?", (blocker,))
federation = get(f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5).json()["metadata"]["federation"] federation = get(
f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
).json()["metadata"]["federation"]
if "mrf_simple" in federation: if "mrf_simple" in federation:
for block_level, blocks in (federation["mrf_simple"] | {"quarantined_instances": federation["quarantined_instances"]}).items(): for block_level, blocks in (
federation["mrf_simple"]
| {"quarantined_instances": federation["quarantined_instances"]}
).items():
for blocked in blocks: for blocked in blocks:
if blocked == "": if blocked == "":
continue continue
c.execute("select domain from instances where domain = ?", (blocked,)) c.execute(
"select domain from instances where domain = ?", (blocked,)
)
if c.fetchone() == None: if c.fetchone() == None:
c.execute("insert into instances select ?, ?, ?", (blocked, get_hash(blocked), get_type(blocked))) c.execute(
c.execute("insert into blocks select ?, ?, '', ?", (blocker, blocked, block_level)) "insert into instances select ?, ?, ?",
(blocked, get_hash(blocked), get_type(blocked)),
)
c.execute(
"insert into blocks select ?, ?, '', ?",
(blocker, blocked, block_level),
)
conn.commit() conn.commit()
# Reasons # Reasons
if "mrf_simple_info" in federation: if "mrf_simple_info" in federation:
for block_level, info in (federation["mrf_simple_info"] | federation["quarantined_instances_info"] if "quarantined_instances_info" in federation else {}).items(): for block_level, info in (
federation["mrf_simple_info"]
| federation["quarantined_instances_info"]
if "quarantined_instances_info" in federation
else {}
).items():
for blocked, reason in info.items(): for blocked, reason in info.items():
c.execute("update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?", (reason["reason"], blocker, blocked, block_level)) c.execute(
"update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?",
(reason["reason"], blocker, blocked, block_level),
)
conn.commit() conn.commit()
except Exception as e: except Exception as e:
print("error:", e, blocker) print("error:", e, blocker)
@ -95,7 +123,9 @@ for blocker, software in c.fetchall():
for instance in blocks: for instance in blocks:
blocked, blocked_hash, reason = instance.values() blocked, blocked_hash, reason = instance.values()
if blocked.count("*") <= 1: if blocked.count("*") <= 1:
c.execute("select hash from instances where hash = ?", (blocked_hash,)) c.execute(
"select hash from instances where hash = ?", (blocked_hash,)
)
if c.fetchone() == None: if c.fetchone() == None:
c.execute( c.execute(
"insert into instances select ?, ?, ?", "insert into instances select ?, ?, ?",