INNER CODE UNIT · Python
fetch_url
crocs-muni/javacard-curated-list · scripts/process_curated_list.py:28
def fetch_url(url, timeout=15):
req = Request(url, headers={"User-Agent": USER_AGENT})
with urlopen(req, timeout=timeout) as resp:
return resp.read().decode("utf-8", errors="replace")
# ---------------------------------------------------------------------------
# Step 1: fetch the raw README.md markdown source from GitHub
# ---------------------------------------------------------------------------
def load_markdown_text(source):
if source.startswith("http://") or source.startswith("https://"):
text = fetch_url(source)
else:
with open(source, encoding="utf-8") as f:
text = f.read()
return text.replace("\r\n", "\n").replace("\r", "\n")