INNER CODE UNIT · Python
text_from_html
aeturrell/coding-for-economists · data_set_prep.py:53
def text_from_html(body):
soup = BeautifulSoup(body, "html.parser")
texts = soup.findAll(text=True)
visible_texts = filter(tag_visible, texts)
return " ".join(t.strip() for t in visible_texts)
def save_smith_book():
"""Downloads part of the 'The Wealth of Nations' and saves it."""
html = urllib.request.urlopen(
"https://www.gutenberg.org/files/3300/3300-h/3300-h.htm"
).read()
# Take the book text only
book_text = (
text_from_html(html)
.split("Produced by Colin Muir, and David Widger")[1]
.split("Conclusion of the Chapter.")[0]
)