How to scrape websites created with SPA

pip install requests-html

# -*- coding: utf-8 -*-
import requests
from requests_html import HTMLSession

def main_render_javascript_page():
    url = 'https://hogehoge'
    session = HTMLSession()
    r = session.get(url)
    r.html.render()
    title =  r.html.find('body', first=True).text
    print(title)
        
def main_normal_page():
    url = 'https://hogehoge'
    r = requests.get(url)
    print(r.text)

if __name__ == '__main__':
    main_normal_page()
    main_render_javascript_page()

Recommended Posts