Grattage Python get_title

Cible

• Obtenez des titres de films à partir des classements des sites de vote

Code source

import requests
from bs4 import BeautifulSoup
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.ui import WebDriverWait
import time

print("Input file_name:")
file_name = input()
Source_file = "/Users/micksmith/home/work/eBay/Python/" + file_name +".csv"


def scroll_down():
    # height = driver.find_element_by_id('pagination')
    # driver.execute_script('arguments[0].scrollIntoView(true);', height)
    # driver.execute_script('scrollBy(0, -150)')
    try:
        while True:
            height = driver.find_element_by_id('pagination')
            driver.execute_script('arguments[0].scrollIntoView(true);', height)
            driver.execute_script('scrollBy(0, -150)')
            time.sleep(1)
            # try:
            #     height.click()
            # except:
            #     pass
            if(len(driver.find_elements_by_id('pagination')) == 0):
                return
            height.click()
            time.sleep(1)
    except:
        return

def get_title(title_Eng):
    scroll_down()
    # for button in driver.find_elements_by_id('pagination__section'):
    #     button.find_element_by_id('pagination').click()
    #     print(button.text)
    # driver.find_element_by_id('pagination').click()
    items = driver.find_elements_by_class_name('listItem__title')
    for item in items:
        title_Eng.append(item.text)
    print("title_Eng:", title_Eng)
    return title_Eng
       
        
if __name__ == "__main__":
    # Open Browser
    options = Options()
    #options.add_argument('--headless')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage') 
    driver = webdriver.Chrome(executable_path="/Users/micksmith/home/work/eBay/Python/chromedriver", chrome_options=options)
    #url = "https://www.ranker.com/list/most-popular-anime-today/ranker-anime"
    url = "https://www.ranker.com/crowdranked-list/top-50-greatest-animated-films-of-all-time?ref=browse_list&l=1"
    
    title_Eng = []
    print("Page_Num:")
    Page_Num = int(input())
    print("MIN_Price:")
    MIN_Price = int(input())
    print("MAX_Price:")
    MAX_Price = int(input())
    
    
    driver.get(url)
    df = pd.DataFrame()
    df["Title_Eng"] = get_title(title_Eng)
    df["Page_Num"] = [Page_Num for i in range(len(df))]
    df["MIN_Price"] = [MIN_Price for i in range(len(df))]
    df["MAX_Price"] = [MAX_Price for i in range(len(df))]
    
    df.columns = ["Title_Eng","Page_Num","MIN_Price","MAX_Price"]
    df.to_csv(Source_file, index=False)
    # df.to_csv(Source_file)
    driver.quit()            
# res = requests.get(url)
# soup = BeautifulSoup(res.text)

# for title in soup.find_all(class_="listItem__data"):
#     title_Eng.append(title.text)

résultat

une analyse

• Après avoir fait défiler vers le bas de la page, le classement suivant ne sera pas affiché à moins que vous ne cliquiez sur le bouton "CHARGER PLUS" qui apparaît.
• fonction scroll_down
• Après avoir obtenu la position de l'élément, affinez la position de défilement (JavaScript)
• Afficher le classement tant que le bouton "CHARGER PLUS" apparaît
• Obtenez tous les titres de films affichés

Document

• Site de vote utilisé
  • https://www.ranker.com/