BeautifulSoup4 데이터 크롤링

우선 pip install 로 beautifulsoup4 를 설치해주자.

• mac os terminal
  pip install beautifulsoup4

from bs4 import BeautifulSoup
import requests
import csv

crawling_url = "https://www.billboard.com/charts/hot-100"

html_doc = requests.get(crawling_url)

bs = BeautifulSoup(html_doc.text, 'html.parser')

# 구글 크롬 개발자도구로 엘리먼트를 선택한 후 오른쪽클릭으로 copy selector를 해준다 
music_name = bs.select("#charts > div > div.chart-list.container > ol > li > button > span.chart-element__information > span.chart-element__information__song.text--truncate")
music_singer = bs.select("#charts > div > div.chart-list.container > ol > li > button > span.chart-element__information > span.chart-element__information__artist.text--truncate")

li = []
for cnt in range (len(music_name)):
    li.append({'rank':cnt+1,'name':music_name[cnt].text ,'singer':music_singer[cnt].text})
    #li += {'rank':cnt+1,'name':music_name[cnt].text,'singer':music_singer[cnt].text}
    # 리스트.append를 사용하지 않고 += 를 해주게되면 딕셔너리형태로 되지않음
print(li)

# csv파일로 쓰고 저장하기 위함
csv_filename_to_write = 'billboard_charts_100.csv'
csv_open = open(csv_filename_to_write,'w+',encoding='utf-8')
csv_writer = csv.writer(csv_open)
csv_writer.writerow(li)
csv_open.close