[Crawling] lck 팀 로스터 크롤링하기

lck 팀 로스터 크롤링하기

1.패키지 설치하기

pip install selenium
pip install webdriver_manager
pip install django

2.settings.py

DATABASES = {
    "default":{
        "ENGINE":"django.db.backends.mysql",
        "NAME":"lol_db", # 접속할 DB명
        "USER":"user", # 접속 계정명
        "PASSWORD":"password", # 암호
        "HOST":"localhost",
        "PORT":"3306", # 접속  포트 번호
    }
}

3.models.py 모델 정의하기

from django.db import models

# 팀 정보
class Player(models.Model):
    team_name = models.CharField(max_length=100)
    position_name = models.CharField(max_length=10)
    player_photo = models.URLField()
    player_name = models.CharField(max_length=100)

    def __str__(self):
        return self.player_name

4.마이그레이션

python manage.py makemigrations
python manage.py migrate

5.management/commands/crawl_players.py

import os
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from django.core.management.base import BaseCommand
from riot_app.models import Player

class Command(BaseCommand):
    help = 'Crawl all teams and their player data and save to database'

    def handle(self, *args, **kwargs):
        # 크롬드라이버 설정
        service = Service(ChromeDriverManager().install())
        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-dev-shm-usage')
        driver = webdriver.Chrome(service=service, options=options)

		# 크롤링할 사이트 입력하기
        team_urls = [
            '크롤링할 사이트', # T1
            '크롤링할 사이트', # 젠지
            '크롤링할 사이트', # KT 롤스터
            '크롤링할 사이트', # Dplus KIA
            '크롤링할 사이트', # 한화생명e스포츠
            '크롤링할 사이트', # BNK 피어엑스
            '크롤링할 사이트', # 광동 프릭스
            '크롤링할 사이트', # OK저축은행 브리온
            '크롤링할 사이트', # DRX
            '크롤링할 사이트'  # 농심 레드포스
        ]

        # 팀 ID와 이름 매핑
        team_names = {
            'R1040': 'T1',
            'R479': '젠지',
            'R105': 'KT 롤스터',
            'R1152': 'Dplus KIA',
            'R480': '한화생명e스포츠',
            'R1070': 'BNK 피어엑스',
            'R1118': '광동 프릭스',
            'R1071': 'OK저축은행 브리온',
            'R1041': 'DRX',
            'R1072': '농심 레드포스'
        }

        # Explicit wait 설정
        wait = WebDriverWait(driver, 20)

        for team_url in team_urls:
            team_id = team_url.split('teamId=')[-1]
            team_name = team_names.get(team_id, 'Unknown')

            driver.get(team_url)
            time.sleep(5)  # 페이지 로딩 대기

            try:
                self.stdout.write(f'DEBUG: Team Name: {team_name}')

                # 로스터 테이블 찾기
                roster_table = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'table.roster_table__3El2v')))
                rows = roster_table.find_elements(By.CSS_SELECTOR, 'tbody tr')
                self.stdout.write(f'DEBUG: Number of rows found: {len(rows)}')

                # 포지션 이름들을 각 열의 헤더에서 찾기
                headers = roster_table.find_elements(By.CSS_SELECTOR, 'thead th')
                position_names = [header.find_element(By.CSS_SELECTOR, 'span.roster_position__aG0p8').text for header in headers]
                self.stdout.write(f'DEBUG: Position Names: {position_names}')

                for row in rows:
                    columns = row.find_elements(By.CSS_SELECTOR, 'td')
                    for position_name, col in zip(position_names, columns):
                        players = col.find_elements(By.CSS_SELECTOR, 'li.roster_item__2Fksn')
                        for player in players:
                            try:
                                player_photo_element = player.find_element(By.CSS_SELECTOR, 'div.roster_thumbnail__2Y8cc')
                                player_photo = player_photo_element.get_attribute('style').split('"')[1]
                                self.stdout.write(f'DEBUG: Player Photo: {player_photo}')
                            except Exception as e:
                                self.stdout.write(self.style.ERROR(f'Error finding player photo: {e}'))
                                continue
                            
                            try:
                                player_name_element = player.find_element(By.CSS_SELECTOR, 'strong.roster_nickname__3c3iB')
                                player_name = player_name_element.text
                                self.stdout.write(f'DEBUG: Player Name: {player_name}')
                            except Exception as e:
                                self.stdout.write(self.style.ERROR(f'Error finding player name: {e}'))
                                continue

                            # 데이터 저장
                            try:
                                Player.objects.create(
                                    team_name=team_name,
                                    position_name=position_name,
                                    player_photo=player_photo,
                                    player_name=player_name
                                )
                                self.stdout.write(self.style.SUCCESS(f'Successfully saved player: {player_name}'))
                            except Exception as e:
                                self.stdout.write(self.style.ERROR(f'Error saving player data: {e}'))
            except Exception as e:
                self.stdout.write(self.style.ERROR(f'Error during crawling team: {e}'))

        driver.quit()
        self.stdout.write(self.style.SUCCESS('Successfully crawled and saved player data for all teams'))

6.크롤링 스크립트 실행

python manage.py crawl_players

데이터베이스에 저장 성공 !

결과 ⭐️

lck 팀 로스터 크롤링하기 성공 !