๐Ÿ•ทData Crawling 3

Gyeomiiยท2022๋…„ 7์›” 14์ผ
0

DDITPython

๋ชฉ๋ก ๋ณด๊ธฐ
15/18
post-thumbnail

1๋ถ„๋งˆ๋‹ค .py์‹คํ–‰ํ•ด์„œ DB์— ์ž๋ฃŒ ์ €์žฅํ•˜๊ธฐ

์ฝ”๋“œ

import requests
from datetime import datetime
from bs4 import BeautifulSoup   
from stockdao import StockDao

sd = StockDao()
ymd = datetime.today().strftime("%Y%m%d.%H%M")

url = "https://vip.mk.co.kr/newSt/rate/item_all.php"

response = requests.get(url)

if response.status_code == 200:
    
    html = response.content.decode('euc-kr','replace')
    soup = BeautifulSoup(html, 'html.parser')
    
    attr1 = {'class' : 'st2', 'width' : '92'}
    
    tds = soup.find_all('td', attrs = attr1)
    #attr2 = {'width':'60'}
    #price = soup.find_all('td', attrs= attr2)
    
    count = 0
    for idx, td in enumerate(tds):
        s_code = td.a['title']
        s_name = td.text
        price = td.parent.find_all("td")[1].text.replace(",","")
        
        cnt = sd.insert(s_code, ymd, s_name, price)
        
        print(idx+1, "cnt", cnt)
        count += 1
        
    print(count, "๊ฐœ์˜ ๋ ˆ์ฝ”๋“œ๊ฐ€ ์ž…๋ ฅ๋˜์—ˆ์Šต๋‹ˆ๋‹ค.")

else : 
    print(response.status_code)

dao

import pymysql

class StockDao:
    def __init__(self):
        self.conn = pymysql.connect(host='localhost', port=3305,
                       user='root', password='python',
                       db='python', charset='utf8')
        
        self.cur = self.conn.cursor(pymysql.cursors.DictCursor)
    
    def insert(self, s_code, ymd, s_name, price):
        sql = f"""
            Insert into stock (s_code, ymd, s_name, price) 
            values ('{s_code}', '{ymd}', '{s_name}', '{price}')
        """
        # ๋ฌธ์ž์—ด ์ž…๋ ฅํ•  ๋•Œ ์•ž๋’ค๋กœ ' ' ์ค‘์š”
        cnt = self.cur.execute(sql)
        self.conn.commit()
        return cnt

    def __del__(self):
        self.cur.close()
        self.conn.close()
        
if __name__ == '__main__':
    dao = StockDao()
  • ์šฐ์„  ๋ฐ์ดํ„ฐ๋ฅผ DB์— ์ €์žฅํ•  ์ˆ˜ ์žˆ๋„๋ก Insert๋ฅผ ํ•ด์ค€๋‹ค.
  • ๊ทธ๋ฆฌ๊ณ  .batํŒŒ์ผ ์„ ๋งŒ๋“ ๋‹ค.

  • ์œˆ๋„์šฐ ์ž‘์—…์Šค์ผ€์ฅด๋Ÿฌ์— 1๋ถ„๋งˆ๋‹ค .batํŒŒ์ผ์„ ์‹คํ–‰ํ•˜๋„๋ก ๋งŒ๋“ ๋‹ค.

  • 1๋ถ„๋งˆ๋‹ค .batํŒŒ์ผ์ด ์‹คํ–‰๋˜๋ฉฐ DB์— ์ž๋ฃŒ๊ฐ€ ์ €์žฅ๋œ๋‹ค.

profile
๊น€์„ฑ๊ฒธ

0๊ฐœ์˜ ๋Œ“๊ธ€

๊ด€๋ จ ์ฑ„์šฉ ์ •๋ณด