Web Data 03. Naver API

01. 검색할 url 만들기

02. 네이버 검색 api사용하기

urllib: http 프로토콜에 따라서 서버의 요청/응답을 처리하기 위한 모듈
urllib.request: 클라이언트의 요청을 처리하는 모듈
urllib.parse: url주소에 대한 분석
print(response_body.decode("utf-8")) : 글자로 읽을 경우,decode utf-8로 디코딩필요
response.getcode(),response,response.code,response.status: 200나오면 정상으로 받아져온 것.

• 검색: 블로그(blog), 카페(cafearticle), 네이버쇼핑(shop), 네이버 백과사전(encyc) : url값만 변경

# 네이버 검색 API 예제 - 블로그 검색
import os
import sys
import urllib.request

client_id = "본인id"
client_secret = "본인secret"
encText = urllib.parse.quote("파이썬")


url = "https://openapi.naver.com/v1/search/blog?query=" + encText # JSON 결과
# url = "https://openapi.naver.com/v1/search/blog.xml?query=" + encText # XML 결과
request = urllib.request.Request(url)
request.add_header("X-Naver-Client-Id",client_id)
request.add_header("X-Naver-Client-Secret",client_secret)
response = urllib.request.urlopen(request)
rescode = response.getcode()
if(rescode==200):
    response_body = response.read()
    print(response_body.decode('utf-8'))
else:
    print("Error Code:" + rescode)

03. 상품검색

# 네이버 검색 API 예제 - 몰스킨 검색
import os
import sys
import urllib.request

client_id = "본인id"
client_secret = "본인secret"
encText = urllib.parse.quote("몰스킨")

url = "https://openapi.naver.com/v1/search/shop?query=" + encText # JSON 결과 #책으로변경
# url = "https://openapi.naver.com/v1/search/blog.xml?query=" + encText # XML 결과
request = urllib.request.Request(url)
request.add_header("X-Naver-Client-Id",client_id)
request.add_header("X-Naver-Client-Secret",client_secret)
response = urllib.request.urlopen(request)
rescode = response.getcode()
if(rescode==200):
    response_body = response.read()
    print(response_body.decode('utf-8'))
else:
    print("Error Code:" + rescode)

1. gen_search_url()

def gen_search_url(api_node, search_text, start_num, disp_num):

    base = "https://openapi.naver.com/v1/search"
    node = "/" + api_node + ".json"
    param_query = "?query=" + urllib.parse.quote(search_text)
    param_start = "&start="+ str(start_num)
    param_disp = "&display=" + str(disp_num)
    return base + node + param_query + param_start + param_disp

gen_search_url("shop","TEST",10,3)

2. get_result_onepage()

import json
import datetime

def get_result_onepage(url):
    request = urllib.request.Request(url)
    request.add_header("X-Naver-Client-Id", client_id)
    request.add_header("X-Naver-Client-Secret", client_secret)
    response = urllib.request.urlopen(request)
    print("[%s] Url Request Suceess" % datetime.datetime.now())
    return json.loads(response.read().decode("utf-8"))

datetime.datetime.now()

#shop에 요청 "몰스킨"을 1번째부터 5번째까지   
url = gen_search_url("shop", "몰스킨", 1, 5)
one_result = get_result_onepage(url)

1) 
one_result

2)
one_result["items"] #딕셔너리 형태로 되어있음. items라는 key값에 접근  

3)
one_result["items"][0]["title"]

3. get_field()

import pandas as pd

def get_field(json_data):
    title = [each["title"] for each in json_data["items"]]
    link = [each["link"] for each in json_data["items"]]
    lprice = [each["lprice"] for each in json_data["items"]]
    mall_name = [each["mallName"] for each in json_data["items"]]
    
    result_pd = pd.DataFrame({
        "title":title,
        "link":link,
        "lprice":lprice,
        "mall":mall_name,
    }, columns=["title","lprice","link","mall"])
    return result_pd

get_field(one_result)

4. html태그 지우기

def delete_tag(input_str):
    input_str= input_str.replace("<b>","")
    input_str= input_str.replace("</b>","")
    return input_str

import pandas as pd

def get_field(json_data):
    title = [delete_tag(each["title"]) for each in json_data["items"]]
    link = [each["link"] for each in json_data["items"]]
    lprice = [each["lprice"] for each in json_data["items"]]
    mall_name = [each["mallName"] for each in json_data["items"]]
    
    result_pd = pd.DataFrame({
        "title":title,
        "link":link,
        "lprice":lprice,
        "mall":mall_name,
    }, columns=["title","lprice","link","mall"])
    return result_pd
    
 get_field(one_result)
 
 url = gen_search_url("shop","몰스킨",1,5) #shop에 요청 첫페이지부터 5페이지까지
json_result= get_result_onepage(url)
pd_result= get_field(json_result)

pd_result

5. actMain()

for n in range(1,1000,100):
    print(n)

result_mol = []

for n in range(1,1000,100):
    url = gen_search_url("shop","몰스킨",n,100) #shop에 요청 첫페이지부터 5페이지까지
    json_result= get_result_onepage(url)
    pd_result= get_field(json_result)
    
    result_mol.append(pd_result)

result_mol = pd.concat(result_mol)

result_mol.info()
result_mol.tail()

result_mol["lprice"]= result_mol["lprice"].astype("float")
result_mol.info()

6. to_excel()

!pip install xlsxwriter

#파이썬으로 엑셀작성하는 문법
writer = pd.ExcelWriter("../data/06_molskin_diary_in_naver_shop.xlsx",engine="xlsxwriter")
result_mol.to_excel(writer,sheet_name="Sheet1")

workbook=writer.book
worksheet = writer.sheets["Sheet1"]
worksheet.set_column("A:A",4)
worksheet.set_column("B:B",60)
worksheet.set_column("C:C",10)
worksheet.set_column("D:D",10)
worksheet.set_column("E:E",50)
worksheet.set_column("F:F",10)

worksheet.conditional_format("C2:C1001",{"type":"3_color_scale"})
writer.save()

!ls "../data/06_molskin_diary_in_naver_shop.xlsx"

7. 시각화

import matplotlib.pyplot as plt
import seaborn as sns
import platform
import matplotlib.font_manager as fm

plt.figure(figsize=(15,6))
sns.countplot(
    x = "mall", #result_mol["mall"]예제인데 나는 안되서 바꿔서 진행 
    data=result_mol,
    palette="RdYlGn",
    order=result_mol["mall"].value_counts().index #쇼핑몰 숫자가 어디가 제일 많은지
)
plt.show()

• 이글은 제로베이스 데이터 취업 스쿨의 강의자료 일부를 발췌하여 작성되었음.