Day 1
🔵클러스터링
유명한 비지도 학습
🟢k-means
- 데이터 포인트는 중심에 할당된다. 각 클러스터는 중심을 가진다.
알고리즘 절차
- 초기 중심점 랜덤 선택
- 각 데이터를 가장 가까운 중심에 할당
- 클러스터 중심 재계산
- if 수렴 조건 만족? 끝.
- 조건에 해당되지 않을 시 2번으로 다시 올라감.
- if 수렴 조건 만족? 끝.
코드 실습
import numpy as np
from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs
import matplotlib.pyplot as plt
X, y = make_blobs(n_samples=300, centers=4, cluster_std=0.60, random_state=0)
- 가짜 데이터 생성(make_bolbs)
kmeans = KMeans(
n_clusters= 4 ,
init='random',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(X)- 파라미터 할당해서 kmeans 모델 선언
- 이후 훈련 시도
labels = kmeans.labels_
centroids = kmeans.cluster_centers_- 데이터의 각 클러스터 정보
- array([0, 2, 1, 2, 0, 0, 3, 1, 2, 2, 3, 2, 1, 2, 0, 1, 1, 0, 3, 3, 0, 0,
- 데이터의 중심점 정보
- array([[ 1.98258281, 0.86771314],
[ 0.94973532, 4.41906906],
[-1.37324398, 7.75368871],
[-1.58438467, 2.83081263]])
- array([[ 1.98258281, 0.86771314],
plt.figure(figsize=(10, 6))
plt.scatter(X[:, 0], X[:, 1], c=labels, s=50, cmap='viridis', alpha=0.7)
plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=200, marker='X', label='Centroids')
plt.title('K-means Clustering Results')
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.legend()
plt.show()- 시각화, 빨간 X가 중심점.
- 장점 : 구현이 간단, 이해하기 쉬움, 구현이 간단
- 단점: 이상치에 민감, k값 정답이 없음, 속성 값이 많으면 사용이 어려움
k값을 유추하는 방법 - Elbow method
- kmeans.inertia_ 활용
total = []
for k in range(1, 10):
kmeans = KMeans(
n_clusters= k ,
init='random',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(X)
total.append(kmeans.inertia_)
plt.plot(range(1, 10), total, marker='o')
plt.xlabel('Number of clusters (K)')
plt.ylabel('WCSS')
plt.title('Elbow Method')
plt.show()
아이리스 데이터로 elbow method 보기
- 각 센트로이드 별로 주변 데이터포인트와의 거리를 계산한다
from sklearn.datasets import load_iris
X = load_iris()['data']
#1 K-means++ 개요
#- 문제점: K-means는 초기 중심점 선택이 무작위라 성능이 불안정.
#- 해결: K-means++는 초기 중심점을 확률적으로 선택하여 성능 향상.
total = []
for k in range(1, 10):
kmeans = KMeans(
n_clusters= k ,
init='k-means++',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(X)
total.append(kmeans.inertia_)
plt.plot(range(1, 10), total, marker='o')
plt.xlabel('Number of clusters (K)')
plt.ylabel('WCSS')
plt.title('Elbow Method')
plt.show()군집이 실제 타겟값처럼 잘 형성된건지 알아본다
kmeans = KMeans(
n_clusters= 3 ,
init='k-means++',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(X)
import pandas as pd
iris = pd.DataFrame(load_iris()['data'])
iris.loc[:, 'target'] = load_iris()['target']
iris.loc[:, 'kmeans' ] = kmeans.predict(X)
pd.set_option('display.max_rows', None)
iris| 0 | 1 | 2 | 3 | target | kmeans |
|---|---|---|---|---|---|
| 0 | 5.1 | 3.5 | 1.4 | 0.2 | 0 |
| 1 | 4.9 | 3.0 | 1.4 | 0.2 | 0 |
| 2 | 4.7 | 3.2 | 1.3 | 0.2 | 0 |
| 3 | 4.6 | 3.1 | 1.5 | 0.2 | 0 |
| 4 | 5.0 | 3.6 | 1.4 | 0.2 | 0 |
아이리스 데이터로 실루엣 계수 보기
- 각 군집별 거리를 확인한다.
- 1에 가까울수록 좋다.
- 치역: -1~1
kmeans = KMeans(
n_clusters= 3 ,
init='k-means++',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(X)
import pandas as pd
iris = pd.DataFrame(load_iris()['data'])
iris.loc[:, 'target'] = load_iris()['target']
iris.loc[:, 'kmeans' ] = kmeans.predict(X)
pd.set_option('display.max_rows', None)
#실루엣 지수
from sklearn.metrics import silhouette_score, silhouette_samples
#개별 데이터 실루엣 계수 값 구하기
socre_samples = silhouette_samples(X, kmeans.predict(X))
# 평균 실루엣 값을 계산
silhouette_score(X, kmeans.predict(X))
iris.loc[:, 'silhouette'] = socre_samples
iris.groupby(['kmeans'])['silhouette'].mean()
kmeans
0 0.417320
1 0.798140
2 0.451105
Name: silhouette, dtype: float64
시각화 함수
def visualize_silhouette(cluster_lists, X_features):
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import math
# 입력값으로 클러스터링 갯수들을 리스트로 받아서, 각 갯수별로 클러스터링을 적용하고 실루엣 개수를 구함
n_cols = len(cluster_lists)
# plt.subplots()으로 리스트에 기재된 클러스터링 수만큼의 sub figures를 가지는 axs 생성
fig, axs = plt.subplots(figsize=(4*n_cols, 4), nrows=1, ncols=n_cols)
# 리스트에 기재된 클러스터링 갯수들을 차례로 iteration 수행하면서 실루엣 개수 시각화
for ind, n_cluster in enumerate(cluster_lists):
# KMeans 클러스터링 수행하고, 실루엣 스코어와 개별 데이터의 실루엣 값 계산.
clusterer = KMeans(n_clusters = n_cluster, max_iter=500, random_state=0)
cluster_labels = clusterer.fit_predict(X_features)
sil_avg = silhouette_score(X_features, cluster_labels)
sil_values = silhouette_samples(X_features, cluster_labels)
y_lower = 10
axs[ind].set_title('Number of Cluster : '+ str(n_cluster)+'\n' \
'Silhouette Score :' + str(round(sil_avg,3)) )
axs[ind].set_xlabel("The silhouette coefficient values")
axs[ind].set_ylabel("Cluster label")
axs[ind].set_xlim([-0.1, 1])
axs[ind].set_ylim([0, len(X_features) + (n_cluster + 1) * 10])
axs[ind].set_yticks([]) # Clear the yaxis labels / ticks
axs[ind].set_xticks([0, 0.2, 0.4, 0.6, 0.8, 1])
# 클러스터링 갯수별로 fill_betweenx( )형태의 막대 그래프 표현.
for i in range(n_cluster):
ith_cluster_sil_values = sil_values[cluster_labels==i]
ith_cluster_sil_values.sort()
size_cluster_i = ith_cluster_sil_values.shape[0]
y_upper = y_lower + size_cluster_i
color = cm.nipy_spectral(float(i) / n_cluster)
axs[ind].fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_sil_values, \
facecolor=color, edgecolor=color, alpha=0.7)
axs[ind].text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
y_lower = y_upper + 10
axs[ind].axvline(x=sil_avg, color="red", linestyle="--")
visualize_silhouette([2, 3, 4, 5], X)
- 응집도와 분리도를 모두 평가
- 데이터가 많아지면 계산이 증가 → 속도가 오래 걸림.
🔵고객 데이터로 CRM 하기
- 고객 세분화 - 고객 타겟 마케팅에 활용할 수 있음.
- RFM
- Recency, Frequent, Monetrary Value
- 가장 최근 상품 구입 날짜를 오늘 날짜 기준 계산
- 상품 구매 횟수
- 총 구매 금액
import pandas as pd
df= pd.read_excel("./Online Retail.xlsx")
print(df.shape)
print(df.head(3))(541909, 8)
InvoiceNo StockCode Description Quantity \
0 536365 85123A WHITE HANGING HEART T-LIGHT HOLDER 6
1 536365 71053 WHITE METAL LANTERN 6
2 536365 84406B CREAM CUPID HEARTS COAT HANGER 8
InvoiceDate UnitPrice CustomerID Country
0 2010-12-01 08:26:00 2.55 17850.0 United Kingdom
1 2010-12-01 08:26:00 3.39 17850.0 United Kingdom
2 2010-12-01 08:26:00 2.75 17850.0 United Kingdom- customerID가 없는 데이터는 제거한다,(subset 활용)
df_pivot = df.pivot_table(
index='CustomerID',
columns='StockCode',
values='Quantity'
).fillna(0)
- stockcode 기준으로 유사도를 구해서 각 고객 별로 유사도를 구해본다.
from sklearn.metrics.pairwise import cosine_similarity
user_similarity = cosine_similarity(df_pivot)
df_simil = pd.DataFrame(user_similarity, columns=df_pivot.index, index=df_pivot.index)
df_simil[12347.0].sort_values(ascending=False)- 12347고객과 유사도가 높은 고객
CustomerID 12347.0 1.000000 14326.0 0.776346 12355.0 0.710717 14257.0 0.624245 13532.0 0.574437
이제 두 고객간 상품을 확인해본다 - 협업 필터링 추천 시스템
a = df[df.CustomerID == 14326.0 ]
b= df[df.CustomerID == 12347.0 ]
#a가 구매한 목록중에서 b 구입하지 않았던 제품들 추출
set(a.StockCode.unique()) - set(b.StockCode.unique())
#협업 필터링(Collaborative Filtering)
{22149, 22427, 22441, '72807B', 84946, 84978, '85035A'}
- a가 산 물건 중, b는 사지 않은 것을 추려 내 b에게 추천한다,
파생변수 선언
df['sale_amount'] = df['Quantity'] * df['UnitPrice']
df['CustomerID'] = df.CustomerID.astype(int)
# 주문 기간
# 주문 횟수
# 주문 금액
agg = {
'InvoiceDate' : 'max',
'InvoiceNo' : 'count',
'sale_amount' : 'sum'
}
cust_df = df.groupby('CustomerID').agg(agg)
cust_df.columns = ['Recency', "Frequency", "Monetary"]
cust_df.reset_index(inplace=True)
RFM 시각화
import matplotlib.pyplot as plt
fig, (ax1,ax2,ax3) = plt.subplots(figsize=(12,4), nrows=1, ncols=3)
ax1.set_title('Recency Histogram')
ax1.hist(cust_df['Recency'])
ax2.set_title('Frequency Histogram')
ax2.hist(cust_df['Frequency'])
ax3.set_title('Monetary Histogram')
ax3.hist(cust_df['Monetary'])
plt.show()
군집 만들기.
- RFM 데이터로 군집 만들기
- scaling 적용 Standard Scaler 사용
- K 값을 몇으로 할까?
- 내부와 외부가 잘 구분이 될까?(실루엣지수)
cust_df
from sklearn.preprocessing import StandardScaler
for col in ['Recency', 'Frequency', 'Monetary']:
ss = StandardScaler()
cust_df[col] = ss.fit_transform(cust_df[[col]])
from sklearn.cluster import KMeans
total = []
for k in range(1, 10):
kmeans = KMeans(
n_clusters= k ,
init='k-means++',
n_init=10,
max_iter=300,
random_state=42
)
kmeans.fit(cust_df[['Recency', 'Frequency', 'Monetary']])
total.append(kmeans.inertia_)
plt.plot(range(1, 10), total, marker='o')
plt.xlabel('Number of clusters (K)')
plt.ylabel('WCSS')
plt.title('Elbow Method')
plt.show()
from sklearn.metrics import silhouette_samples, silhouette_score
kmeans = KMeans(n_clusters=3, random_state=0)
labels_ = kmeans.fit_predict(cust_df[['Recency', 'Frequency', 'Monetary']])
cust_df.loc[:, 'labels_'] = labels_
silhouette_data = silhouette_samples( cust_df[['Recency', 'Frequency', 'Monetary']], labels_ )
cust_df.loc[:, 'silhouette_score'] = silhouette_data
cust_df.groupby('labels_').mean().drop('CustomerID',axis=1)
import numpy as np
def visualize_silhouette(cluster_lists, X_features):
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import math
# 입력값으로 클러스터링 갯수들을 리스트로 받아서, 각 갯수별로 클러스터링을 적용하고 실루엣 개수를 구함
n_cols = len(cluster_lists)
# plt.subplots()으로 리스트에 기재된 클러스터링 수만큼의 sub figures를 가지는 axs 생성
fig, axs = plt.subplots(figsize=(4*n_cols, 4), nrows=1, ncols=n_cols)
# 리스트에 기재된 클러스터링 갯수들을 차례로 iteration 수행하면서 실루엣 개수 시각화
for ind, n_cluster in enumerate(cluster_lists):
# KMeans 클러스터링 수행하고, 실루엣 스코어와 개별 데이터의 실루엣 값 계산.
clusterer = KMeans(n_clusters = n_cluster, max_iter=500, random_state=0)
cluster_labels = clusterer.fit_predict(X_features)
sil_avg = silhouette_score(X_features, cluster_labels)
sil_values = silhouette_samples(X_features, cluster_labels)
y_lower = 10
axs[ind].set_title('Number of Cluster : '+ str(n_cluster)+'\n' \
'Silhouette Score :' + str(round(sil_avg,3)) )
axs[ind].set_xlabel("The silhouette coefficient values")
axs[ind].set_ylabel("Cluster label")
axs[ind].set_xlim([-0.1, 1])
axs[ind].set_ylim([0, len(X_features) + (n_cluster + 1) * 10])
axs[ind].set_yticks([]) # Clear the yaxis labels / ticks
axs[ind].set_xticks([0, 0.2, 0.4, 0.6, 0.8, 1])
# 클러스터링 갯수별로 fill_betweenx( )형태의 막대 그래프 표현.
for i in range(n_cluster):
ith_cluster_sil_values = sil_values[cluster_labels==i]
ith_cluster_sil_values.sort()
size_cluster_i = ith_cluster_sil_values.shape[0]
y_upper = y_lower + size_cluster_i
color = cm.nipy_spectral(float(i) / n_cluster)
axs[ind].fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_sil_values, \
facecolor=color, edgecolor=color, alpha=0.7)
axs[ind].text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
y_lower = y_upper + 10
axs[ind].axvline(x=sil_avg, color="red", linestyle="--")
visualize_silhouette([2, 3, 4, 5], cust_df[['Recency', 'Frequency', 'Monetary']])
cust_df.labels_.value_counts()labels_
1 3245
0 1080
2 13
Name: count, dtype: int64
- 부트스트랩 통계 = 랜덤포레스트.
- 데이터 계절성 탄다.
- hive, hadoop
🟢dbscan
import pandas as pd
from sklearn.cluster import DBSCAN
Uber = pd.read_csv("./uber.csv")
Uber_raw = pd.read_csv("./Uber_raw.xls")
df=Uber_raw[['Lat',"Lon"]].sample(n=10000,random_state=42)
from sklearn.preprocessing import StandardScaler
ss=StandardScaler()
df_scaler= ss.fit_transform(df)import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(15,10))
sns.scatterplot(x = df['Lon'], y= df['Lat'])
plt.show()우버를 탄 곳의 위치를 시각화.
- DBSCAN 에서는 한 군집에 최소 5명 있어야한다.
dbscan = DBSCAN(eps=0.5, min_samples=50)
clusters = dbscan.fit_predict(df_scaler)
df['cluster']=clusters
df.cluster.value_counts()cluster
0 9478
-1 253
1 203
2 66
Name: count, dtype: int64
import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(15,10))
sns.scatterplot(x = 'Lon', y= 'Lat', hue='cluster', data = df, palette='viridis', alpha=0.6)
plt.show()
- 밀도 기반
- EPS 값으로 크기를 설정
- min_sample은 한 군집을 이루기 위한 최소한의 데이터의 수
🟢polar로 데이터 처리
- 느긋한 연산법
import polars as pl
q = (
pl.scan_csv('title.basics.tsv.gz', separator='\t', quote_char=None, truncate_ragged_lines=True, ignore_errors=True)
.filter(pl.col('titleType') == 'movie')
.select(['tconst', 'primaryTitle', 'genres'])
)
polar_df = q.collect()
df2= polar_df.to_pandas()- polar 데이터를 특수하게 처리 가능.
- pyspark -11번가에서 사용
uv → 의존성 관리, 에이전트 설계에 유용하다.
Day 2
아침에 새로 파이썬 코드 좀 고쳤는데 뻗음
windows 보안에서 데이터 무결성 끄니까 렉이 해결됨. 아침에 죽. 싶 .
추천 시스템을 구축해본다
import polars as pl
links = pl.scan_csv("./ml-20m/links.csv").\
with_columns( (pl.lit('tt') + pl.col('imdbId').cast(pl.String).str.pad_start(7, '0')).alias('tconst'))
links.explain()
links.collect()
데이터 세팅
lazy_basics = pl.scan_csv('title.basics.tsv.gz', separator='\t', null_values='\\N',
quote_char=None,
truncate_ragged_lines=True, ignore_errors=True).select(
['tconst', 'primaryTitle', 'titleType', 'genres']
)
lazy_basics.show(3)
movies = (links.join(lazy_basics, on='tconst', how='inner')
.filter(pl.col('titleType') == 'movie')
.drop('titleType')
)
movies.show(3)
principals = pl.scan_csv('title.principals.tsv.gz', separator='\t', null_values='\\N',
quote_char=None,
truncate_ragged_lines=True, ignore_errors=True).select(
['tconst', 'nconst', 'category']
)
target_categories = ['director', 'actor', 'actress']
lazy_principals = (
principals.filter(pl.col('category').is_in(target_categories))
.join(movies.select('tconst'), on='tconst', how='semi')
)
lazy_names = pl.scan_csv('./name.basics.tsv.gz', separator='\t', null_values='\\N',
quote_char=None,
truncate_ragged_lines=True, ignore_errors=True).select(
['nconst', 'primaryName']
)
lazy_crew_info = lazy_principals.join(lazy_names, on='nconst', how='left')
lazy_crew_info.show(3)
# 영화별 감독이름, 배우 이름 병합
lazy_directors = (
lazy_crew_info.filter(pl.col('category') == 'director')
.group_by('tconst')
.agg(pl.col('primaryName').drop_nulls().str.join(', ').alias('directors'))
)
lazy_directors.show(3)
lazy_actors = (
lazy_crew_info.filter(pl.col('category').is_in(['actor', 'actress']))
.group_by('tconst')
.agg(pl.col('primaryName').drop_nulls().str.join(', ').alias('cast'))
)
lazy_actors.show(3)
shape: (3, 4)
tconst primaryTitle titleType genres
str str str str
"tt0000001" "Carmencita" "short" "Documentary,Short"
"tt0000002" "Le clown et ses chiens" "short" "Animation,Short"
"tt0000003" "Poor Pierrot" "short" "Animation,Comedy,Romance"
shape: (3, 6)
movieId imdbId tmdbId tconst primaryTitle genres
i64 i64 i64 str str str
89977 10071 70803 "tt0010071" "Don't Change Your Husband" "Comedy"
90907 10821 28629 "tt0010821" "Eerie Tales" "Fantasy,Horror,Mystery"
53853 11870 77621 "tt0011870" "Within Our Gates" "Drama,Romance"
shape: (3, 4)
tconst nconst category primaryName
str str str str
"tt0181530" "nm0063136" "actor" "Erwan Baynaud"
"tt0110425" "nm0063136" "actor" "Erwan Baynaud"
"tt0109816" "nm0063237" "actor" "Eddie Baytos"
shape: (3, 2)
tconst directors
str str
"tt0116165" "Bill Couturié"
"tt0446463" "Vincent Perez"
"tt0138681" "Myles Connell"
shape: (3, 2)
tconst cast
str str
"tt0077321" "Eileen Brennan, Sid Caesar, Ja…
"tt0115493" "Dirk Benedict, Ben Cardinal, D…
"tt0063694" "Sylvia Anderson, Keith Alexand…
최종 조인
lazy_final_dataset = (
movies.join(lazy_directors, on='tconst', how='left')
.join(lazy_actors, on='tconst', how='left')
.with_columns(pl.col(pl.String).fill_null(''))
)
print(lazy_final_dataset.explain()) WITH_COLUMNS:
[col("tconst").fill_null([""]), col("primaryTitle").fill_null([""]), col("genres").fill_null([""]), col("directors").fill_null([""]), col("cast").fill_null([""])]
LEFT JOIN:
LEFT PLAN ON: [col("tconst")]
LEFT JOIN:
LEFT PLAN ON: [col("tconst")]
simple π 6/6 ["movieId", "imdbId", "tmdbId", ... 3 other columns]
INNER JOIN:
LEFT PLAN ON: [col("tconst")]
WITH_COLUMNS:
["tt".str.concat_horizontal([col("imdbId").strict_cast(String).str.pad_start([7])]).alias("tconst")]
Csv SCAN [./ml-20m/links.csv]
PROJECT 3/3 COLUMNS
ESTIMATED ROWS: 38006
RIGHT PLAN ON: [col("tconst")]
simple π 4/4 ["tconst", "primaryTitle", ... 2 other columns]
Csv SCAN [title.basics.tsv.gz]
PROJECT 4/9 COLUMNS
SELECTION: [(col("titleType")) == ("movie")]
ESTIMATED ROWS: 9611960
END INNER JOIN
RIGHT PLAN ON: [col("tconst")]
AGGREGATE[maintain_order: false]
[col("primaryName").drop_nulls().str.concat_vertical().alias("directors")] BY [col("tconst")]
FROM
simple π 3/3 ["tconst", "primaryName", ... 1 other column]
LEFT JOIN:
LEFT PLAN ON: [col("nconst")]
SEMI JOIN:
LEFT PLAN ON: [col("tconst")]
Csv SCAN [title.principals.tsv.gz]
PROJECT 3/6 COLUMNS
SELECTION: [(col("category").is_in([["director", "actor", "actress"]])) & ([(col("category")) == ("director")])]
ESTIMATED ROWS: 58003856
RIGHT PLAN ON: [col("tconst")]
simple π 1/1 ["tconst"]
INNER JOIN:
LEFT PLAN ON: [col("tconst")]
WITH_COLUMNS:
["tt".str.concat_horizontal([col("imdbId").strict_cast(String).str.pad_start([7])]).alias("tconst")]
Csv SCAN [./ml-20m/links.csv]
PROJECT 1/3 COLUMNS
ESTIMATED ROWS: 38006
RIGHT PLAN ON: [col("tconst")]
Csv SCAN [title.basics.tsv.gz]
PROJECT 2/9 COLUMNS
SELECTION: [(col("titleType")) == ("movie")]
ESTIMATED ROWS: 9611960
END INNER JOIN
END SEMI JOIN
RIGHT PLAN ON: [col("nconst")]
Csv SCAN [./name.basics.tsv.gz]
PROJECT 2/6 COLUMNS
ESTIMATED ROWS: 8791635
END LEFT JOIN
END LEFT JOIN
RIGHT PLAN ON: [col("tconst")]
AGGREGATE[maintain_order: false]
[col("primaryName").drop_nulls().str.concat_vertical().alias("cast")] BY [col("tconst")]
FROM
simple π 3/3 ["tconst", "primaryName", ... 1 other column]
LEFT JOIN:
LEFT PLAN ON: [col("nconst")]
SEMI JOIN:
LEFT PLAN ON: [col("tconst")]
Csv SCAN [title.principals.tsv.gz]
PROJECT 3/6 COLUMNS
SELECTION: [(col("category").is_in([["director", "actor", "actress"]])) & (col("category").is_in([["actor", "actress"]]))]
ESTIMATED ROWS: 58003856
RIGHT PLAN ON: [col("tconst")]
simple π 1/1 ["tconst"]
INNER JOIN:
LEFT PLAN ON: [col("tconst")]
WITH_COLUMNS:
["tt".str.concat_horizontal([col("imdbId").strict_cast(String).str.pad_start([7])]).alias("tconst")]
Csv SCAN [./ml-20m/links.csv]
PROJECT 1/3 COLUMNS
ESTIMATED ROWS: 38006
RIGHT PLAN ON: [col("tconst")]
Csv SCAN [title.basics.tsv.gz]
PROJECT 2/9 COLUMNS
SELECTION: [(col("titleType")) == ("movie")]
ESTIMATED ROWS: 9611960
END INNER JOIN
END SEMI JOIN
RIGHT PLAN ON: [col("nconst")]
Csv SCAN [./name.basics.tsv.gz]
PROJECT 2/6 COLUMNS
ESTIMATED ROWS: 8791635
END LEFT JOIN
END LEFT JOIN- explain() ⇒ 실행계획 보는 코드
final_dataset = lazy_final_dataset.collect()- 데이터 담아내기
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
final_dataset.write_csv("./content_based.csv")
df = pd.read_csv('./content_based.csv')
def clean_data(x):
if isinstance(x, str):
return x.replace(' ', '').replace(',', ' ')
return ''
for col in ['genres', 'directors', 'cast']:
df[col] = df[col].apply(clean_data)
df['soup'] = df['genres'] + ' ' + df['directors'] + ' ' + df['cast']
df.head(2)| movieId | imdbId | tmdbId | tconst | primaryTitle | genres | directors | cast | soup |
|---|---|---|---|---|---|---|---|---|
| 0 | 89323 | 2461 | 46758.0 | tt0002461 | The Life and Death of King Richard III | Drama | RobertGemp FrederickWarde AlbertGardner JamesK... | |
| 1 | 100946 | 2844 | 56508.0 | tt0002844 | Fantômas: In the Shadow of the Guillotine | Crime Drama | LouisFeuillade | RenéNavarre RenéNavarre EdmundBreon GeorgesMel... |
TF - IDF 활용해보기
tfidf = TfidfVectorizer(stop_words='english')
tfidf_matrix = tfidf.fit_transform(df['soup'])
cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)
indices = pd.Series(df.index, index=df['primaryTitle']).drop_duplicates()- 기초적 통계 방법론 활용
- 불용어 설정
- 코사인 유사도 활용해서 추천 시스템 설계
추천 받아보기
def get_recommendations(title, cosine_sim=cosine_sim, top_n=10):
if title not in indices:
return []
idx = indices[title]
if isinstance(idx, pd.Series):
idx = idx.iloc[0]
sim_scores = list(enumerate(cosine_sim[idx]))
sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
sim_scores = sim_scores[1:top_n+1]
movie_indices = [i[0] for i in sim_scores]
return df['movieId'].iloc[movie_indices].tolist()
순위 불러오기
ratings = pd.read_csv('./ml-20m/ratings.csv')
from datetime import datetime,timezone
ratings['time'] = ratings.timestamp.apply(lambda x : datetime.fromtimestamp(x, tz=timezone.utc))
links = pd.read_csv("./ml-20m/links.csv")
links.query('movieId==1254')| movieId | imdbId | tmdbId | |
|---|---|---|---|
| 1226 | 1254 | 40897 | 3090.0 |
ratings = pd.read_csv("./title.ratings.tsv.gz", sep='\t')
df.query(" tconst == 'tt0111161'")| movieId | imdbId | tmdbId | tconst | primaryTitle | genres | directors | cast | soup |
|---|---|---|---|---|---|---|---|---|
| 10667 | 318 | 111161 | 278.0 | tt0111161 | The Shawshank Redemption | Drama | FrankDarabont | TimRobbins MorganFreeman BobGunton WilliamSadl... |
df[df.movieId.isin(get_recommendations('The Shawshank Redemption',cosine_sim,10))]| movieId | imdbId | tmdbId | tconst | primaryTitle | genres | directors | cast | soup |
|---|---|---|---|---|---|---|---|---|
| 2881 | 6431 | 47860 | 51426.0 | tt0047860 | Battle Cry | Drama Romance War | RaoulWalsh | VanHeflin AldoRay MonaFreeman NancyOlson James... |
| 9765 | 1171 | 103850 | 10608.0 | tt0103850 | Bob Roberts | Comedy Drama | TimRobbins | TimRobbins GiancarloEsposito AlanRickman RayWi... |
| 11523 | 692 | 117688 | 29621.0 | tt0117688 | Solo | Action Sci-Fi Thriller | NorbertoBarba | MarioVanPeebles WilliamSadler WilliamSadler Ba... |
| 11862 | 8455 | 119561 | 113290.0 | tt0119561 | The Long Way Home | Documentary History War | MarkJonathanHarris | MorganFreeman |
| 11971 | 1646 | 120029 | 36797.0 | tt0120029 | RocketMan | Comedy Family Sci-Fi | StuartGillard | HarlandWilliams JessicaLundy WilliamSadler Jef... |
| 12156 | 3147 | 120689 | 497.0 | tt0120689 | The Green Mile | Crime Drama Fantasy | FrankDarabont | TomHanks MichaelClarkeDuncan DavidMorse Bonnie... |
| 12160 | 1752 | 120696 | 11258.0 | tt0120696 | Hard Rain | Action Crime Drama | MikaelSalomon | MorganFreeman ChristianSlater RandyQuaid Minni... |
| 14338 | 4994 | 268995 | 11086.0 | tt0268995 | The Majestic | Drama Romance | FrankDarabont | JimCarrey MartinLandau BobBalaban JeffreyDeMun... |
| 16260 | 8375 | 391225 | 26254.0 | tt0391225 | The Hunting of the President | Documentary | NickolasPerry HarryThomason | MorganFreeman |
| 18546 | 56145 | 884328 | 5876.0 | tt0884328 | The Mist | Horror Sci-Fi Thriller | FrankDarabont | ThomasJane MarciaGayHarden LaurieHolden AndreB... |
실제 데이터 활용해보기
- surprise는 버전 맞추는 것이 참으로 쉽지 않다………
import pandas as pd
df = pd.read_excel("./sk25.xlsx")
df.rename( columns= { 'Unnamed: 0' : 'name'}, inplace=True)
df2 = df.set_index('name').stack().reset_index()
df2.columns = ['user_id', 'item', 'rating']
df2.loc[df2.rating == '빈칸 채우락끼', 'rating'] = 0
df2 = df2.fillna(0)
df2_train = df2[df2.rating > 0]
df2_train.head()| user_id | item | rating | |
|---|---|---|---|
| 0 | 서찬웅 | 북창동순두부 | 9.0 |
| 1 | 서찬웅 | 유경정육식당 | 8.0 |
| 2 | 서찬웅 | 송셰프 | 9.0 |
| 3 | 서찬웅 | 우리집 | 8.0 |
| 4 | 서찬웅 | 광화문 미진 | 7.0 |
- P 행렬과 Q 행렬로 R 행렬을 만든다.
예측 함수
import numpy as np
class numpy_MF():
def __init__(self, dataframe, K, alpha, beta, iterations):
"""
dataframe: user_id, item, rating 컬럼을 가진 판다스 데이터프레임
K: 잠재 요인(Latent Factor)의 수
alpha: 학습률 (Learning Rate)
beta: 정규화 계수 (Regularization Parameter)
iterations: 반복 횟수 (Epochs)
"""
self.df = dataframe
self.K = K
self.alpha = alpha
self.beta = beta
self.iterations = iterations
# ID를 인덱스로 변환하기 위한 매핑
self.users = self.df['user_id'].unique()
self.items = self.df['item'].unique()
self.user_map = {id: i for i, id in enumerate(self.users)}
self.item_map = {id: i for i, id in enumerate(self.items)}
self.num_users = len(self.users)
self.num_items = len(self.items)
# 파라미터 초기화
self.P = np.random.normal(scale=1./self.K, size=(self.num_users, self.K))
self.Q = np.random.normal(scale=1./self.K, size=(self.num_items, self.K))
self.b_u = np.zeros(self.num_users)
self.b_i = np.zeros(self.num_items)
self.b = np.mean(self.df['rating'])
# 학습용 데이터 리스트 생성 (u, i, r)
self.samples = [
(self.user_map[u], self.item_map[i], r)
for u, i, r in zip(self.df['user_id'], self.df['item'], self.df['rating'])
]
def train(self):
for i in range(self.iterations):
np.random.shuffle(self.samples)
self.sgd()
rmse = self.rmse()
if (i + 1) % 10 == 0:
print(f"Iteration: {i + 1} ; error = {rmse:.4f}")
def sgd(self):
for u, i, r in self.samples:
prediction = self.get_rating(u, i)
e = (r - prediction)
# 편향(Bias) 업데이트
self.b_u[u] += self.alpha * (e - self.beta * self.b_u[u])
self.b_i[i] += self.alpha * (e - self.beta * self.b_i[i])
# 잠재 요인 행렬 업데이트
self.P[u, :] += self.alpha * (e * self.Q[i, :] - self.beta * self.P[u, :])
self.Q[i, :] += self.alpha * (e * self.P[u, :] - self.beta * self.Q[i, :])
def get_rating(self, u, i):
return self.b + self.b_u[u] + self.b_i[i] + self.P[u, :].dot(self.Q[i, :].T)
def rmse(self):
xs, ys = self.df['user_id'].map(self.user_map), self.df['item'].map(self.item_map)
predicted = self.b + self.b_u[xs] + self.b_i[ys] + np.sum(self.P[xs] * self.Q[ys], axis=1)
return np.sqrt(np.mean((self.df['rating'] - predicted)**2))
def predict(self, user_id, food_item):
u = self.user_map.get(user_id)
i = self.item_map.get(food_item)
if u is not None and i is not None:
return self.get_rating(u, i)
else:
return self.b
mf = numpy_MF(df2_train, K=20, alpha=0.01, beta=0.01, iterations=300)
mf.train()mf.predict("홍길동","메가메가")np.float64(8.384516979320342)
지루하게 선명하기보다는 흐릿해도 흥미롭게