[빅데이터시스템] pandas

Pandas

• 데이터 분석을 위한 데이터 처리 라이브러리
• 대용량 데이터를 처리하기 위한 함수, 메소드들이 제공

Series (one-dimensional array)

: index와 data로 이루어진 pandas의 데이터구조

DataFrame (two-dimensional array)

: 각각의 인덱스가 series 로 이루어져 있는 pandas의 데이터구조

Pandas Tutorial

import pandas as pd
import numpy as np
import matplotlib as plt

#check pandas version
pd.__version__

'0.24.1'

#creating a series from a list
arr = [0,1,2,3,4]
print(type(arr))
s1 = pd.Series(arr)
s1

<class 'list'>





0    0
1    1
2    2
3    3
4    4
dtype: int64

s1.index

RangeIndex(start=0, stop=5, step=1)

npArr = np.arange(5)
print(type(npArr))
s1_1=pd.Series(npArr)
s1_1

<class 'numpy.ndarray'>





0    0
1    1
2    2
3    3
4    4
dtype: int64

np.array([0,1,2,3,4])

array([0, 1, 2, 3, 4])

#change index
arr = [0,1,2,3,4]
s1 = pd.Series(arr)
s1

0    0
1    1
2    2
3    3
4    4
dtype: int64

order = [1,2,3,4,5]
s2 = pd.Series(arr, index = order) # index 값을 할당해서 인덱스 변경
s2

1    0
2    1
3    2
4    3
5    4
dtype: int64

n = np.random.randn(5)
n

array([ 0.86042598, -0.96561393, -0.34827275, -0.41878316,  0.24770315])

pd.Series(n)

0    0.860426
1   -0.965614
2   -0.348273
3   -0.418783
4    0.247703
dtype: float64

index = ['a','b','c','d','e']
s2 = pd.Series(n, index = index)
s2

a    0.860426
b   -0.965614
c   -0.348273
d   -0.418783
e    0.247703
dtype: float64

# create a series from dictionary
d = {'a':1, 'b':2, 'c':3, 'd':4, 'e':5}
s3 = pd.Series(d)
s3

a    1
b    2
c    3
d    4
e    5
dtype: int64

# you can modify the index of series
print(s1)
print(s1.index)

0    0
1    1
2    2
3    3
4    4
dtype: int64
RangeIndex(start=0, stop=5, step=1)

s1.index = ['A','B','C','D','E']
s1

A    0
B    1
C    2
D    3
E    4
dtype: int64

print(s1.index)

Index(['A', 'B', 'C', 'D', 'E'], dtype='object')

# slicing
a = s1[:3]
a

A    0
B    1
C    2
dtype: int64

s1[:-1]

A    0
B    1
C    2
D    3
dtype: int64

s1[-2:]

D    3
E    4
dtype: int64

print(s1)
print(s3)
s4 = s1.append(s3)
print(s4)

A    0
B    1
C    2
D    3
E    4
dtype: int64
a    1
b    2
c    3
d    4
e    5
dtype: int64
A    0
B    1
C    2
D    3
E    4
a    1
b    2
c    3
d    4
e    5
dtype: int64

# drop
s4

A    0
B    1
C    2
D    3
E    4
a    1
b    2
c    3
d    4
e    5
dtype: int64

s4.drop('e') # e를 제외하고 출력

A    0
B    1
C    2
D    3
E    4
a    1
b    2
c    3
d    4
dtype: int64

s4 # s4 자체는 변하지 않음

A    0
B    1
C    2
D    3
E    4
a    1
b    2
c    3
d    4
e    5
dtype: int64

s4 = s4.drop('e') # s4를 변화
s4

A    0
B    1
C    2
D    3
E    4
a    1
b    2
c    3
d    4
dtype: int64

Series operation

arr1 = [0,1,2,3,4,5,7]
arr2 = [6,7,8,9,5]

s5 = pd.Series(arr2)
s5

0    6
1    7
2    8
3    9
4    5
dtype: int64

s6 = pd.Series(arr1)
s6

0    0
1    1
2    2
3    3
4    4
5    5
6    7
dtype: int64

s5.add(s6) # 같은 인덱스까리 더해주고, 없는 것들은 NaN

0     6.0
1     8.0
2    10.0
3    12.0
4     9.0
5     NaN
6     NaN
dtype: float64

s5.sub(s6)

0    6.0
1    6.0
2    6.0
3    6.0
4    1.0
5    NaN
6    NaN
dtype: float64

s5.mul(s6)

0     0.0
1     7.0
2    16.0
3    27.0
4    20.0
5     NaN
6     NaN
dtype: float64

s5.div(s6) #  inf

0     inf
1    7.00
2    4.00
3    3.00
4    1.25
5     NaN
6     NaN
dtype: float64

s6

0    0
1    1
2    2
3    3
4    4
5    5
6    7
dtype: int64

print("median", s6.median())
print("max", s6.max())
print("min", s6.min())

median 3.0
max 7
min 0

s7 = s5.mul(s6)
s7

0     0.0
1     7.0
2    16.0
3    27.0
4    20.0
5     NaN
6     NaN
dtype: float64

# NaN은 제외하고 계산
print("median", s5.median())
print("max", s5.max())
print("min", s5.min())

median 7.0
max 9
min 5

Create Dataframe

dates = pd.date_range('today', periods=6)
dates

DatetimeIndex(['2020-09-29 19:59:15.088382', '2020-09-30 19:59:15.088382',
               '2020-10-01 19:59:15.088382', '2020-10-02 19:59:15.088382',
               '2020-10-03 19:59:15.088382', '2020-10-04 19:59:15.088382'],
              dtype='datetime64[ns]', freq='D')

num_arr = np.random.randn(6,4)
num_arr

array([[ 0.38406967, -1.02847711,  0.83911484,  0.79294576],
       [-0.23224823, -0.78362413, -3.17592451,  0.61708198],
       [-1.59406003,  1.646996  ,  0.45997278,  0.28394742],
       [ 0.0244321 , -0.15328843, -1.70151417,  0.64868737],
       [-0.56298435, -0.83298394,  1.26453586, -1.59056282],
       [ 0.01777079,  0.50983863,  2.29359854,  2.04781485]])

columns = ['A','B','C','D']
df1 = pd.DataFrame(num_arr, index = dates, columns = columns)
df1

pd.DataFrame(num_arr)

df1 = pd.DataFrame(num_arr, index = [1,2,3,4,5,6], columns = columns)
df1

# create dataframe with dictionary array

data = {'animal':['cat', 'cat', 'snake', 'dog', 'dog', 'cat', 'snake', 'cat', 'dog','dog'],
       'age':[2.5, 3, 0.5, np.nan, 5, 2, 4.5 , np.nan, 7, 3],
       'visits':[1,3,2,3,2,3,1,1,2,1],
       'priority':['yes', 'yes', 'no', 'no', 'yes', 'yes', 'no', 'no', 'yes', 'no']}
df2 = pd.DataFrame(data)
df2

data = {'animal':['cat', 'cat', 'snake', 'dog', 'dog', 'cat', 'snake', 'cat', 'dog','dog'],
       'age':[2.5, 3, 0.5, np.nan, 5, 2, 4.5 , np.nan, 7, 3],
       'visits':[1,3,2,3,2,3,1,1,2,1],
       'priority':['yes', 'yes', 'no', 'no', 'yes', 'yes', 'no', 'no', 'yes', 'no']}

labels = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']
df2 = pd.DataFrame(data, index = labels)
df2

df2.dtypes

animal       object
age         float64
visits        int64
priority     object
dtype: object

df2.head() # 데이터프레임의 앞 5개만 나옴

df2.head(3)

df3 = df2.head(3)
df3

df2.tail() # tail은 뒤에 5개만 나옴

df2.index

Index(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'], dtype='object')

df2.columns

Index(['animal', 'age', 'visits', 'priority'], dtype='object')

df2.values

array([['cat', 2.5, 1, 'yes'],
       ['cat', 3.0, 3, 'yes'],
       ['snake', 0.5, 2, 'no'],
       ['dog', nan, 3, 'no'],
       ['dog', 5.0, 2, 'yes'],
       ['cat', 2.0, 3, 'yes'],
       ['snake', 4.5, 1, 'no'],
       ['cat', nan, 1, 'no'],
       ['dog', 7.0, 2, 'yes'],
       ['dog', 3.0, 1, 'no']], dtype=object)

type(df2.values)

numpy.ndarray

df2

df2.describe() # 데이터프레임을 바탕을 여러가지 통계값을 출력한다.

df2.T # transpose

df2.sort_values(by='age') # age에 따라서 인덱스들이 정렬된다.

#slicing dataframe
df2[1:3]

df2.sort_values(by='age')[1:3]

# query dataframe by tag
df2

# query dataframe by tag
df2[['age', 'visits']] # [] 대괄호 두개가 필요

type(df2[['age','visits']])

pandas.core.frame.DataFrame

df2

#iloc  slice와 같음
df2.iloc[1:3]

df2[1:3]

print(type(df2.iloc[1:3]))
print(type(df2[1:3]))

<class 'pandas.core.frame.DataFrame'>
<class 'pandas.core.frame.DataFrame'>

df3 = df2.copy()
df3

df3.isnull() # Nan에 True임을 확인할 수 있다

df3

#change value
df3.loc['f','age']

2.0

df3.loc['f','age'] = 1.5
df3 # f의 age가 1.5로 변경

# mean 숫자가 들어있는 row의 mean을 구함
df3.mean()

age       3.166667
visits    1.863636
dtype: float64

df3[['age']].mean()

age    3.166667
dtype: float64

df3['age'].mean()

3.1666666666666665

# 대괄호를 두개 입력해야함. 타입이 달라짐
print(type(df3[['age']].mean()))
print(type(df3['age'].mean()))

<class 'pandas.core.series.Series'>
<class 'float'>

print(df3['age'])
type(df3['age'])

a       2.5
b       3.0
c       0.5
d       NaN
e       5.0
f       1.5
g       4.5
h       NaN
i       7.0
j       3.0
fage    1.5
Name: age, dtype: float64





pandas.core.series.Series

print(df3[['age']])
type(df3[['age']])

      age
a     2.5
b     3.0
c     0.5
d     NaN
e     5.0
f     1.5
g     4.5
h     NaN
i     7.0
j     3.0
fage  1.5





pandas.core.frame.DataFrame

print(df3['visits'].sum())
print(type(df3['visits'].sum()))

20.5
<class 'numpy.float64'>

df3[['visits']].sum()
print(type(df3[['visits']].sum()))

<class 'pandas.core.series.Series'>

# lowwer, upper
string = pd.Series(['A','C','D','Aaa','BaCa',np.nan,'CBA', 'cow', 'owl'])
string

0       A
1       C
2       D
3     Aaa
4    BaCa
5     NaN
6     CBA
7     cow
8     owl
dtype: object

string.str.upper()

0       A
1       C
2       D
3     AAA
4    BACA
5     NaN
6     CBA
7     COW
8     OWL
dtype: object

string.str.lower()

0       a
1       c
2       d
3     aaa
4    baca
5     NaN
6     cba
7     cow
8     owl
dtype: object

Operations for DataFrame missing values

df4 = df3.copy()
df4

df4.fillna(4, inplace=True) # NaN 값에 4를 넣어줌
df4

CSV file

• CSV (comma separated values) file read
• read.csv("csv file")
• head

import pandas as pd

df = pd.read_csv("./8.intropandas/olympics.csv")
df.head()

Index and Skiprows

df = pd.read_csv("./8.intropandas/olympics.csv", index_col=0) # index를 컬럼 0으로 지정
df.head()

df = pd.read_csv("./8.intropandas/olympics.csv", index_col=0, skiprows = 1) # 시작 row를 1번으로 지정
df.head()

Columns and indexs

df.columns

Index(['№ Summer', '01 !', '02 !', '03 !', 'Total', '№ Winter', '01 !.1',
       '02 !.1', '03 !.1', 'Total.1', '№ Games', '01 !.2', '02 !.2', '03 !.2',
       'Combined total'],
      dtype='object')

df.index

Index(['Afghanistan (AFG)', 'Algeria (ALG)', 'Argentina (ARG)',
       'Armenia (ARM)', 'Australasia (ANZ) [ANZ]', 'Australia (AUS) [AUS] [Z]',
       'Austria (AUT)', 'Azerbaijan (AZE)', 'Bahamas (BAH)', 'Bahrain (BRN)',
       ...
       'Uzbekistan (UZB)', 'Venezuela (VEN)', 'Vietnam (VIE)',
       'Virgin Islands (ISV)', 'Yugoslavia (YUG) [YUG]',
       'Independent Olympic Participants (IOP) [IOP]', 'Zambia (ZAM) [ZAM]',
       'Zimbabwe (ZIM) [ZIM]', 'Mixed team (ZZX) [ZZX]', 'Totals'],
      dtype='object', length=147)

Cleaning Dataframes

df.columns 를 보면, 01!, 02!, 03! 으로 금,은,동메달을 표현하는 등 데이터 저장을 위한 표기법을 사용한 경우가 있음. 데이터를 시각화 할 때 이와같은 데이터들을 보기좋게 표현할 수 있음.

for col in df.columns:
    if col[:2] == '01' :
        df.rename(columns={col:"Gold"+col[4:]}, inplace = True)
    if col[:2] == '02' : 
        df.rename(columns={col:"Silver"+col[4:]}, inplace = True)
    if col[:2] == '03' :
        df.rename(columns={col:"Bronze"+col[4:]}, inplace = True)
    if col[:1] == '№' :
        df.rename(columns={col:"#"+col[2:]}, inplace = True)
df.head()

Boolean Masking

df['Gold'].head()

Afghanistan (AFG)           0
Algeria (ALG)               5
Argentina (ARG)            18
Armenia (ARM)               1
Australasia (ANZ) [ANZ]     3
Name: Gold, dtype: int64

(df['Gold'] > 0).head()

Afghanistan (AFG)          False
Algeria (ALG)               True
Argentina (ARG)             True
Armenia (ARM)               True
Australasia (ANZ) [ANZ]     True
Name: Gold, dtype: bool

gold = df.where(df["Gold"]>0)
gold.head()
# True는 값이 나오고 false는 NaN 출력

#drop NaN -> dropna
gold["Gold"].count()

100

df["Gold"].count()

147

gold1 = gold.dropna()
gold1.head()

gold1["Gold"].count()

100

Boolean indexing

df.head()

a = df["Gold"]>0
a.head()

Afghanistan (AFG)          False
Algeria (ALG)               True
Argentina (ARG)             True
Armenia (ARM)               True
Australasia (ANZ) [ANZ]     True
Name: Gold, dtype: bool

b = df[a]
b.head()

And Or

• 가능한 ( )를 사용하여 operation 순서를 정해주는 것이 필요함

df[ (df["Gold"]>0) | \
    (df["Gold.1"]>0) \
  ]

df[ (df["Gold.1"]>0) & \
    (df["Gold"] == 0) \
     ]

Add column

df.head()

df.index

Index(['Afghanistan (AFG)', 'Algeria (ALG)', 'Argentina (ARG)',
       'Armenia (ARM)', 'Australasia (ANZ) [ANZ]', 'Australia (AUS) [AUS] [Z]',
       'Austria (AUT)', 'Azerbaijan (AZE)', 'Bahamas (BAH)', 'Bahrain (BRN)',
       ...
       'Uzbekistan (UZB)', 'Venezuela (VEN)', 'Vietnam (VIE)',
       'Virgin Islands (ISV)', 'Yugoslavia (YUG) [YUG]',
       'Independent Olympic Participants (IOP) [IOP]', 'Zambia (ZAM) [ZAM]',
       'Zimbabwe (ZIM) [ZIM]', 'Mixed team (ZZX) [ZZX]', 'Totals'],
      dtype='object', length=147)

df["Country"] = df.index
df.head()

set_index, reset_index

• 특정한 컬럼을 인덱스로 할 수 있음

df = df.set_index("Gold") # Gold 컬럼을 인덱스로 설정. 
df.head()