pySpark2 - 기본연산

reduceByKey add

import pyspark
from operator import add 
sc = pyspark.SparkContext.getOrCreate()
rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
sorted(rdd.reduceByKey(add).collect())

groupByKey, mapValues

rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])

#key 별 갯수
rdd.groupByKey().mapValues(len).collect()
#[('a', 2), ('b', 1)]

#key 별 values list
sorted(rdd.groupByKey().mapValues(list).collect())
#[('a', [1, 1]), ('b', [1])]

#key sort 
tmp = [('a', 1), ('b', 2), ('1', 3), ('d', 4), ('2', 5)]
sc.parallelize(tmp).sortByKey().first()
#('1', 3)

#key추출
rdd = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
rdd.keys()
#['a', 'b', 'a']

#inner join 
x = sc.parallelize([("a", 1), ("b", 4)])
y = sc.parallelize([("a", 2), ("a", 3)])
sorted(x.join(y).collect())
#[('a', (1, 2)), ('a', (1, 3))]