Pandas 与 PySpark 强强联手，功能与速度齐飞！

import pyspark.pandas as ps
1.

# import Pandas-on-Spark 
import pyspark.pandas as ps

# 使用 Pandas-on-Spark 创建一个 DataFrame 
ps_df = ps.DataFrame(range(10))

# 将 Pandas-on-Spark Dataframe 转换为 Pandas Dataframe 
pd_df = ps_df.to_pandas()

# 将 Pandas Dataframe 转换为 Pandas-on-Spark Dataframe 
ps_df = ps.from_pandas(pd_df)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

# 使用 Pandas-on-Spark 创建一个 DataFrame 
ps_df = ps.DataFrame(range(10))

# 将 Pandas-on-Spark Dataframe 转换为 Spark Dataframe 
spark_df = ps_df.to_spark()

# 将 Spark Dataframe 转换为 Pandas-on-Spark Dataframe 
ps_df_new = spark_df.to_pandas_on_spark()
1.
2.
3.
4.
5.
6.
7.
8.

>>> sdf = spark.createDataFrame([
...     (1, Decimal(1.0), 1., 1., 1, 1, 1, datetime(2020, 10, 27), "1", True, datetime(2020, 10, 27)),
... ], 'tinyint tinyint, decimal decimal, float float, double double, integer integer, long long, short short, timestamp timestamp, string string, boolean boolean, date date')
>>> sdf
1.
2.
3.
4.

DataFrame[tinyint: tinyint, decimal: decimal(10,0),
float: float, double: double, integer: int,
long: bigint, short: smallint, timestamp: timestamp, 
string: string, boolean: boolean, date: date]
1.
2.
3.
4.

psdf = sdf.pandas_api()
psdf.dtypes
1.
2.

tinyint                int8
decimal              object
float               float32
double              float64
integer               int32
long                  int64
short                 int16
timestamp    datetime64[ns]
string               object
boolean                bool
date                 object
dtype: object
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.

# 运行Spark
from pyspark.sql import SparkSession
spark = SparkSession.builder \
          .appName("Spark") \
          .getOrCreate()
# 在Spark上运行Pandas
import pyspark.pandas as ps
1.
2.
3.
4.
5.
6.
7.

# SPARK 
sdf = spark.read.options(inferSchema='True', 
              header='True').csv('iris.csv')
# PANDAS-ON-SPARK 
pdf = ps.read_csv('iris.csv')
1.
2.
3.
4.
5.

# SPARK 
sdf.select("sepal_length","sepal_width").show()
# PANDAS-ON-SPARK 
pdf[["sepal_length","sepal_width"]].head()
1.
2.
3.
4.

# SPARK 
sdf.drop('sepal_length').show()# PANDAS-ON-SPARK 
pdf.drop('sepal_length').head()
1.
2.
3.

# SPARK 
sdf.dropDuplicates(["sepal_length","sepal_width"]).show()
# PANDAS-ON-SPARK 
pdf[["sepal_length", "sepal_width"]].drop_duplicates()
1.
2.
3.
4.

# SPARK 
sdf.filter( (sdf.flower_type == "Iris-setosa") & (sdf.petal_length > 1.5) ).show()
# PANDAS-ON-SPARK 
pdf.loc[ (pdf.flower_type == "Iris-setosa") & (pdf.petal_length > 1.5) ].head()
1.
2.
3.
4.

# SPARK 
sdf.filter(sdf.flower_type == "Iris-virginica").count()
# PANDAS-ON-SPARK 
pdf.loc[pdf.flower_type == "Iris-virginica"].count()
1.
2.
3.
4.

# SPARK 
sdf.select("flower_type").distinct().show()
# PANDAS-ON-SPARK 
pdf["flower_type"].unique()
1.
2.
3.
4.

# SPARK 
sdf.sort("sepal_length", "sepal_width").show()
# PANDAS-ON-SPARK 
pdf.sort_values(["sepal_length", "sepal_width"]).head()
1.
2.
3.
4.

# SPARK 
sdf.groupBy("flower_type").count().show()
# PANDAS-ON-SPARK 
pdf.groupby("flower_type").count()
1.
2.
3.
4.

# SPARK 
sdf.replace("Iris-setosa", "setosa").show()
# PANDAS-ON-SPARK 
pdf.replace("Iris-setosa", "setosa").head()
1.
2.
3.
4.

#SPARK 
sdf.union(sdf)
# PANDAS-ON-SPARK 
pdf.append(pdf)
1.
2.
3.
4.

DataFrame.transform() 
DataFrame.apply()
DataFrame.pandas_on_spark.transform_batch()  
DataFrame.pandas_on_spark.apply_batch()  
Series.pandas_on_spark.transform_batch()
1.
2.
3.
4.
5.

# transform
psdf = ps.DataFrame({'a': [1,2,3], 'b':[4,5,6]})
def pandas_plus(pser):
    return pser + 1  # 应该总是返回与输入相同的长度。

psdf.transform(pandas_plus)

# apply
psdf = ps.DataFrame({'a': [1,2,3], 'b':[5,6,7]})
def pandas_plus(pser):
    return pser[pser % 2 == 1]  # 允许任意长度

psdf.apply(pandas_plus)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.

psdf = ps.DataFrame({'a': [1,2,3], 'b':[4,5,6]})
def pandas_plus(pser):
    return sum(pser)  # 允许任意长度
psdf.apply(pandas_plus, axis='columns')
1.
2.
3.
4.

psdf = ps.DataFrame({'a': [1,2,3], 'b':[4,5,6]})
def pandas_plus(pdf):
    return pdf + 1  # 应该总是返回与输入相同的长度。

psdf.pandas_on_spark.transform_batch(pandas_plus)

psdf = ps.DataFrame({'a': [1,2,3], 'b':[4,5,6]})
def pandas_plus(pdf):
    return pdf[pdf.a > 1]  # 允许任意长度

psdf.pandas_on_spark.apply_batch(pandas_plus)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

import pyspark.pandas as ps
psdf = ps.DataFrame({'a': [1, 2], 'b':[3, 4]})
psdf.columns = ["a", "a"]
1.
2.
3.

Reference 'a' is ambiguous, could be: a, a.;
1.

import pyspark.pandas as ps
psdf = ps.DataFrame({'a': [1, 2], 'A':[3, 4]})
1.
2.

Reference 'a' is ambiguous, could be: a, a.;
1.

from pyspark.sql import SparkSession
builder = SparkSession.builder.appName("pandas-on-spark")
builder = builder.config("spark.sql.caseSensitive", "true")
builder.getOrCreate()

import pyspark.pandas as ps
psdf = ps.DataFrame({'a': [1, 2], 'A':[3, 4]})
psdf
1.
2.
3.
4.
5.
6.
7.
8.

a  A
0  1  3
1  2  4
1.
2.
3.

>>> import pandas as pd
>>> max(pd.Series([1, 2, 3]))
3
>>> min(pd.Series([1, 2, 3]))
1
>>> sum(pd.Series([1, 2, 3]))
6
1.
2.
3.
4.
5.
6.
7.

>>> import pyspark.pandas as ps
>>> ps.Series([1, 2, 3]).max()
3
>>> ps.Series([1, 2, 3]).min()
1
>>> ps.Series([1, 2, 3]).sum()
6
1.
2.
3.
4.
5.
6.
7.

import pandas as pd
data = []
countries = ['London', 'New York', 'Helsinki']
pser = pd.Series([20., 21., 12.], index=countries)
for temperature in pser:
     assert temperature > 0
     if temperature > 1000:
         temperature = None
     data.append(temperature ** 2)

pd.Series(data, index=countries)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

London      400.0
New York    441.0
Helsinki    144.0
dtype: float64
1.
2.
3.
4.

import pyspark.pandas as ps
import numpy as np
countries = ['London', 'New York', 'Helsinki']
psser = ps.Series([20., 21., 12.], index=countries)
def square(temperature) -> np.float64:
     assert temperature > 0
     if temperature > 1000:
         temperature = None
     return temperature ** 2

psser.apply(square)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

London      400.0
New York    441.0
Helsinki    144.0
1.
2.
3.