[Python] Pre-processing tricks

Sample data creation

Create a DataFrame from iris data

import pandas as pd
from sklearn.datasets import load_iris


iris = load_iris()
df = pd.DataFrame(iris.data, columns=iris.feature_names)

Create a DataFrame from a dictionary

import pandas as pd
input = {'a': ['red', 'yellow', 'blue'], 'b': [0.5, 0.25, 0.125]}
df = pd.DataFrame(input)

Data read

import pandas as pd

#Excel
df = pd.read_excel('file name.xlsx')

# CSV
df = pd.read_csv('filename.csv', low_memory=False, sep=',', delim_whitespace=False, names=col_names, header=True)

Data confirmation

Statistics

train.describe(include='all')

Pair plot

import seaborn as sns

sns.pairplot(df, vars=df.columns, hue="target")

null check

df.isnull().sum()

Unique number in each column (Distinct)

df.nunique()

frequency

df.value_counts()

histogram

df3['Column name'].plot.hist(bins=40)

sort

#In index order
df.sort_index()

Data processing

One Hot Encoding

import pandas as pd
import numpy as np

df = pd.DataFrame({'A': ['Senior citizens', 'adult', 'adult', "Toddler", "Toddler"], 'B': [2020,2020,2021,2021,1993],
                   'C': [1.0, 2.0, 1.0, np.nan, np.inf], "D":[0,1,2,3,4]})

pd.get_dummies(df, columns=["A", "B"])

#One Hot
df = pd.get_dummies(df, columns=["Column name"], drop_first=True)

#Get only rows that meet the conditions
df = df[df['Column name'] ==value]

#Label names with the word "curry" at 1 and names without the word "curry" at 0
train['curry'] = train['name'].apply(lambda x : 1 if x.find("curry") >=0 else 0)

Handling of DataFrame

#Combine Dataframes vertically
pd.concat([df1, df2, df3], axis=0, ignore_index=True)

#Combine Dataframes horizontally
pd.concat([df1, df2, df3], axis=1)

Handling of columns

#Rename column
df = df.rename(columns={'Change before':'After change'})

#Add column
df = df.assign('Column name'='value')

#Delete column
df = df.drop('Column name', axis=1)

Handling of NULL (NaN)

#Delete lines that contain even one NULL
df = df.dropna(how='any')

#Replace NULL
df = df.fillna({'Column name':value})

One Hot Decode

animals = pd.DataFrame({"monkey":[0,1,0,0,0],"rabbit":[1,0,0,0,0],"fox":[0,0,1,0,0]})

def get_animal(row):
    for c in animals.columns:
        if row[c]==1:
            return c
animals.apply(get_animal, axis=1)

output

#csv output
df.to_csv('file name.csv', index=False)

reference

• https://qiita.com/ao_log/items/fe9bd42fd249c2a7ee7a
• https://qiita.com/chusan/items/d7b210243f3b646375ba
• https://stackoverflow.com/questions/38334296/reversing-one-hot-encoding-in-pandas/38334528
• Use Pandas get_dummies for One-Hot encoding | Shikoan's ML Blog
• https://www.renom.jp/ja/notebooks/tutorial/preprocessing/category_encoding/notebook.html
• https://qiita.com/uratatsu/items/8bedbf91e22f90b6e64b