import numpy as np
import pandas as pd
import warnings
import seaborn as sns
warnings.filterwarnings("ignore")
import missingno as msno
import matplotlib.pyplot as plt
from collections import Counter
import re
from datetime import datetime
from dateutil.relativedelta import relativedelta
import os
%matplotlib inline
plt.style.use('seaborn')
sns.set(font_scale=1.5,style='darkgrid', context='notebook', palette='deep')
import warnings
from matplotlib import font_manager, rc
font_path = "C:/Windows/Fonts/batang.ttc"
font = font_manager.FontProperties(fname=font_path).get_name()
rc('font', family=font)class Baseline:
def __init__(self):
self.df=''
self.t_a = datetime.now()
self.t_b = datetime.now()
def read_csv(self,path,encoding='utf-8'):
try:
df = pd.read_csv(path,encoding=encoding)
self.df = df
return df
except Exception as e:
return print('err:',str(e))
def time_diff(self,t_a, t_b): ##코드 실행 시간 구하는 함수
t_diff = relativedelta(t_b, t_a) # 끝나는시간이 앞에와야함
return '{h}h {m}m {s}s'.format(h=t_diff.hours, m=t_diff.minutes, s=t_diff.seconds)
def df_info(self,df):
print(df.shape)
df.info()
def msno_mtrx(self,df): ##데이터프레임 결측치 시각화
msno.matrix(df, sort='ascending')
def isnull_rate(self,df): ##데이터프레임 결측치 비율로 시각화
null_rate = [df[i].isna().sum() / len(df) for i in df.columns]
fig, ax = plt.subplots(1,1,figsize=(25, 10))
sns.barplot(x=df.columns, y=null_rate, ax=ax,color='green')
ax.set_title("Missing Value Rate (Train)")
ax.set_xticklabels(df.columns, rotation=50,fontsize=14)
ax.axhline(y=0.9, color='red')
plt.show()
def feature_types_list(self,df): ## 컬럼들 자료형별로 list에 담는 함수
self.int_features = list(df.select_dtypes(include=np.integer).columns)
self.float_features = list(df.select_dtypes(include=np.float).columns)
self.object_features = list(df.select_dtypes(include='object').columns)
print("int featres:", self.int_features ,'\n')
print("float featres:", self.float_features,'\n')
print("object featres:", self.object_features)
def val_count_pieplot(self,df, column_name, sort=True): ## 특정 컬럼에 대한 pieplot 출력
value_count = df[column_name].value_counts(sort=sort).reset_index().rename(columns={column_name:"value_count","index":column_name}).set_index(column_name)
value_count["percentage"] = df[column_name].value_counts(sort=sort,normalize=True)*100
value_count = value_count.reset_index()
display(value_count.sort_values(by=[column_name]))
value_count.set_index(column_name).plot.pie(y="value_count", figsize=(10,8), legend=False, ylabel="",autopct='%1.0f%%',fontsize=17)
plt.show()
def describe(self,df,drop_columns:list =[]): ## 데이터프레임 기초통계량 출력
return df.drop(columns=drop_columns).describe().T\
.style.bar(subset=['mean'] ,color ='plum' )\
.background_gradient(subset=['std'], cmap='Greens')\
.background_gradient(subset=['50%'], cmap='BuGn')
def distploting(self,df,use_features=None,feature_len =None): ##feature들의 distplot 출력(분포)
if use_features ==None:
use_features=self.df.columns
self.use_features = use_features
if feature_len==None:
feature_len= self.use_features.__len__()
plt_psitn = 1
plt.figure()
fig, ax = plt.subplots(feature_len//3+1, 3,figsize=(22,45))
for feature in use_features[:feature_len]:
plt.subplot(feature_len//3+1, 3,plt_psitn)
sns.distplot(df[feature],label=feature,bins=50)
plt.xlabel(feature, fontsize=12)
plt.legend()
plt_psitn += 1
plt.show()
def countploting(self,df,use_features=None,feature_len =None): ## feature들의 countplot 출력(갯수), categorical feature에만 쓰는걸 권유
if use_features ==None:
use_features=self.df.columns
self.use_features = use_features
if feature_len==None:
feature_len= self.use_features.__len__()
plt_psitn = 1
plt.figure()
fig, ax = plt.subplots(feature_len//3+1, 3,figsize=(22,45))
for feature in use_features[:feature_len]:
plt.subplot(feature_len//3+1, 3,plt_psitn)
sns.countplot(df[feature],label=feature,order=df[feature].value_counts().index)
plt.xlabel(feature, fontsize=13)
# plt.xticks()
# plt.legend()
plt_psitn += 1
plt.show()
def countplot_sep(self,df,use_features=None,feature_len =None,target_column='target'): ## feature들을 목표변수 기준으로 countplot출력(갯수), categorical feature에만 쓰는걸 권유
if use_features ==None:
use_features=self.df.columns
self.use_features = use_features
if feature_len==None:
feature_len= self.use_features.__len__()
plt_psitn = 1
plt.figure()
fig, ax = plt.subplots(feature_len//3+1, 3,figsize=(22,45))
for feature in use_features[:feature_len]:
plt.subplot(feature_len//3+1, 3,plt_psitn)
sns.countplot(df[feature],data=df,order=df[feature].value_counts().index,hue=target_column)
plt.xlabel(feature, fontsize=12)
plt.legend()
plt_psitn += 1
plt.show()
def scatterploting(self,df,use_features=None,feature_len =None,target_column='target'):## feature들의 산점도 출력, numeric 에만 쓸것 target도 numuric 일때 쓰는걸 권유
if use_features ==None:
use_features=self.df.columns
self.use_features = use_features
if feature_len==None:
feature_len= self.use_features.__len__()
plt_psitn = 1
plt.figure()
fig, ax = plt.subplots(feature_len//3+1, 3,figsize=(22,45))
for feature in use_features[:feature_len]:
plt.subplot(feature_len//3+1, 3,plt_psitn)
sns.scatterplot(df[feature],y=df[target_column],label=feature)
plt.xlabel(feature, fontsize=12)
plt.legend()
plt_psitn += 1
plt.show()
def boxploting(self,df,drop_columns:list=['id','target']): ## df boxplot 출력 (데이터 이상치 감지용)
plt.subplots(1,1,figsize = (12,10))
sns.boxplot(data=df.drop(columns=drop_columns), orient="h")
def remove_outlires(self,df, columns_list): ##아웃라이어 리무버 (상세 수치는 사용자 판단에 따라 조정)
for col in columns_list:
Q1 = np.percentile(df[col], 25, interpolation = 'midpoint')
Q3 = np.percentile(df[col], 75, interpolation = 'midpoint')
IQR = Q3 - Q1
# Upper bound
upper = np.where(df[col] >= (Q3+1.5*IQR))
# lower bound
lower = np.where(df[col] <= (Q1-1.5*IQR))
# drop outlires
df.drop(upper[0], errors='ignore', inplace = True)
df.drop(lower[0], errors='ignore', inplace = True)
def corr_heatmap(self,df): ## 상관관계분석 출력
mask = np.triu(np.ones_like(df.corr(), dtype=bool))
fig,ax=plt.subplots(1,1,figsize=(20,15))
g = sns.heatmap(df.corr(),cmap='viridis' , center=0,annot=True,annot_kws={'size': 15}, linewidths=.05 , mask=mask, fmt='.2f')
def corr_heatmap_head(self,df): ## 상관관계분석 - 상위 k개 만큼만 출력
k= 10
cols = df.drop(columns=['id']).corr().nlargest(k,'target')['target'].index
print(cols)
cm = np.corrcoef(df[cols].values.T)
fig,ax=plt.subplots(1,1,figsize=(16,10))
g = sns.heatmap(cm, cmap='viridis' , vmin=-1, vmax=1, center=0,annot=True, linewidths=.05 , fmt='.2f', \
annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
def pairplot(self,df): ## 여러 feature 간 산점도 와 히스토그램 출력, numeric 에만 쓸것
sns.pairplot(df.iloc[:,:],hue='target', kind="hist", corner=True)내가 쓰려고 올리는 EDA 베이스라인
