Basic Data Exploration - youdar/How-to GitHub Wiki

Basic data exploration

Other ref

Pedro Marcelino: Comprehensive data exploration with Python

Exploring features...

When having a data with mix numeric and non-numeric fields

• Convert the non-numeric fields to numeric (not using One-Hot encoding method)
• Update the data set, without creating new columns
• Plot simple heatmap

Note that what I show here is not quite the best way...
I converted categorical values to numbers and then used correlation matrix.
Consider working on the categorical features separately, and use Chi-Square
to evaluate correlation.

import pandas as pd   
import numpy as np 
from sklearn.preprocessing import LabelEncoder 
import matplotlib.pyplot as plt 
import seaborn as sns 
from itertools import product

pd.set_option('display.width', 1000)

# Collect the data
df = pd.read_csv(file_name, low_memory=False)

# Making all columns numerical types
cols = df.select_dtypes(include=['object', 'bool','O']).columns.tolist()
# Make sure there are no mixed types columns
df[cols] = df[cols].astype(np.str)
le = LabelEncoder()
df[cols] = df[cols].apply(le.fit_transform, axis=0, result_type='broadcast').astype('int')

# map columns names to letters, to avoid having to long names, so that they will plot nicer
col_names = df.columns.tolist()
names_opts = [''.join(x) for x in product('ABCDEFGHIJ', repeat=3)]
col_to_tmp = dict([x for x in zip(col_names, names_opts)])
tmp_to_col = dict([x for x in zip(names_opts, col_names)])
df = df.rename(columns=col_to_tmp)

# Correlation Heapmap, method : {'pearson', 'kendall', 'spearman'}
corr_matrix = df.corr(method='pearson').abs() 
# Try to reduce the correlation matrix by removing rows and columns that are not of interest
corr_matrix = corr_matrix[corr_matrix > 0.9].dropna(how='all' ,axis=1)
corr_matrix = corr_matrix[corr_matrix > 0.9].dropna(how='all', axis=0)

# to look at a section of the correlation matrix (if is to big)
# and to make the high values stand out
corr = corr_matrix.iloc[:50, :50].copy()
corr = corr.applymap(lambda x: x if x >= 0.9 else 0)

# Mask top half of the heatmap
mask = np.zeros_like(corr)
mask[np.triu_indices_from(mask)] = True
sns.set(rc={'figure.figsize': (22.5, 12)})
sns.axes_style("white")
ax = sns.heatmap(corr, mask=mask, annot=False, cmap='Blues', xticklabels=True, yticklabels=True)
# when looking at a large number of fields, consider using xticklabels=False, yticklabels=False
# to get a sense of the data


# If not in jupyter notebook and want to wait
plt.show()
input('Wait for input...')

# look at fields of interest using a scatter-plot
sns.set()
# Which columns to plot
cols_to_use = ['ABC','ACD','ABA','ABH']
ax2 = sns.pairplot(df[cols_to_use], dropna=True)

print(tmp_to_col['ABC'])
print(tmp_to_col['ACD'])  

# Get a list of correlating fields
# Convert NaN to 0
sdf = corr.copy().applymap(lambda x: x if not pd.isna(x) else 0)
# or 
sdf = corr.copy().replace(np.nan,0).head()
#
sdf = csr_matrix(sdf.values)
values_pos = find(sdf)
corr_fields_list_1 = [tmp_to_col[x] for x in corr.index[values_pos[0]]]
corr_fields_list_2 = [tmp_to_col[x] for x in corr.index[values_pos[1]]]
corr_value = values_pos[2].copy()
for f1, f2, v in zip(corr_fields_list_1, corr_fields_list_2, corr_value):
    if f1 != f2:
        print('{:<50} : {:<50} : {:.2f}'.format(f1, f2, v)) 

Reducing dimensionality

Basic Principal component analysis (PCA) task