import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import plotly
from scipy import stats
import warnings
warnings.filterwarnings("ignore")

from sklearn import preprocessing
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeRegressor

data = pd.read_csv('car_prices.csv', on_bad_lines='skip')
print("row number: ", len(data))
print("column number: ", len(data.columns))

data.head(3)

data.info()

print("Most important features relative to selling price:")
corr = data.corr()
corr.sort_values(["sellingprice"], ascending = False, inplace = True)
print(corr.sellingprice)

data.describe()

data.isnull().sum()

data = data.dropna(how='any')
data.drop(columns=['vin', 'seller', 'saledate','mmr'], inplace=True)

data.shape

%matplotlib inline
data.hist(bins=50, figsize=(20,15))
plt.show()

listtrain = data['make']
  
# prints the missing in listtrain 
print("Missing values in first list:", (set(listtrain).difference(listtrain))) 

data['transmission'].replace(['manual', 'automatic'],
                        [0, 1], inplace=True)

prev_unique = len(data['body'].unique())

for col in data.columns:
    if type(data[col][0]) is str:
        data[col] = data[col].apply(lambda x: x.lower())
        
curr_unique = len(data['body'].unique())   

data.head(5)

print(prev_unique, curr_unique)

data.isnull().sum()

data.dtypes

data.describe()

data.head(3)

sns.distplot(data['sellingprice'])

print("Skewness: %f" % data['sellingprice'].skew())
print("Kurtosis: %f" % data['sellingprice'].kurt())

# applying log transformation
data['sellingprice'] = np.log(data['sellingprice'])
# transformed histogram and normal probability plot
sns.distplot(data['sellingprice'], fit=None);
fig = plt.figure()
res = stats.probplot(data['sellingprice'], plot=plt)

import copy
df_train=copy.deepcopy(data)

cols=np.array(data.columns[data.dtypes != object])
for i in df_train.columns:
    if i not in cols:
        df_train[i]=df_train[i].map(str)
df_train.drop(columns=cols,inplace=True)

df_train.head(10)

from sklearn.preprocessing import LabelEncoder
from collections import defaultdict

# build dictionary function
cols = np.array(data.columns[data.dtypes != object])
d = defaultdict(LabelEncoder)

# only for categorical columns apply dictionary by calling fit_transform 
df_train = df_train.apply(lambda x: d[x.name].fit_transform(x))
df_train[cols] = data[cols]

df_train.head(10)

print("Most important features relative to selling price:")
corr = df_train.corr()
corr.sort_values(["sellingprice"], ascending = False, inplace = True)
print(corr.sellingprice)

ftrain = ['year', 'make', 'model','trim', 'body', 'transmission', 
          'state', 'condition', 'odometer', 'color', 'interior', 'sellingprice']

def Definedata():
    # define dataset
    data2 = df_train[ftrain]
    X = data2.drop(columns=['sellingprice']).values
    y0 = data2['sellingprice'].values
    lab_enc = preprocessing.LabelEncoder()
    y = lab_enc.fit_transform(y0)
    return X, y

%%time
from sklearn.ensemble import RandomForestRegressor
model = RandomForestRegressor()

X, y = Definedata()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25)
model.fit(X_train,y_train)

model.score(X_test, y_test)

0.9487117477394104