import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from statsmodels.stats.outliers_influence import variance_inflation_factor
import seaborn as sns
%matplotlib inline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
import sys
from statsmodels.stats.diagnostic import normal_ad

if not sys.warnoptions:
    import warnings
    warnings.simplefilter("ignore")

from google.colab import drive
drive.mount('/content/drive/')

%cd drive/MyDrive/dataset/uber-and-lyft-dataset-boston-ma

df = pd.read_csv('rideshare_kaggle.csv')
df

#Mengetahui jumlah kolom, serta tipe data
df.info()

df.dtypes.value_counts()

#Mengecek kolom yang bertipe numerik
df.describe()

df.duplicated().sum()

df_uber = df[(df['cab_type']=='Uber')]
df_lyft = df[(df['cab_type']=='Lyft')]

df.isna().sum()

#Filling missing data in "price" column by it's median
df['price'].fillna(df['price'].median(), inplace=True)

mean_uber = df_uber['price'].mean()
stdev_uber = df_uber['price'].std()

mean_lyft = df_lyft['price'].mean()
stdev_lyft = df_lyft['price'].std()

data = {"Uber":[df_uber['price'].mean(), df_lyft['price'].mean()],
        "Lyft":[df_lyft['price'].mean(), df_lyft['price'].std(),]
        };

index = ["Mean", "Standard Deviation"];     

dataFrame = pd.DataFrame(data=data, index=index);

dataFrame.plot.bar(rot=0,title="Perbandingan Rata-rata dan Standar Deviasi dari Harga Uber vs Lyft", color=['crimson','steelblue'],figsize=(10,5));
plt.gca().legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.show(block=True);

df.hour.max()

df.hour.min()

df.loc[(0<=df['hour']) & (df['hour']<=3), 'hourGrouped'] = '00-03' 
df.loc[(4<=df['hour']) & (df['hour']<=6), 'hourGrouped'] = '04-06'
df.loc[(7<=df['hour']) & (df['hour']<=9), 'hourGrouped'] = '07-09'
df.loc[(10<=df['hour']) & (df['hour']<=12), 'hourGrouped'] = '10-12'
df.loc[(13<=df['hour']) & (df['hour']<=15), 'hourGrouped'] = '13-15'
df.loc[(16<=df['hour']) & (df['hour']<=18), 'hourGrouped'] = '16-18'
df.loc[(19<=df['hour']) & (df['hour']<=21), 'hourGrouped'] = '19-21'
df.loc[(22<=df['hour']) & (df['hour']<=24), 'hourGrouped'] = '22-24'

dv11 = df[(df['hourGrouped']=='00-03') & (df['cab_type']=='Uber')]
dv12 = df[(df['hourGrouped']=='04-06') & (df['cab_type']=='Uber')]
dv13 = df[(df['hourGrouped']=='07-09') & (df['cab_type']=='Uber')]
dv14 = df[(df['hourGrouped']=='10-12') & (df['cab_type']=='Uber')]
dv15 = df[(df['hourGrouped']=='13-15') & (df['cab_type']=='Uber')]
dv16 = df[(df['hourGrouped']=='16-18') & (df['cab_type']=='Uber')]
dv17 = df[(df['hourGrouped']=='19-21') & (df['cab_type']=='Uber')]
dv18 = df[(df['hourGrouped']=='22-24') & (df['cab_type']=='Uber')]

dv19 = df[(df['hourGrouped']=='00-03') & (df['cab_type']=='Lyft')]
dv110 = df[(df['hourGrouped']=='04-06') & (df['cab_type']=='Lyft')]
dv111 = df[(df['hourGrouped']=='07-09') & (df['cab_type']=='Lyft')]
dv112 = df[(df['hourGrouped']=='10-12') & (df['cab_type']=='Lyft')]
dv113 = df[(df['hourGrouped']=='13-15') & (df['cab_type']=='Lyft')]
dv114 = df[(df['hourGrouped']=='16-18') & (df['cab_type']=='Lyft')]
dv115 = df[(df['hourGrouped']=='19-21') & (df['cab_type']=='Lyft')]
dv116 = df[(df['hourGrouped']=='22-24') & (df['cab_type']=='Lyft')]

data = {"Uber mean":[dv11['price'].mean(), dv12['price'].mean(), dv13['price'].mean(), 
                     dv14['price'].mean(), dv15['price'].mean(), dv16['price'].mean(),
                     dv17['price'].mean(),dv18['price'].mean()],
        "Lyft mean":[dv19['price'].mean(), dv110['price'].mean(), dv111['price'].mean(),
                     dv112['price'].mean(), dv113['price'].mean(), dv114['price'].mean(),
                     dv115['price'].mean(), dv116['price'].mean()]
        };

index = ["00-03", "04-06",'07-09','10-12','13-15','16-18','19-21','22-24'];     

dataFrame = pd.DataFrame(data=data, index=index);

dataFrame.plot.bar(rot=0,title="Rata-rata Harga Uber vs Lyft Per 3 Jam", color=['#f48668','#73a580'],figsize=(15,5));
plt.gca().legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.show(block=True);

df.temperature.min()

df.temperature.max()

df.loc[(15<=df['temperature']) & (df['temperature']<20), 'temperatureGrouped'] = '15-20' 
df.loc[(20<=df['temperature']) & (df['temperature']<25), 'temperatureGrouped'] = '20-25'
df.loc[(25<=df['temperature']) & (df['temperature']<30), 'temperatureGrouped'] = '25-30'
df.loc[(30<=df['temperature']) & (df['temperature']<35), 'temperatureGrouped'] = '30-35'
df.loc[(35<=df['temperature']) & (df['temperature']<40), 'temperatureGrouped'] = '35-40'
df.loc[(40<=df['temperature']) & (df['temperature']<45), 'temperatureGrouped'] = '40-45'
df.loc[(45<=df['temperature']) & (df['temperature']<50), 'temperatureGrouped'] = '45-50'
df.loc[(50<=df['temperature']) & (df['temperature']<55), 'temperatureGrouped'] = '50-55'
df.loc[(55<=df['temperature']) & (df['temperature']<60), 'temperatureGrouped'] = '55-60'

dv21 = df[(df['temperatureGrouped']=='15-20') & (df['cab_type']=='Uber')]
dv22 = df[(df['temperatureGrouped']=='20-25') & (df['cab_type']=='Uber')]
dv23 = df[(df['temperatureGrouped']=='25-30') & (df['cab_type']=='Uber')]
dv24 = df[(df['temperatureGrouped']=='30-35') & (df['cab_type']=='Uber')]
dv25 = df[(df['temperatureGrouped']=='35-40') & (df['cab_type']=='Uber')]
dv26 = df[(df['temperatureGrouped']=='40-45') & (df['cab_type']=='Uber')]
dv27 = df[(df['temperatureGrouped']=='45-50') & (df['cab_type']=='Uber')]
dv28 = df[(df['temperatureGrouped']=='50-55') & (df['cab_type']=='Uber')]
dv29 = df[(df['temperatureGrouped']=='553. Mengumpulkan data untuk pengguna Uber-60') & (df['cab_type']=='Uber')]

dv210 = df[(df['temperatureGrouped']=='15-20') & (df['cab_type']=='Lyft')]
dv211 = df[(df['temperatureGrouped']=='20-25') & (df['cab_type']=='Lyft')]
dv212 = df[(df['temperatureGrouped']=='25-30') & (df['cab_type']=='Lyft')]
dv213 = df[(df['temperatureGrouped']=='30-35') & (df['cab_type']=='Lyft')]
dv214 = df[(df['temperatureGrouped']=='35-40') & (df['cab_type']=='Lyft')]
dv215 = df[(df['temperatureGrouped']=='40-45') & (df['cab_type']=='Lyft')]
dv216 = df[(df['temperatureGrouped']=='45-50') & (df['cab_type']=='Lyft')]
dv217 = df[(df['temperatureGrouped']=='50-55') & (df['cab_type']=='Lyft')]
dv218 = df[(df['temperatureGrouped']=='55-60') & (df['cab_type']=='Lyft')]

data = {"Uber mean":[dv21['price'].mean(), dv22['price'].mean(), dv23['price'].mean(), 
                     dv24['price'].mean(), dv25['price'].mean(), dv26['price'].mean(),
                     dv27['price'].mean(),dv28['price'].mean(),dv29['price'].mean()],
        "Lyft mean":[dv210['price'].mean(), dv211['price'].mean(),dv212['price'].mean(), 
                     dv213['price'].mean(), dv214['price'].mean(),dv215['price'].mean(), 
                     dv216['price'].mean(),dv217['price'].mean(),dv218['price'].mean(),  ]
        };

index = ["15-20", "20-25",'25-30','30-35','35-40','40-45','45-50','50-55','55-60'];     

dataFrame = pd.DataFrame(data=data, index=index);

dataFrame.plot.bar(rot=0,title="Rata-rata Harga Uber vs Lyft Per 5 Derajat Temperatur", color=['darkorange','royalblue'],figsize=(15,5));
plt.gca().legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.show(block=True);

dv41 = df_uber[['source','destination']]
dv42 = df_lyft[['source','destination']]

colors1 = sns.color_palette("Spectral",12)
colors2 = sns.color_palette("rainbow",12)

dv411 = dv41.groupby(['source'],as_index=True).agg({'source':'count'}, index=False)
dv412 = dv41.groupby(['destination'],as_index=True).agg({'source':'count'}, index=False)
dv413 = dv42.groupby(['source'],as_index=True).agg({'source':'count'}, index=False)
dv414 = dv42.groupby(['destination'],as_index=True).agg({'source':'count'}, index=False)

dv411.plot(kind='pie',figsize=(12, 7),autopct='%1.4f%%',startangle=90,shadow=True,subplots=True,colors=colors1,
           textprops={'fontsize': 8},labels=dv411.index,legend=False,wedgeprops={'linewidth': 2.0, 'edgecolor': 'white'})
plt.title('Proporsi Source Uber', loc='center',size ='15')
plt.axis('off')
plt.show()

dv412.plot(kind='pie',figsize=(12, 7),autopct='%1.4f%%',startangle=90,shadow=True,subplots=True,colors=colors1,
           textprops={'fontsize': 8},labels=dv411.index,legend=False,wedgeprops={'linewidth': 2.0, 'edgecolor': 'white'})
plt.title('Proporsi Destination Uber', loc='center',size ='15')
plt.axis('off')
plt.show()

dv413.plot(kind='pie',figsize=(12, 7),autopct='%1.4f%%',startangle=90,shadow=True,subplots=True,colors=colors2,
           textprops={'fontsize': 8},labels=dv411.index,legend=False,wedgeprops={'linewidth': 2.0, 'edgecolor': 'white'})
plt.title('Proporsi Source Lyft', loc='center',size ='15')
plt.axis('off')
plt.show()

dv414.plot(kind='pie',figsize=(12, 7),autopct='%1.4f%%',startangle=90,shadow=True,subplots=True,colors=colors2,
           textprops={'fontsize': 8},labels=dv411.index,legend=False,wedgeprops={'linewidth': 2.0, 'edgecolor': 'white'})
plt.title('Proporsi Destination Lyft', loc='center',size ='15')
plt.axis('off')
plt.show()

grid1 = sns.FacetGrid(df, col='cab_type', height=5, aspect=1.6)
grid1.map(sns.distplot, 'price',bins=50, color = 'c')
plt.subplots_adjust(top=0.85)
grid1.fig.suptitle('Distribusi dari Harga Uber vs Lyft')

# Performing the test on the Uber Price
p_value = normal_ad(df_uber['price'])[1]
print('p-value Uber Price from the test Anderson-Darling test below 0.05 generally means non-normal:', p_value)

# Reporting the normality of the Uber Price
if p_value < 0.05:
    print('Harga Uber tidak normally distributed','\n')
else:
    print('Harga Uber normally distributed','\n')
    
# Performing the test on the Lyft Price    
p_value1 = normal_ad(df_lyft['price'])[1]
print('p-value Lyft Price from the test Anderson-Darling test below 0.05 generally means non-normal:', p_value1)

# Reporting the normality of the Lyft Price
if p_value1 < 0.05:
    print('Harga Lyft tidak normally distributed')
else:
    print('Harga Lyft normally distributed')

df_cor = df.drop(['id','timestamp','day','month','datetime','timezone',
               'source','destination','cab_type','product_id','name',
               'latitude','longitude','apparentTemperature','precipProbability',
               'windGustTime','temperatureHighTime','temperatureLowTime',
               'apparentTemperatureHighTime','apparentTemperatureHighTime',
               'visibility.1','uvIndexTime','short_summary','long_summary',
               'icon','sunriseTime','sunsetTime','moonPhase',
               'temperatureMinTime','temperatureMaxTime','apparentTemperatureMax',
               'apparentTemperatureMaxTime','apparentTemperatureMinTime'], axis=1)
df_cor

fig, ax = plt.subplots(figsize=(30,30))
sns.heatmap(df_cor.corr(), annot=True, fmt='.2%',annot_kws={"size": 10},cmap="inferno")
plt.title("Korelasi Antar Variabel", loc='center',size ='15')
plt.show()

df1 = df[['price','distance','surge_multiplier','cab_type','name']]
df1

cat_col = df1.select_dtypes(include=['object','category']).columns.tolist()
print(cat_col)

for col in cat_col:
    encoder = OneHotEncoder(handle_unknown='ignore')
    enc_df = pd.DataFrame(encoder.fit_transform(df1[[col]]).toarray())
    enc_df.columns = encoder.get_feature_names([col])
    df1 = df1.drop(col, axis=1)
    df1 = pd.concat([df1, enc_df], axis=1)

df1

train, test = train_test_split(df1, test_size=0.25, random_state=2)
train_index = train.index
test_index = test.index

x_train = train.drop(['price'],axis=1)
y_train = train[['price']]

x_test = test.drop(['price'],axis=1)
y_test = test[['price']]

scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)

x_test = scaler.transform(x_test)

lr = LinearRegression()
model = lr.fit(x_train, y_train)  #create model dari LR dari X dan y (berdasarkan data yang dimiliki)

r_sq = model.score(x_train,y_train)
print('Coefficient of determination:',r_sq)
print('Intercept:',model.intercept_)
print('Slope:',model.coef_)

model.coef_[0][12]

model.coef_[0][13]

model.coef_[0][16]

y_pred = model.predict(x_train)
train['Estimated Y'] = np.round(y_pred,2)
train

y_pred = model.predict(x_test)
test['Estimated Y'] = np.round(y_pred,2)
test

model.score(x_test, y_test)

compared = pd.DataFrame({'Keterangan':['Training Data','Testing Data'],'Tingkat Akurasi':[model.score(x_train,y_train), model.score(x_test, y_test)]})
compared