import numpy as np # library to handle data in a vectorized manner

import pandas as pd # library for data analsysis
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

import json # library to handle JSON files
from bs4 import BeautifulSoup

#!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab
from geopy.geocoders import Nominatim # convert an address into latitude and longitude values

import requests # library to handle requests
from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe

# Matplotlib and associated plotting modules
import matplotlib.cm as cm
import matplotlib.colors as colors

# import k-means from clustering stage
from sklearn.cluster import KMeans

#!conda install -c conda-forge folium=0.5.0 --yes # uncomment this line if you haven't completed the Foursquare API lab
import folium # map rendering library

print('Libraries imported.')

!wget -q -O 'canada_data.json' https://en.wikipedia.org/w/index.php?title=List_of_postal_codes_of_Canada:_M&oldid=947772202
print('Data downloaded!')

req = requests.get("https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M")

soup = BeautifulSoup(req.content,'lxml')

table = soup.find_all('table')[0]

df = pd.read_html(str(table))

neighborhood_row=pd.DataFrame(df[0])
neighborhood_row

print('The dataframe has {} boroughs and {} neighborhoods.'.format(
        len(neighborhood_row['Borough'].unique()),
        neighborhood_row.shape[0]
    )
)

canada_neighborhood = neighborhood_row[neighborhood_row['Borough'] != 'Not assigned'].reset_index(drop=True)
canada_neighborhood

canada_neighborhood[canada_neighborhood['Postal code'] == 'M5A']

#but there are more itens to be replaced let's see how it goes

#Removing the slash from the 'Neighborhood'column to replace by comma

canada_neighborhood['Neighborhood']= canada_neighborhood.Neighborhood.str.replace('Regent Park / Harbourfront','Regent Park , Harbourfront')
canada_neighborhood

canada_neighborhood.replace(canada_neighborhood['Neighborhood']== 'NaN', canada_neighborhood['Borough'])

print('The dataframe has now {} boroughs and {} neighborhoods.'.format(
        len(canada_neighborhood['Borough'].unique()),
        canada_neighborhood.shape[0]
    )
)