# check if you have plotly installed, using !conda plotly
# if not installed you will not see it listed, install plotly according to your venv 
# use conda install plotly
# or use pip install plotly
# finsstr, is a windows command line tool to search for plotly in the list
# !pip list | findstr plotly
!conda list | findstr plotly

# If you find Plotly is not installed with the above checks,
# ensure you have Conda installed and your Conda environment activated.
# Activate your Conda environment using: conda activate <your_env_name>
# Then, install Plotly by running the command: conda install plotly
# Note: You don't need to be in the project folder to install packages with Conda,
# but make sure the correct environment is activated

# after installing plotly confirm it is installed
!conda list | findstr plotly

# load packages
import pandas as pd 
import plotly.express as px

from itables import init_notebook_mode, show #to fully take advantage of the ability to display our data as a table format, let's use the itables library 
%matplotlib inline 

init_notebook_mode(all_interactive=True) #After this, any Pandas DataFrame, or Series, is displayed as interactive table, which lets you explore, filter or sort your data

# the following code cell will give you a nice hover highlighting on your tables when used with itables library (for details on itables see the itables lesson on my Git)

%%html
<style>
  .dataTables_wrapper tbody tr:hover {
    background-color: #6495ED; /* Cornflower Blue */
  }
</style>


<!-- #1E3A8A (Dark Blue) -->
<!-- #0D9488 (Teal) -->
<!-- #065F46 (Dark Green) -->
<!-- #4C1D95 (Dark Purple) -->
<!-- #991B1B (Dark Red) -->
<!-- #374151 (Dark Gray) -->
<!-- #B45309 (Deep Orange) -->
<!-- #164E63 (Dark Cyan) -->
<!-- #4A2C2A (Dark Brown) -->
<!-- #831843 (Dark Magenta) -->
<!-- #1E3A8A (Dark Blue ) -->

<!-- Suggested Light Colors for Light Backgrounds -->
<!-- #AED9E0 (Light Blue) -->
<!-- #A7F3D0 (Light Teal) -->
<!-- #D1FAE5 (Light Green) -->
<!-- #DDD6FE (Light Purple) -->
<!-- #FECACA (Light Red) -->
<!-- #E5E7EB (Light Gray) -->
<!-- #FFEDD5 (Light Orange) -->
<!-- #B2F5EA (Light Cyan) -->
<!-- #FED7AA (Light Brown) -->
<!-- #FBCFE8 (Light Magenta) -->

# we will use  a plotly dataset
# built into the plotly libary  to explore grouping in pandas and then explore plotting with plotly

# load the dataset
gapminder_df = px.data.gapminder()

# confirm is it what we think it is ie., a dataframe

print(type(gapminder_df)) # will output: <class 'pandas.core.frame.DataFrame'>

# this will print a table of the data
gapminder_df

gapminder_df.info()

# From the column info() summary, we can see 
# the dataset has 1704 non-null records, 
# 5 numeric columns, including floats and integer data 

# method to print the number of unique continents
print("Number of unique continents is:", gapminder_df["continent"].nunique())

# Group data by continent, save to a variable 

grouped_continents =  gapminder_df.groupby("continent")

# Calculate the number of non-null life expectancy values for each continent
# This uses the count function .count() 
    # to compute the number of non-null 'lifeExp' values for each unique'continent' group
    # count() after a groupby operation actually counts all non-null entries in each group, not distinct values
    
World_LifeExp = grouped_continents["lifeExp"].count()

print("Number of non-null life expectancy values per continent:\n",World_LifeExp)

# IF you are into the Olympics, 5 continents might be familiar - continent numbers can be defined differently, but not important here 
#  What is important here: what do the count values (624,300,396,360,24) in the life expectancy column actually represent?
    # We have grouped by continent and counted values of life expectancy for each group 
        # These values are the number of non-null data points for each group
        # Tells us Africa has the most data points with 624 records, whereas Oceania (Australia and other islands) has the least
        # Enables us to cross check with total records we expect, and decide whether some data is less robust, such as Oceania

# Lets quickly check the total values in the dataset to make sure we can account for all the data after grouping
    # Chain the sum function, a common type of summary statistic, to count the function
    # sum, aggregates the counts from each continent to provide a total count of non-null 'lifeExp' values across the dataset

sum_World_LifeExp = grouped_continents["lifeExp"].count().sum()

print("\n") # add spaced between printed outputs

print("Number of total non-null values for all continents is:", sum_World_LifeExp)

# The sum of non-null records across all 5 continents checks out, as we know we have 1704 non-null records in this dataset

# Calculate the average life expectancy for each continent
# This uses the mean function, a common type of summary statistic, to compute the average 'lifeExp' values grouped by 'continent'
avg_World_LifeExp = gapminder_df.groupby("continent")["lifeExp"].mean()

avg_World_LifeExp

# Here we grouped by continents and averaged life expectancy within each continent
# Interestingly, we see some differences with lowest average life expectancy in the continent of Africa and highest in Oceania

# Plot the data using pandas' built-in plotting capabilities
    # Boxplots are useful for comparing distributions by visually representing the median, quartiles, and outliers among different groups

# Key parameters used in this boxplot are:
    #   'column': Specifies the column for which the distribution is calculated and displayed, in this case, 'lifeExp' for life expectancy
    #   'by': Determines the grouping of data, similar to a 'groupby' operation, which in this plot is set to 'continent' to compare across continents
gapminder_df.boxplot(column="lifeExp", by="continent")

# Useful visual plot of the distribution of life expectancies across these 5 continents

# Further explore pandas' built-in plotting capabilities
    # Customize the plot further to make it more visually appealing
    # accepts a number of different parameters, including figsize, rot, grid, and even patch_artist that will add color

gapminder_df.boxplot(column="lifeExp", by="continent", grid=False, figsize=(10, 6), rot=45, patch_artist=True) # patch artist fills the boxes

# lets customize a little bit further to add some labels, for this we will need to load matplot lib directly
#  after creating the plot with pandas
import matplotlib.pyplot as plt

# Create the boxplot
gapminder_df.boxplot(column="lifeExp", by="continent", grid=False, figsize=(10, 6), rot=45, patch_artist=True)

# Turn off the default title, and add a new title
plt.suptitle('')  # Removes the default title
plt.title('Variations in Life Expectancies across 5 Continents') 

# Set the ylabel
plt.ylabel('Years')



# Show the plot
plt.show()

# Pretty good plot, right?

# General Insights:
# - The boxplot highlights the variability and distribution of life expectancy within and between continents
# - The presence of outliers indicates significant deviations from typical life expectancy ranges, which vary by continent

# Specific Observations on Life Expectancy by Continent:
# - Africa shows a broad range of life expectancy, with its median much lower than other continents. This indicates generally lower life expectancy
# - In Africa, the lower quartile for life expectancy is around 45 years, indicating that at least 25% of the countries in the dataset have a life expectancy of 45 years or less
# - Outliers in Africa suggest that while low life expectancy is common, there are countries or regions within Africa where life expectancy approaches the highest observed globally. This could indicate specific areas with better health care, living conditions, or genetic factors.

# In-depth Analysis of Outliers:
# - Outliers on the lower end in continents like Europe indicate rare cases of significantly lower life expectancy, which could be due to specific health crises or localized issues
# - Conversely, outliers on the higher end in Africa provide a stark contrast to the continent's overall lower life expectancy metrics(i.e., lower mean, median, range, Quartiles) 
    #- These outliers warrant follow up analysis to understand the factors contributing to them, which may include better medical facilities, economic conditions, or other socio-economic factors

# Next Steps:
# - The following section will focus specifically on these high life expectancy outliers in Africa to identify potential patterns or causes contributing to these exceptional cases

# Some interesting data was seen using boxplot of life expectancy across continents
# Dig deeper by grouping the data by continent
grouped_data= gapminder_df.groupby("continent")



### OPTIONAL Step #################################
## uncommnent to run this portion 
## import json # OPTIONAL step to convert dictionary to json to get an easier to read print output for the grouped data

# #     Access and print each group using the group attribute of the groupby object
# #     the .groups attribute provides a dictionary mapping group labels to the row indices for those groups



##     # OPTIONAL: Use the json module and json.dumps() to format the .groups dictionary of groupby object data as a json with indenation to better read the data
##     # because the groups are pandas index objects we need to convert to python list before converting json
# print(json.dumps(list(grouped_data.groups), indent=4))

### End of OPTIONAL Step #################################



# Use .groups attribute to have groups printed out as a dictionary, i.e., a high-level or structural view of the data 
# Shows the keys as the unique group names from the columns and rows as index positions that belong to each group
print( grouped_data.groups)

# Get the data for a specific group
africa_data= grouped_data.get_group("Africa")

africa_data

# Filter the dataset for rows where the continent is 'Africa' using the brackets method
# It's good practice to wrap each condition in parentheses, even if not strictly required i.e., it can work but is less clear

africa_df = gapminder_df[(gapminder_df["continent"] == "Africa")]

# Sort the filtered data by life expectancy in descending order
# This allows us to see the countries within Africa with the highest life expectancy at the top
africa_df.sort_values(by="lifeExp", ascending=False)

# From the lower life expectancy sort we see that that in 2007 Reunion has a life expectancy of ~76 years,
#  however, Reunion is not really what most people think about when they hear 'Africa'
#  in fact it is an island and French territory off that is off the south-east coast of Africa

# Although we can filter the entire dataset to Africa with brackets,
#  let's use the grouped data groupby object we created earlier
# Let's use the grouped data for Africa we created earlier

# Filter the Africa dataset to include only rows where Life expectancy less than 40
                 #dataset       # Filter criteria 
africa_df_bel40= africa_data[(africa_data["lifeExp"]<40)]

# Sort the filtered data by life expectancy in descending order to see the countries with the highest life expectancy first
africa_df.sort_values(by="lifeExp", ascending=False)

# We can also achieve the same result from the original data, without starting with the grouped data 

# Filter the original dataset to include only rows where the continent is 'Africa' and Life expectancy less than 40

                    #dataset       # Filter criteria 1                  # Filter criteria 2
africa_df_bel40= gapminder_df[(gapminder_df["continent"]=="Africa") & (gapminder_df["lifeExp"]<40)]

# Sort the filtered data by life expectancy in descending order to see the countries with the highest life expectancy first
africa_df_bel40.sort_values(by="lifeExp", ascending=False)

# Multi-sort 
# we add in year in descending order
# Sort the filtered data 
# by life expectancy in ascending,
    # Adding year  in this way is likely not going to change the results 
    # because it is unlikely we will have ties in Life Expectancy which will then trigger use of the sorting by year to break those ties
africa_data.sort_values(by=["lifeExp","year"], ascending=[True,False])

# From the lower life expectancy sort we see that in 1992 Rwanda had a life expectancy of ~24 years
# so question number 1 answered - now we understand that outlier we saw in the boxplot was Rwanda
# can we be sure of this?

# print summary stats for africa alone and the most relevant columns to assist our analysis
africa_data[["lifeExp","year"]].describe()

# From these two tables of descriptive summary statistics on the African continent subset we observe that:
    # years range from 1952-2007
    # Confirmed that there exists a large range of data for life expectancy,
    #  including a data point with a life expectancy of about 24 years!

# Filter the dataset to rows where continent is africa and life expectancy is less than 40 
africa_data.query('lifeExp<40')

# Add the @ character to pass in a variable to the query

# Filter the dataset to rows where continent is africa and life expectancy is 40

lifeExp_Threshold = 40 # store our filter criteria value in a variable

africa_data.query('lifeExp<@lifeExp_Threshold')

# Use the query method to filter the dataset to rows where continent is africa, 
# life expectancy less than 40 and year prior to 2000 


africa_data.query('lifeExp<@lifeExp_Threshold&year<2000')

# Say we don't know the group we are looking for ...
# but we know that the criteria is it should be the continent with average life expectancy less than 50

continents_LEbelow50= gapminder_df.groupby("continent").filter(lambda x: x["lifeExp"].mean()<50)

print(continents_LEbelow50["continent"].unique()) # this will give us the unique list of continents
 
continents_LEbelow50 # its clear that the only continent with life expectancy below 50 is Africa

# Say we don't know the group we are looking for ...
# but we know that the criteria is it should be the continent with average life expectancy less than 50

country_LEbelow50= africa_data.groupby("country").filter(lambda x: x["lifeExp"].max()<50)

print(country_LEbelow50["country"].unique()) # this will give us the unique list of continents
 
country_LEbelow50 # its clear that the only continent with life expectancy below 50 is Africa

# Say we don't know the group we are looking for ...
# but we know that the criteria is it should be the continent with max life expectancy over 70

country_LEover70= africa_data.groupby("country").filter(lambda x: x["lifeExp"].max()>70)

print(country_LEover70["country"].unique()) # this will give us the unique list of continents
 
country_LEover70 # Now we have identified those African countries where life expectancy is above average 

gapminder_df.groupby("continent").agg({"lifeExp": "mean", "pop": "median"})

# Lets look at the time trend for Life Expectancy in Africa 

# Plotting the mean life expectancy over time
# A line plot is ideal for this time series data as it shows trends clearly
africa_df.groupby("year")["lifeExp"].mean().plot(kind='line')

# Improve the line plot

# Group the overall Africa dataset by year and calculate mean life expectancy
africa_life_exp = africa_df.groupby("year")["lifeExp"].mean()

# plot the line with some customization   
africa_life_exp.plot(
    kind='line', legend=True, figsize=(14, 8), color='red', label='Africa', 
    grid=False, xlabel="Year",  title="Life Expectancy in Africa Over Time",ylabel="Life Expectancy", fontsize=14, rot=45)  # added additional plot parameters: labels, change figure size and color etc.

# Say you want to iterate over a group in the groupby and print data for each group

# Sample DataFrame
df = pd.DataFrame({
    'Category': ['A', 'B', 'A', 'B'],
    'Values': [10, 20, 30, 40]
})

# Grouping by 'Category'
grouped_data = df.groupby('Category')

# Iterating over each group
# Pandas will unpack each item grouped_data (the groupby object) into its group key and corresponding dataframe slice
for group_name, group_df in grouped_data:
    print(f"Group Name: {group_name}")
    print(f"Data:\n{group_df}\n")

# In this example:

# group_name will be 'A' in the first iteration and 'B' in the second
# group_df will be the subset of df where Category is 'A' in the first iteration, and where Category is 'B' in the second.=
# This pattern is very powerful for performing operations on each group individually within a loop

# What we did here was iterate over a groupby object:
    # In the loop we represent the current group by its group_name and group_df, which are aliases
    # group_name is used to represent the 'key' or unique name of the current group in the iteration
    # group_df is to represent the dataframe slice that corresponds to the current group_name in the iteration

# Plot trends in life expectency for all continents on the same plot

for g_name, g_data  in grouped_continents:
        print('This is group named', g_name)
        # we chained functions together, rather than making a variables 
        g_data.groupby("year")["lifeExp"].mean().plot(kind='line',title= 'Trend in life expectancy on 5 continents',xlabel='Year', ylabel='Life Expectancy' , label=g_name,legend=True , linestyle='-', marker='o', rot=45, figsize=( 14,8))

# OK. We have seen that africa has the most diverse life expectancy 
# Let's dig a bit deeper 

# Count the total number of records in the Africa data subset, regardless of whether there are any null values
print("The number of total records in the Africa data subset is:", len(africa_data))

# Count the number of non-null records in the 'lifeExp' column of the Africa data subset, does not count rows that are NaN (Not a Number)
print("Count of non-null records in the 'lifeExp' column of the Africa data subset:", africa_data["lifeExp"].count())

# Count how many unique countries are in the Africa data subset
print("Number of unique countries in the Africa data subset:", africa_data["country"].nunique())

# Count non-null records for Rwanda across all columns. This shows the count per column
print("Count of non-null records for the African country of Rwanda (per column):", africa_data[africa_data["country"] == 'Rwanda'].count())

# Let's compare life expectancy trends between all of Africa versus Rwanda

# Filter the African dataset for Rwanda and assign the grouped data to rwanda_df
rwanda_df = africa_df[africa_df["country"]=="Rwanda"] # This not only makes the code cleaner but also allows for easier debugging and additional manipulation if needed later.

# Group Rwanda data by year and calculate mean life expectancy, then plot
rwanda_life_exp = rwanda_df.groupby("year")["lifeExp"].mean()
rwanda_life_exp.plot(
    kind='line', legend=True, figsize=(14, 8), color='green', label='Rwanda', 
    grid=False, linestyle='dashdot', title="Life Expectancy in Rwanda vs Africa Over Time", marker='o')

# Group the overall Africa dataset by year and calculate mean life expectancy, then plot
africa_life_exp = africa_df.groupby("year")["lifeExp"].mean()
africa_life_exp.plot(
    kind='line', legend=True, figsize=(14, 8), color='red', label='Africa', 
    grid=False, xlabel="Year", ylabel="Life Expectancy", fontsize=14, rot=45, marker='o')

# We still have not understood the African countries with high life Expectancy
# Let's try to plot all the data and see if that can get somewhere

african_countries = africa_data.groupby("country")
# Plot trends in life expectency for all continents on the same plot

import matplotlib.pyplot as plt

# Initialize the plot with the first group to get the 'ax' object
fig, ax = plt.subplots(figsize=(14, 8))

for country_name, g_data in african_countries:
    # Plot on the same 'ax' object
    g_data.groupby("year")["lifeExp"].mean().plot(ax=ax, kind='line', title='Trend in life expectancy in African countries', xlabel='Year', ylabel='Life Expectancy', label=country_name, legend=True, linestyle='-', marker='o')

# Rotate the x-axis tick labels
plt.xticks(rotation=45)

# Move the legend outside the plot area
ax.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')

plt.show()

# This is not too helpful

# Compute the maximum life expectancy for each african country
# Here, 'african_countries' is a GroupBy object created by grouping 'africa_data' by the "country" column
    # The operation '["lifeExp"].max()' computes the maximum life expectancy for each country
    # This aggregation condenses the multiple records per country into a single record per country, representing the maximum life expectancy found in the dataset for that country
max_lifeExp_per_country = african_countries["lifeExp"].max()

# Find the top 10 countries with the largest maximum life expectancy
    # The 'nlargest(10)' method selects the top 10 values from 'max_lifeExp_per_country', which contains the maximum life expectancy for each country
    # This results in a Series of the top 10 countries with the highest maximum life expectancy values, sorted in descending order
top10_lifeExp_afr_countries = max_lifeExp_per_country.nlargest(10)

print("Top 10 african countries for max life expectancy:\n", top10_lifeExp_afr_countries)


# Find the bottom 10 countries with the lowest maximum life expectancy
    # The 'nsmallest(10)' method selects the bottom 10 values from 'max_lifeExp_per_country', which contains the maximum life expectancy for each country
    # This results in a Series of the bottom 10 countries for maximum life expectancy values, sorted in ascending order
bot10_lifeExp_afr_countries = max_lifeExp_per_country.nsmallest(10)

print("Bottom 10 african countries for max life expectancy:\n", bot10_lifeExp_afr_countries)

# Before diving into plotting the life expectancy trends, 
# it's important to extract the names of the countries we're interested in analyzing
# For the bottom 10 countries with the lowest life expectancy, 
# we can access the index of the `bot10_lifeExp_afr_countries`, contains these country names. 
# This index is then converted to a list, 
# Will be used to filter our original dataset for these specific countries and then plot them

# Extracting the index (country names) of the bottom 10 records
print(bot10_lifeExp_afr_countries.index)

# Converting the index to a list for easier use in filtering
list_bottom10 = bot10_lifeExp_afr_countries.index.to_list()

# Displaying the list of bottom 10 countries
list_bottom10

# This method of extracting and utilizing the list of countries will apply to the top 10 countries with the highest life expectancy, 
# allowing us to efficiently compare the life expectancy trends between these two groups

import matplotlib.pyplot as plt


# Extract the country names to a list
top_countries= top10_lifeExp_afr_countries.index.to_list()

bottom_countries= bot10_lifeExp_afr_countries.index.to_list()


# Filter the original data using the country lists
Filtered_top_bot_data = africa_data[africa_data["country"].isin(top_countries+bottom_countries)]


# plot the data

plt.figure(figsize=(10,6))

# Plot data for top countries
for country in top_countries:
    country_data = Filtered_top_bot_data[Filtered_top_bot_data["country"]== country]
    plt.plot(country_data['year'], country_data['lifeExp'], label=country, linestyle='-', marker='o')

# Plot data for bottom countries with a different style
for country in bottom_countries:
    country_data = Filtered_top_bot_data[Filtered_top_bot_data["country"]== country]
    plt.plot(country_data['year'], country_data['lifeExp'], label=country, linestyle='--', marker='x')

plt.title('Life Expectancy Over time for Top and Bottom African Countries')

plt.xlabel('Year')

plt.ylabel('Life Expectancy')

plt.legend(bbox_to_anchor=(1.05,1), loc='upper left', title= "Countries")
plt.tight_layout()

from geopy.geocoders import Nominatim
import time

# Increase the timeout to 10 seconds
geolocator = Nominatim(user_agent="my_africa_countries_application", timeout=10)

# Function to get latitude and longitude
def get_lat_lon(country_name):
    try:
        location = geolocator.geocode(country_name)
        return (location.latitude, location.longitude)
    except:
        return (None, None)

# Retrieve coordinates for top and bottom countries
coordinates = {country: get_lat_lon(country) for country in top_countries + bottom_countries}

# To avoid hitting the request limit, add a small sleep time in the loop if necessary
time.sleep(1)

print(coordinates)

import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib.pyplot as plt

# Assuming you have a dictionary `coordinates` with country names as keys and (latitude, longitude) tuples as values

fig = plt.figure(figsize=(10, 5))

# Create a map using the PlateCarree projection
ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
ax.add_feature(cfeature.COASTLINE)
ax.add_feature(cfeature.BORDERS, linestyle=':')

# Set the extent to cover the African continent
ax.set_extent([-25.0, 70.0, -35.0, 47.0], crs=ccrs.PlateCarree())


# Plot each country's coordinates
# Assuming `top_countries` and `bottom_countries` are lists of country names
for country, (lat, lon) in coordinates.items():
    if lat and lon:  # Check if lat and lon are not None
        color = 'green' if country in top_countries else 'red'
        plt.plot(lon, lat, marker='o', color=color, markersize=5, transform=ccrs.Geodetic())
        plt.text(lon, lat, country, transform=ccrs.Geodetic())

plt.title('Top and Bottom African Countries by Life Expectancy')
plt.show()

import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib.cm as cm


# Convert Filtered_top_bot_data to a dictionary mapping countries to life expectancy
life_expectancy = Filtered_top_bot_data.set_index('country')['lifeExp'].to_dict()
print(life_expectancy)
print(list(life_expectancy.values()))
# Normalize life expectancy values to [0, 1] for colormap
life_expectancy_values = list(life_expectancy.values())
norm = plt.Normalize(min(life_expectancy_values), max(life_expectancy_values))
cmap = cm.ScalarMappable(norm=norm, cmap='coolwarm')  # Choose a colormap
cmap.set_array(life_expectancy_values)  # Set the array to the actual life expectancy values

fig = plt.figure(figsize=(10, 5))
ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
ax.add_feature(cfeature.COASTLINE)
ax.add_feature(cfeature.BORDERS, linestyle=':')

# Set the extent to cover the African continent
ax.set_extent([-25.0, 70.0, -35.0, 47.0], crs=ccrs.PlateCarree())

for country, (lat, lon) in coordinates.items():
    if lat and lon and country in life_expectancy:  # Check if lat, lon, and life expectancy are not None
        color = cmap.to_rgba(life_expectancy[country])
        ax.plot(lon, lat, marker='o', color=color, markersize=5, transform=ccrs.Geodetic())
        ax.text(lon, lat, country, transform=ccrs.Geodetic())

# Specify the ax argument to plt.colorbar()
plt.colorbar(cmap, label='Life Expectancy', ax=ax, orientation='vertical')
plt.show()