# import libraries
import pandas as pd
from math import sqrt
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# store the movie information into a pandas dataframe
movies_df = pd.read_csv('movies1.csv')

# store the ratings information into a pandas dataframe
ratings_df = pd.read_csv('ratings.csv')

movies_df.head()

# dimensions of the dataframes
print(movies_df.shape)
print(ratings_df.shape)

# use regular expressions to find a year stored between parantheses
# we specify the parantheses so we don't conflict with movies that have years in their titles
movies_df['year'] = movies_df.title.str.extract('(\(\d\d\d\d\))', expand=False)

# remove the parentheses
movies_df['year'] = movies_df.year.str.extract('(\d\d\d\d)', expand=False)

# remove the years from the 'title' column
movies_df['title'] = movies_df.title.str.replace('(\(\d\d\d\d\))', '')

# apply the strip finction to get rid of any ending whitespace characters that may have appeared
movies_df['title'] = movies_df['title'].apply(lambda x: x.strip())
movies_df.head()

# every genre is separated by a |. So call the split function on |.
movies_df['genres'] = movies_df.genres.str.split('|')
movies_df.head()

# copy the movie dataframe into a new one
moviesWithGenres_df = movies_df.copy()

# for every row in the dataframe, iterate through the list of genres and place a 1 in the corresponding column
for index, row in movies_df.iterrows():
    for genre in row['genres']:
        moviesWithGenres_df.at[index, genre] = 1
        
# fill in the NaN values with 0 to show that a movie doesn't have that column's genre
moviesWithGenres_df = moviesWithGenres_df.fillna(0)
moviesWithGenres_df.head()

ratings_df.head()

ratings_df = ratings_df.drop('timestamp',1)
ratings_df.head()

userInput = [
    {'title':'Mission: Impossible - Fallout', 'rating':5},
    {'title':'Top Gun', 'rating':4.5},
    {'title':'Jerry Maguire', 'rating':3},
    {'title':'Vanilla Sky', 'rating':2.5},
    {'title':'Minority Report', 'rating':4},
]
inputMovies = pd.DataFrame(userInput)
inputMovies

# filter the movies by title
inputId = movies_df[movies_df['title'].isin(inputMovies['title'].tolist())]

# merge it to get the movieId
inputMovies = pd.merge(inputId, inputMovies)

# drop information we won't use from the input dataframe
inputMovies = inputMovies.drop('genres', 1).drop('year', 1)

# final input dataframe
inputMovies

# filter out the movies from the input
userMovies = moviesWithGenres_df[moviesWithGenres_df['movieId'].isin(inputMovies['movieId'].tolist())]
userMovies

# reset the index
userMovies = userMovies.reset_index(drop=True)

# drop unnecessary columns
userGenreTable = userMovies.drop('movieId', 1).drop('title', 1).drop('genres', 1).drop('year', 1)
userGenreTable

inputMovies['rating']

# dot product to get weights
userProfile = userGenreTable.transpose().dot(inputMovies['rating'])

# the user profile
userProfile

# get the genre of every movie in our original dataframe
genreTable = moviesWithGenres_df.set_index(moviesWithGenres_df['movieId'])

# drop unnecessary columns
genreTable = genreTable.drop('movieId', 1).drop('title', 1).drop('genres', 1).drop('year', 1)
genreTable.head()

genreTable.shape

# multiply the genre by the weights and then take the weighted average
recommendationTable_df = ((genreTable*userProfile).sum(axis=1)) / (userProfile.sum())
recommendationTable_df.head()

movies_df.loc[movies_df['movieId'].isin(recommendationTable_df.head(20).keys())]

userInput = [
    {'title':'Mission: Impossible - Fallout', 'rating':5},
    {'title':'Top Gun', 'rating':4.5},
    {'title':'Jerry Maguire', 'rating':3},
    {'title':'Vanilla Sky', 'rating':2.5},
    {'title':'Minority Report', 'rating':4},
]
inputMovies = pd.DataFrame(userInput)
inputMovies

# filter the movies by title
inputId = movies_df[movies_df['title'].isin(inputMovies['title'].tolist())]

# merge it to get the movieId
inputMovies = pd.merge(inputId, inputMovies)

# drop information we won't use from the input dataframe
inputMovies = inputMovies.drop('genres', 1).drop('year', 1)

# final input dataframe
inputMovies

# filter out users that have watched movies that the input has watched and storing it
userSubset = ratings_df[ratings_df['movieId'].isin(inputMovies['movieId'].tolist())]
userSubset.head()

# groupby creates several sub dataframes where they all have the same value in the column specified as the parameter
userSubsetGroup = userSubset.groupby(['userId'])

userSubsetGroup.get_group(4)

userSubsetGroup = sorted(userSubsetGroup, key=lambda x: len(x[1]), reverse=True)

userSubsetGroup[0:3]

userSubsetGroup = userSubsetGroup[0:100]

pearsonCorrelationDict = {}

#For every user group in our subset
for name, group in userSubsetGroup:
    #Let's start by sorting the input and current user group so the values aren't mixed up later on
    group = group.sort_values(by='movieId')
    inputMovies = inputMovies.sort_values(by='movieId')
    #Get the N for the formula
    nRatings = len(group)
    #Get the review scores for the movies that they both have in common
    temp_df = inputMovies[inputMovies['movieId'].isin(group['movieId'].tolist())]
    #And then store them in a temporary buffer variable in a list format to facilitate future calculations
    tempRatingList = temp_df['rating'].tolist()
    #Let's also put the current user group reviews in a list format
    tempGroupList = group['rating'].tolist()
    #Now let's calculate the pearson correlation between two users, so called, x and y
    Sxx = sum([i**2 for i in tempRatingList]) - pow(sum(tempRatingList),2)/float(nRatings)
    Syy = sum([i**2 for i in tempGroupList]) - pow(sum(tempGroupList),2)/float(nRatings)
    Sxy = sum( i*j for i, j in zip(tempRatingList, tempGroupList)) - sum(tempRatingList)*sum(tempGroupList)/float(nRatings)
    
    #If the denominator is different than zero, then divide, else, 0 correlation.
    if Sxx != 0 and Syy != 0:
        pearsonCorrelationDict[name] = Sxy/sqrt(Sxx*Syy)
    else:
        pearsonCorrelationDict[name] = 0

pearsonCorrelationDict.items()

pearsonDF = pd.DataFrame.from_dict(pearsonCorrelationDict, orient='index')
pearsonDF.columns = ['similarityIndex']
pearsonDF['userId'] = pearsonDF.index
pearsonDF.index = range(len(pearsonDF))
pearsonDF.head()

topUsers = pearsonDF.sort_values(by='similarityIndex', ascending=False)[0:50]
topUsers.head()

# merge two tables
topUsersRating=topUsers.merge(ratings_df, left_on='userId', right_on='userId', how='inner')
topUsersRating.head()

# multiply the similarity by the user's ratings
topUsersRating['weightedRating'] = topUsersRating['similarityIndex']*topUsersRating['rating']
topUsersRating.head()

# apply a sum to the topUsers after grouping it by userId
tempTopUsersRating = topUsersRating.groupby('movieId').sum()[['similarityIndex','weightedRating']]
tempTopUsersRating.columns = ['sum_similarityIndex','sum_weightedRating']
tempTopUsersRating.head()

# create an empty dataframe
recommendation_df = pd.DataFrame()

# take the weighted average
recommendation_df['weighted average recommendation score'] = tempTopUsersRating['sum_weightedRating']/tempTopUsersRating['sum_similarityIndex']
recommendation_df['movieId'] = tempTopUsersRating.index
recommendation_df.head()

recommendation_df = recommendation_df.sort_values(by='weighted average recommendation score', ascending=False)
recommendation_df.head()

movies_df.loc[movies_df['movieId'].isin(recommendation_df.head(20)['movieId'].tolist())]