Spark 下的 CF 推荐 --"Hello world"

import os
# recommendation source code: SLOT{__all__ = ['MatrixFactorizationModel', 'ALS', 'Rating']}
# mllib: https://spark.apache.org/docs/1.6.0/api/python/pyspark.mllib.html
from recommendation import ALS, Rating
# save my logname
name = !whoami

datapath = '/Users/{}/Dropbox/MachLearn/movielens'.format(name[0])

# extract basic data path 
movies = os.path.join(datapath, 'movies.dat')
ratings = os.path.join(datapath, 'ratings.dat')
users = os.path.join(datapath, 'users.dat')

# show first three lines of data 
# uid, mid, rating, ts
for line in data.take(3):
    print(line)

# count ratings
r_c = data.count() # 1000209

# count users, movies

users = data.map(lambda x: x.split("::")[0])
u_c = users.distinct().count() # 6040 

movies = data.map(lambda x: x.split("::")[1])
m_c = movies.distinct().count() # 3706
print("Totally {} ratings, from {} users on {} movies".format(r_c, u_c, m_c))

# start to learn/ split dataset first into train/test


splits = data.randomSplit(weights=[0.8, 0.2], seed=1234)
splits  # PythonRDD[33]  PythonRDD[34]

training = splits[0].repartition(5) #MapPartitionsRDD[39] 
test = splits[1].repartition(5) # MapPartitionsRDD[44]

# prepare use ALS

# parameter list
rank = 12
regParam = .01
maxIter = 20
rating = data.map(lambda l: l.split("::"))\
    .map(lambda x: Rating(int(x[0]), int(x[1]), float(x[2])))
rating.first() # Rating(user=1, product=1193, rating=5.0)

# fit model
als = ALS.train(rating, rank, maxIter, regParam)
# 

als.userFeatures()
# PythonRDD[459] at RDD at PythonRDD.scala:43
als.userFeatures().count()# 6040
als.productFeatures().count() # 3706

# test
test_usersProducts = test.map(lambda x: ( int(x.split("::")[0]), int(x.split("::")[1])))
test_usersProducts.first()

# predictions 
predictions = als.predictAll(test_usersProducts).map(lambda x: ((x[0], x[1]), x[2] ))
predictions.first()  # ((4904, 3704), 4.5491977106240515)
# compare predictions & raw scores
raw_ratings = test.map(lambda x : x.split("::")).map(lambda x: ((int(x[0]), int(x[1])), float(x[2]) )  )  
result = raw_ratings.join(predictions)

result.first()  # ((445, 1286), (4.0, 3.944807967269904))
# calculate RMSE  root of Mean Square Error

mse = result.map(lambda v: (v[1][0] - v[1][1]) ** 2).mean()
import numpy as np
rmse = np.sqrt(mse)
print("rmse = {:.3f}".format(rmse))

# recommendation 

# make a new list which consist of "fresh" movies with high ratings.

# find 5 users randomly

users.distinct().map(lambda x: int(x)).sample(False, 0.1, seed=123).take(5) 

#  [4891, 834, 5830, 4732, 1294]

# for example 1294, recommendation list length: K = 10
K = 10
uid = 1294
example = als.recommendProducts(uid, K)
"""
[Rating(user=1294, product=649, rating=6.0175035084789625),
 Rating(user=1294, product=2157, rating=5.98393452320548),
 Rating(user=1294, product=2931, rating=5.7852798188197285),
 Rating(user=1294, product=1930, rating=5.769173682117222),
 Rating(user=1294, product=557, rating=5.635388794170502),
 Rating(user=1294, product=2938, rating=5.616837050299939),
 Rating(user=1294, product=771, rating=5.56238448199258),
 Rating(user=1294, product=3637, rating=5.5359485902940415),
 Rating(user=1294, product=2681, rating=5.498737474804576),
 Rating(user=1294, product=3816, rating=5.497431078795904)]
"""

from scipy impxort linalg, mat, dot


def cosineSimi(a, b):
    c = dot(a, b.T)/linalg.norm(a)/linalg.norm(b)

    return c[0, 0]


a = mat([-0.711,0.730])
b = mat([-1.099,0.124])
# make test
cosineSimi(a, b)

# find simis of Item[2055]
itemid = 2055 
itemfactor = als.productFeatures().lookup(itemid)
itemvecotr = mat(itemfactor)  # yeah, its my type, and on purpose.
cosineSimi(itemvecotr, itemvecotr)

# 查看和2055 最相似的10个物品. 
sims = als.productFeatures().map(lambda x: (x[0], mat(x[1]))).mapValues(lambda x: cosineSimi(x, itemvecotr) )
# order
from operator import itemgetter
sortedSims = sims.top(K, key=itemgetter(1))
# 这里最相似的前K个商品中第一个是自身, 所以 取 前K+1 个 再排除第一个. 
sortedSims = sims.top(K+1, key=itemgetter(1))[1:]

for k, v in sortedSims:
    print("商品编号: {:>10}, \t相似度: {:.3f}".format(k, v))