• Python推荐模块
    • 新的数据集
    • 项目实践

    Python推荐模块

    我将本章学到的内容都汇集成了一个Python类，虽然代码有些长，我还是贴在了这里：

    1. import codecs 
    2. from math import sqrt
    4. users = {"Angelica": {"Blues Traveler": 3.5, "Broken Bells": 2.0,
    5.                       "Norah Jones": 4.5, "Phoenix": 5.0,
    6.                       "Slightly Stoopid": 1.5,
    7.                       "The Strokes": 2.5, "Vampire Weekend": 2.0},
    9.          "Bill":{"Blues Traveler": 2.0, "Broken Bells": 3.5,
    10.                  "Deadmau5": 4.0, "Phoenix": 2.0,
    11.                  "Slightly Stoopid": 3.5, "Vampire Weekend": 3.0},
    13.          "Chan": {"Blues Traveler": 5.0, "Broken Bells": 1.0,
    14.                   "Deadmau5": 1.0, "Norah Jones": 3.0, "Phoenix": 5,
    15.                   "Slightly Stoopid": 1.0},
    17.          "Dan": {"Blues Traveler": 3.0, "Broken Bells": 4.0,
    18.                  "Deadmau5": 4.5, "Phoenix": 3.0,
    19.                  "Slightly Stoopid": 4.5, "The Strokes": 4.0,
    20.                  "Vampire Weekend": 2.0},
    22.          "Hailey": {"Broken Bells": 4.0, "Deadmau5": 1.0,
    23.                     "Norah Jones": 4.0, "The Strokes": 4.0,
    24.                     "Vampire Weekend": 1.0},
    26.          "Jordyn":  {"Broken Bells": 4.5, "Deadmau5": 4.0,
    27.                      "Norah Jones": 5.0, "Phoenix": 5.0,
    28.                      "Slightly Stoopid": 4.5, "The Strokes": 4.0,
    29.                      "Vampire Weekend": 4.0},
    31.          "Sam": {"Blues Traveler": 5.0, "Broken Bells": 2.0,
    32.                  "Norah Jones": 3.0, "Phoenix": 5.0,
    33.                  "Slightly Stoopid": 4.0, "The Strokes": 5.0},
    35.          "Veronica": {"Blues Traveler": 3.0, "Norah Jones": 5.0,
    36.                       "Phoenix": 4.0, "Slightly Stoopid": 2.5,
    37.                       "The Strokes": 3.0}
    38.         }
    41. class recommender:
    43.     def __init__(self, data, k=1, metric='pearson', n=5):
    44.         """ 初始化推荐模块
    45.         data   训练数据
    46.         k      K邻近算法中的值
    47.         metric 使用何种距离计算方式
    48.         n      推荐结果的数量
    49.         """
    50.         self.k = k
    51.         self.n = n
    52.         self.username2id = {}
    53.         self.userid2name = {}
    54.         self.productid2name = {}
    55.         # 将距离计算方式保存下来
    56.         self.metric = metric
    57.         if self.metric == 'pearson':
    58.             self.fn = self.pearson
    59.         #
    60.         # 如果data是一个字典类型，则保存下来，否则忽略
    61.         #
    62.         if type(data).__name__ == 'dict':
    63.             self.data = data
    65.     def convertProductID2name(self, id):
    66.         """通过产品ID获取名称"""
    67.         if id in self.productid2name:
    68.             return self.productid2name[id]
    69.         else:
    70.             return id
    72.     def userRatings(self, id, n):
    73.         """返回该用户评分最高的物品"""
    74.         print ("Ratings for " + self.userid2name[id])
    75.         ratings = self.data[id]
    76.         print(len(ratings))
    77.         ratings = list(ratings.items())
    78.         ratings = [(self.convertProductID2name(k), v)
    79.                    for (k, v) in ratings]
    80.         # 排序并返回结果
    81.         ratings.sort(key=lambda artistTuple: artistTuple[1],
    82.                      reverse = True)
    83.         ratings = ratings[:n]
    84.         for rating in ratings:
    85.             print("%s\t%i" % (rating[0], rating[1]))
    87.     def loadBookDB(self, path=''):
    88.         """加载BX数据集，path是数据文件位置"""
    89.         self.data = {}
    90.         i = 0
    91.         #
    92.         # 将书籍评分数据放入self.data
    93.         #
    94.         f = codecs.open(path + "BX-Book-Ratings.csv", 'r', 'utf8')
    95.         for line in f:
    96.             i += 1
    97.             #separate line into fields
    98.             fields = line.split(';')
    99.             user = fields[0].strip('"')
    100.             book = fields[1].strip('"')
    101.             rating = int(fields[2].strip().strip('"'))
    102.             if user in self.data:
    103.                 currentRatings = self.data[user]
    104.             else:
    105.                 currentRatings = {}
    106.             currentRatings[book] = rating
    107.             self.data[user] = currentRatings
    108.         f.close()
    109.         #
    110.         # 将书籍信息存入self.productid2name
    111.         # 包括isbn号、书名、作者等
    112.         #
    113.         f = codecs.open(path + "BX-Books.csv", 'r', 'utf8')
    114.         for line in f:
    115.             i += 1
    116.             #separate line into fields
    117.             fields = line.split(';')
    118.             isbn = fields[0].strip('"')
    119.             title = fields[1].strip('"')
    120.             author = fields[2].strip().strip('"')
    121.             title = title + ' by ' + author
    122.             self.productid2name[isbn] = title
    123.         f.close()
    124.         #
    125.         #  将用户信息存入self.userid2name和self.username2id
    126.         #
    127.         f = codecs.open(path + "BX-Users.csv", 'r', 'utf8')
    128.         for line in f:
    129.             i += 1
    130.             #print(line)
    131.             #separate line into fields
    132.             fields = line.split(';')
    133.             userid = fields[0].strip('"')
    134.             location = fields[1].strip('"')
    135.             if len(fields) > 3:
    136.                 age = fields[2].strip().strip('"')
    137.             else:
    138.                 age = 'NULL'
    139.             if age != 'NULL':
    140.                 value = location + '  (age: ' + age + ')'
    141.             else:
    142.                 value = location
    143.             self.userid2name[userid] = value
    144.             self.username2id[location] = userid
    145.         f.close()
    146.         print(i)
    148.     def pearson(self, rating1, rating2):
    149.         sum_xy = 0
    150.         sum_x = 0
    151.         sum_y = 0
    152.         sum_x2 = 0
    153.         sum_y2 = 0
    154.         n = 0
    155.         for key in rating1:
    156.             if key in rating2:
    157.                 n += 1
    158.                 x = rating1[key]
    159.                 y = rating2[key]
    160.                 sum_xy += x * y
    161.                 sum_x += x
    162.                 sum_y += y
    163.                 sum_x2 += pow(x, 2)
    164.                 sum_y2 += pow(y, 2)
    165.         if n == 0:
    166.             return 0
    167.         # 计算分母
    168.         denominator = (sqrt(sum_x2 - pow(sum_x, 2) / n)
    169.                        * sqrt(sum_y2 - pow(sum_y, 2) / n))
    170.         if denominator == 0:
    171.             return 0
    172.         else:
    173.             return (sum_xy - (sum_x * sum_y) / n) / denominator
    175.     def computeNearestNeighbor(self, username):
    176.         """获取邻近用户"""
    177.         distances = []
    178.         for instance in self.data:
    179.             if instance != username:
    180.                 distance = self.fn(self.data[username],
    181.                                    self.data[instance])
    182.                 distances.append((instance, distance))
    183.         # 按距离排序，距离近的排在前面
    184.         distances.sort(key=lambda artistTuple: artistTuple[1],
    185.                        reverse=True)
    186.         return distances
    188.     def recommend(self, user):
    189.        """返回推荐列表"""
    190.        recommendations = {}
    191.        # 首先，获取邻近用户
    192.        nearest = self.computeNearestNeighbor(user)
    193.        #
    194.        # 获取用户评价过的商品
    195.        #
    196.        userRatings = self.data[user]
    197.        #
    198.        # 计算总距离
    199.        totalDistance = 0.0
    200.        for i in range(self.k):
    201.           totalDistance += nearest[i][1]
    202.        # 汇总K邻近用户的评分
    203.        for i in range(self.k):
    204.           # 计算饼图的每个分片
    205.           weight = nearest[i][1] / totalDistance
    206.           # 获取用户名称
    207.           name = nearest[i][0]
    208.           # 获取用户评分
    209.           neighborRatings = self.data[name]
    210.           # 获得没有评价过的商品
    211.           for artist in neighborRatings:
    212.              if not artist in userRatings:
    213.                 if artist not in recommendations:
    214.                    recommendations[artist] = (neighborRatings[artist]
    215.                                               * weight)
    216.                 else:
    217.                    recommendations[artist] = (recommendations[artist]
    218.                                               + neighborRatings[artist]
    219.                                               * weight)
    220.        # 开始推荐
    221.        recommendations = list(recommendations.items())
    222.        recommendations = [(self.convertProductID2name(k), v)
    223.                           for (k, v) in recommendations]
    224.        # 排序并返回
    225.        recommendations.sort(key=lambda artistTuple: artistTuple[1],
    226.                             reverse = True)
    227.        # 返回前n个结果
    228.        return recommendations[:self.n]

    运行示例

    首先构建一个推荐类，然后获取推荐结果：

    1. >>> r = recommender(users)
    2. >>> r.recommend('Jordyn')
    3. [('Blues Traveler', 5.0)]
    4. >>> r.recommend('Hailey')
    5. [('Phoenix', 5.0), ('Slightly Stoopid', 4.5)]

    新的数据集

    现在让我们使用一个更为真实的数据集。Cai-Nicolas Zeigler从图书漂流站收集了超过100万条评价数据——278,858位用户为271,379本书打了分。

    这份数据（匿名）可以从这个地址获得，有SQL和CSV两种格式。由于特殊符号的关系，这些数据无法直接加载到Python里。

    我做了一些清洗，可以从这里下载。

    CSV文件包含了三张表：

    • 用户表，包括用户ID、位置、年龄等信息。其中用户的姓名已经隐去；
    • 书籍表，包括ISBN号、标题、作者、出版日期、出版社等；
    • 评分表，包括用户ID、书籍ISBN号、以及评分（0-10分）。

    上文Python代码中的loadBookDB方法可以加载这些数据，用法如下：

    1. >>> r.loadBookDB('/Users/raz/Downloads/BX-Dump/')
    2. 1700018
    3. >>> r.recommend('171118')

    注意 由于数据集比较大，大约需要几十秒的时间加载和查询。

    项目实践

    只有运行调试过书中的代码后才能真正掌握这些方法，以下是一些实践建议：

    1. 实现一个计算曼哈顿距离和欧几里得距离的方法；
    2. 本书的网站上有一个包含25部电影评价的数据集，实现一个推荐算法。