推荐算法之协同过滤实例

其他 2018-07-06 10:40:39 阅读次数: 0

接着上次的数据进行协同过滤算法应用

应用的知识

python的surprise
k折交叉验证
R数据构建
KNNBasic
KNNWithMeans
KNNWithZScore

数据处理与算法

# 协同过滤算法数据构建

user_artist_sum_weight <- sqldf::sqldf('select userID,artistID, sum(weight) as play_num from user_artists group by userID,artistID')
user_artist_sum_weight <- as.data.table(user_artist_sum_weight)

names(artists)[1] <- 'artistID'

artists_part <- artists[,c(1:2)]

data0 <- artists_part[user_artist_sum_weight,on='artistID']

user_taggedartists0 <- user_taggedartists[,1:3]

data1 <- user_taggedartists0[data0,on=c('userID','artistID')]
data2 <- tags[data1,on='tagID']

no_tags <- data2[is.na(data2$tagID),] 
no_tags <- no_tags[,user_artist_activity:=0]
head(no_tags)
have_tags <- data2[!is.na(data2$tagID),]
head(have_tags)

have_tags0 <- sqldf::sqldf('select userID,artistID,count(1) as user_artist_activity from have_tags group by userID,artistID')
have_tags1 <- sqldf::sqldf('select distinct userID,artistID,play_num from have_tags')

have_tags0 <- as.data.table(have_tags0)
have_tags1 <- as.data.table(have_tags1)

merge_have_tags <- have_tags0[have_tags1,on=c('userID','artistID')]
no_tags <- no_tags[,c("userID" ,"artistID" , "user_artist_activity" ,"play_num")]
merge_data <- rbind(merge_have_tags,no_tags)


# 构建等深分箱函数（Equal frequency intervals）

# play_num_EFI(1:4:4:1)
play_num_EFI <- EFI(merge_data$play_num,10,min(merge_data$play_num))
play_num_EFI$value
merge_data$play_num <- ifelse(merge_data$play_num<=37,'1',
                              ifelse(merge_data$play_num<=260,'2',
                                     ifelse(merge_data$play_num<=1387,'3','4')))

merge_data1 <- merge_data[user_artist_activity>0,]
user_artist_activity_EFI <- EFI(merge_data1$user_artist_activity,10,min(merge_data$user_artist_activity))
user_artist_activity_EFI$value
merge_data$user_artist_activity <- ifelse(merge_data$user_artist_activity==0,'0',
                                          ifelse(merge_data$user_artist_activity<=5,'1','2'))

summary(merge_data)
merge_data$user_artist_activity <- as.numeric(merge_data$user_artist_activity)
merge_data$play_num <- as.numeric(merge_data$play_num)
head(merge_data)
model_data <- sqldf::sqldf('select userID,artistID,sum(user_artist_activity)+sum(play_num) as score 
                           from merge_data group by userID,artistID')

python_data <- data.frame(user=model_data$userID,item=model_data$artistID,rating=model_data$score,timestamp=1)
# write.csv(python_data,'python_data.csv',row.names = FALSE)

python 之算法应用

# 可以使用上面提到的各种推荐系统算法
from surprise import Dataset
from surprise import  print_perf
from surprise import KNNBasic,KNNWithMeans,KNNWithZScore, KNNBaseline
import os
from surprise import Reader, Dataset
from surprise.model_selection import cross_validate
from pandas import DataFrame 
import numpy as np
import pandas as pd
######################################## KNNBasic
#algo = KNNBasic(k=1,sim_options = {'name': 'pearson','user_based': True})# 皮尔逊基于用户

## 指定文件路径
file_path = os.path.expanduser('./python_data.txt')
## 指定文件格式\n",
reader = Reader(line_format='user item rating timestamp', sep=',')
## 从文件读取数据
data = Dataset.load_from_file(file_path, reader=reader)

algo1 = KNNBasic(k=1,sim_options = {'name': 'pearson','user_based': True})
algo2 = KNNBasic(k=3,sim_options = {'name': 'pearson','user_based': True})
algo3 = KNNBasic(k=5,sim_options = {'name': 'pearson','user_based': True})
algo4 = KNNBasic(k=7,sim_options = {'name': 'pearson','user_based': True})
algo5 = KNNBasic(k=9,sim_options = {'name': 'pearson','user_based': True})
algo6 = KNNBasic(k=11,sim_options = {'name': 'pearson','user_based': True})
algo7 = KNNBasic(k=13,sim_options = {'name': 'pearson','user_based': True})
algo8 = KNNBasic(k=15,sim_options = {'name': 'pearson','user_based': True})
algo9 = KNNBasic(k=17,sim_options = {'name': 'pearson','user_based': True})
algo10 = KNNBasic(k=19,sim_options = {'name': 'pearson','user_based': True})
algo11 = KNNBasic(k=21,sim_options = {'name': 'pearson','user_based': True})
algo12 = KNNBasic(k=23,sim_options = {'name': 'pearson','user_based': True})
algo13 = KNNBasic(k=25,sim_options = {'name': 'pearson','user_based': True})
algo14 = KNNBasic(k=27,sim_options = {'name': 'pearson','user_based': True})
algo15 = KNNBasic(k=29,sim_options = {'name': 'pearson','user_based': True})

# 在数据集上测试一下效果
perf_1 = cross_validate(algo1,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_2 = cross_validate(algo2,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_3 = cross_validate(algo3,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_4 = cross_validate(algo4,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_5 = cross_validate(algo5,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_6 = cross_validate(algo6,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_7 = cross_validate(algo7,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_8 = cross_validate(algo8,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_9 = cross_validate(algo9,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_10 = cross_validate(algo10,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_11 = cross_validate(algo11,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_12 = cross_validate(algo12,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_13 = cross_validate(algo13,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_14 = cross_validate(algo14,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_15 = cross_validate(algo15,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)

perf_result_1=[]
for i in range(1,16):
  perf_result_1.append('perf_'+ str(i)) 
MAE=[]
for perf in perf_result_1:
  MAE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,1]),4))

RMSE=[]
for perf in perf_result_1:
  RMSE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,2]),4))


FIT_TIME=[]
for perf in perf_result_1:
  FIT_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,0]),4))


TEST_TIME=[]
for perf in perf_result_1:
  TEST_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,3]),4))

MAE = DataFrame(MAE,columns=['MAE'])
RMSE = DataFrame(RMSE,columns=['RMSE'])
FIT_TIME = DataFrame(FIT_TIME,columns=['FIT_TIME'])
TEST_TIME = DataFrame(TEST_TIME,columns=['TEST_TIME'])
k = DataFrame([1,3,5,7,9,11,13,15,17,19,21,23,25,27,29],columns=['k'])

KNNBasic_result = pd.concat([k,MAE,RMSE,FIT_TIME,TEST_TIME],axis=1)
KNNBasic_result.to_csv('./result_data/KNNBasic_result1.csv',header=True,encoding='utf-8')


############################################# KNNWithMeans

#algo = KNNWithMeans(k=1,sim_options = {'name': 'pearson','user_based': True})# 皮尔逊基于用户

## 指定文件路径
file_path = os.path.expanduser('./python_data.txt')
## 指定文件格式\n",
reader = Reader(line_format='user item rating timestamp', sep=',')
## 从文件读取数据
data = Dataset.load_from_file(file_path, reader=reader)

algo1 = KNNWithMeans(k=1,sim_options = {'name': 'pearson','user_based': True})
algo2 = KNNWithMeans(k=3,sim_options = {'name': 'pearson','user_based': True})
algo3 = KNNWithMeans(k=5,sim_options = {'name': 'pearson','user_based': True})
algo4 = KNNWithMeans(k=7,sim_options = {'name': 'pearson','user_based': True})
algo5 = KNNWithMeans(k=9,sim_options = {'name': 'pearson','user_based': True})
algo6 = KNNWithMeans(k=11,sim_options = {'name': 'pearson','user_based': True})
algo7 = KNNWithMeans(k=13,sim_options = {'name': 'pearson','user_based': True})
algo8 = KNNWithMeans(k=15,sim_options = {'name': 'pearson','user_based': True})
algo9 = KNNWithMeans(k=17,sim_options = {'name': 'pearson','user_based': True})
algo10 = KNNWithMeans(k=19,sim_options = {'name': 'pearson','user_based': True})
algo11 = KNNWithMeans(k=21,sim_options = {'name': 'pearson','user_based': True})
algo12 = KNNWithMeans(k=23,sim_options = {'name': 'pearson','user_based': True})
algo13 = KNNWithMeans(k=25,sim_options = {'name': 'pearson','user_based': True})
algo14 = KNNWithMeans(k=27,sim_options = {'name': 'pearson','user_based': True})
algo15 = KNNWithMeans(k=29,sim_options = {'name': 'pearson','user_based': True})

# 在数据集上测试一下效果
perf_01 = cross_validate(algo1,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_02 = cross_validate(algo2,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_03 = cross_validate(algo3,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_04 = cross_validate(algo4,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_05 = cross_validate(algo5,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_06 = cross_validate(algo6,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_07 = cross_validate(algo7,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_08 = cross_validate(algo8,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_09 = cross_validate(algo9,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_010 = cross_validate(algo10,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_011 = cross_validate(algo11,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_012 = cross_validate(algo12,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_013 = cross_validate(algo13,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_014 = cross_validate(algo14,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_015 = cross_validate(algo15,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)

perf_result_1=[]
for i in range(1,16):
  perf_result_1.append('perf_0'+ str(i)) 
MAE=[]
for perf in perf_result_1:
  MAE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,1]),4))

RMSE=[]
for perf in perf_result_1:
  RMSE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,2]),4))


FIT_TIME=[]
for perf in perf_result_1:
  FIT_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,0]),4))


TEST_TIME=[]
for perf in perf_result_1:
  TEST_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,3]),4))

MAE = DataFrame(MAE,columns=['MAE'])
RMSE = DataFrame(RMSE,columns=['RMSE'])
FIT_TIME = DataFrame(FIT_TIME,columns=['FIT_TIME'])
TEST_TIME = DataFrame(TEST_TIME,columns=['TEST_TIME'])
k = DataFrame([1,3,5,7,9,11,13,15,17,19,21,23,25,27,29],columns=['k'])

KNNWithMeans_result = pd.concat([k,MAE,RMSE,FIT_TIME,TEST_TIME],axis=1)
KNNWithMeans_result.to_csv('./result_data/KNNWithMeans_result1.csv',header=True,encoding='utf-8')


############################################## KNNWithZScore

#algo = KNNWithZScore(k=1,sim_options = {'name': 'pearson','user_based': True})# 皮尔逊基于用户

## 指定文件路径
file_path = os.path.expanduser('./python_data.txt')
## 指定文件格式\n",
reader = Reader(line_format='user item rating timestamp', sep=',')
## 从文件读取数据
data = Dataset.load_from_file(file_path, reader=reader)

algo1 = KNNWithZScore(k=1,sim_options = {'name': 'pearson','user_based': True})
algo2 = KNNWithZScore(k=3,sim_options = {'name': 'pearson','user_based': True})
algo3 = KNNWithZScore(k=5,sim_options = {'name': 'pearson','user_based': True})
algo4 = KNNWithZScore(k=7,sim_options = {'name': 'pearson','user_based': True})
algo5 = KNNWithZScore(k=9,sim_options = {'name': 'pearson','user_based': True})
algo6 = KNNWithZScore(k=11,sim_options = {'name': 'pearson','user_based': True})
algo7 = KNNWithZScore(k=13,sim_options = {'name': 'pearson','user_based': True})
algo8 = KNNWithZScore(k=15,sim_options = {'name': 'pearson','user_based': True})
algo9 = KNNWithZScore(k=17,sim_options = {'name': 'pearson','user_based': True})
algo10 = KNNWithZScore(k=19,sim_options = {'name': 'pearson','user_based': True})
algo11 = KNNWithZScore(k=21,sim_options = {'name': 'pearson','user_based': True})
algo12 = KNNWithZScore(k=23,sim_options = {'name': 'pearson','user_based': True})
algo13 = KNNWithZScore(k=25,sim_options = {'name': 'pearson','user_based': True})
algo14 = KNNWithZScore(k=27,sim_options = {'name': 'pearson','user_based': True})
algo15 = KNNWithZScore(k=29,sim_options = {'name': 'pearson','user_based': True})

# 在数据集上测试一下效果
perf_001 = cross_validate(algo1,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_002 = cross_validate(algo2,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_003 = cross_validate(algo3,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_004 = cross_validate(algo4,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_005 = cross_validate(algo5,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_006 = cross_validate(algo6,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_007 = cross_validate(algo7,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_008 = cross_validate(algo8,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_009 = cross_validate(algo9,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0010 = cross_validate(algo10,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0011 = cross_validate(algo11,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0012 = cross_validate(algo12,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0013 = cross_validate(algo13,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0014 = cross_validate(algo14,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)
perf_0015 = cross_validate(algo15,data, measures=['RMSE', 'MAE'],cv=5,verbose=True)

perf_result_1=[]
for i in range(1,16):
  perf_result_1.append('perf_00'+ str(i)) 
MAE=[]
for perf in perf_result_1:
  MAE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,1]),4))

RMSE=[]
for perf in perf_result_1:
  RMSE.append(round(np.mean(DataFrame(eval(perf)).iloc[:,2]),4))


FIT_TIME=[]
for perf in perf_result_1:
  FIT_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,0]),4))


TEST_TIME=[]
for perf in perf_result_1:
  TEST_TIME.append(round(np.mean(DataFrame(eval(perf)).iloc[:,3]),4))

MAE = DataFrame(MAE,columns=['MAE'])
RMSE = DataFrame(RMSE,columns=['RMSE'])
FIT_TIME = DataFrame(FIT_TIME,columns=['FIT_TIME'])
TEST_TIME = DataFrame(TEST_TIME,columns=['TEST_TIME'])
k = DataFrame([1,3,5,7,9,11,13,15,17,19,21,23,25,27,29],columns=['k'])

KNNWithZScore_result = pd.concat([k,MAE,RMSE,FIT_TIME,TEST_TIME],axis=1)
KNNWithZScore_result.to_csv('./result_data/KNNWithZScore_result1.csv',header=True,encoding='utf-8')

R之数据可视化

##############协同过滤结果可视化
library(ggplot2)
library(ggthemr)
library(easyGgplot2)

KNNBasic_result <- read.csv('KNNBasic_result1.csv',encoding = 'utf-8')
KNNWithMeans_result <- read.csv('KNNWithMeans_result1.csv',encoding = 'utf-8')
KNNWithZScore_result <- read.csv('KNNWithZScore_result1.csv',encoding = 'utf-8')


KNNBasic_result <- as.data.table(KNNBasic_result)
KNNWithMeans_result <- as.data.table(KNNWithMeans_result)
KNNWithZScore_result <- as.data.table(KNNWithZScore_result)


KNNBasic_result <- KNNBasic_result[,KNN_class:='KNNBasic']
KNNWithMeans_result <- KNNWithMeans_result[,KNN_class:='KNNWithMeans']
KNNWithZScore_result <- KNNWithZScore_result[,KNN_class:='KNNWithZScore']


merge_KNN_result <- rbind(KNNBasic_result,KNNWithMeans_result,KNNWithZScore_result)
merge_KNN_result <- merge_KNN_result[,-1]

# plot result

colour <- c('#34495e','#3498db','#2ecc71','#f1c40f','#e74c3c','#9b59b6','#1abc9c')
mycol <- define_palette(swatch = colour,gradient = c(lower=colour[1L],upper=colour[2L]))
ggthemr(mycol)



p001 <- ggplot(data= merge_KNN_result, aes(x=k, y= MAE,group=KNN_class, shape=KNN_class, color=KNN_class)) +
  geom_line()+
  geom_point()+
  xlab('最近邻居数（k）')+
  theme(legend.position="top",axis.text.x = element_text(angle = 50, hjust = 0.5, vjust = 0.5),
        text = element_text(color = "black", size = 12))

p002 <- ggplot(data= merge_KNN_result, aes(x=k, y= RMSE,group=KNN_class, shape=KNN_class, color=KNN_class)) +
  geom_line()+
  geom_point()+
  xlab('最近邻居数（k）')+
  theme(legend.position="top",axis.text.x = element_text(angle = 50, hjust = 0.5, vjust = 0.5),
        text = element_text(color = "black", size = 12))


p0001 <- ggplot(data= merge_KNN_result, aes(x=k, y= FIT_TIME,group=KNN_class, shape=KNN_class, color=KNN_class)) +
  geom_line()+
  geom_point()+
  xlab('最近邻居数（k）')+
  theme(legend.position="top",axis.text.x = element_text(angle = 50, hjust = 0.5, vjust = 0.5),
        text = element_text(color = "black", size = 12))


p0002 <- ggplot(data= merge_KNN_result, aes(x=k, y= TEST_TIME,group=KNN_class, shape=KNN_class, color=KNN_class)) +
  geom_line()+
  geom_point()+
  xlab('最近邻居数（k）')+
  theme(legend.position="top",axis.text.x = element_text(angle = 50, hjust = 0.5, vjust = 0.5),
        text = element_text(color = "black", size = 12))


x11()
ggplot2.multiplot(p001,p002,cols = 2)
x11()
ggplot2.multiplot(p0001,p0002,cols = 2)

算法结果比较

这里写图片描述

这里写图片描述

猜你喜欢

转载自blog.csdn.net/u012111465/article/details/80306132
今日推荐
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
【转】spring中对控制反转和依赖注入的理解
tms webcore 安装和使用
java程序员进阶相关书籍
SpringMVC接受请求参数、
如何保存训练好的机器学习模型
MyEclipse、Eclipse设置项目JDK的三个地方
商超行业微信小程序开发定制一般多少钱 （行业技术人员解读）
Markdown编辑器语言——30分钟入门到到精通
Linux系统下MongoDB的简单安装与基本操作
Power Strings
每日归档
更多
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022