-----[p.293]--Measuring the performance of classifiers-------------------------------
> install.packages("caret") 時間がかかります
Installing package into ‘C:/Users/ekaminuma/Documents/R/win-library/3.1’
(as ‘lib’ is unspecified)
also installing the dependencies ‘colorspace’, ‘RColorBrewer’, ‘dichromat’, ‘munsell’, ‘labeling’, ‘SparseM’, ‘MatrixModels’, ‘stringi’, ‘magrittr’, ‘minqa’, ‘nloptr’, ‘RcppEigen’, ‘profileModel’, ‘digest’, ‘gtable’, ‘scales’, ‘proto’, ‘pbkrtest’, ‘quantreg’, ‘stringr’, ‘Rcpp’, ‘iterators’, ‘lme4’, ‘brglm’, ‘ggplot2’, ‘car’, ‘reshape2’, ‘foreach’, ‘plyr’, ‘BradleyTerry2’
> library(caret)
> library(e1071) SVM用
> data(iris)
> iris2<-iris[1:100,] 本と違うので注意
> indices<-sample(1:nrow(iris2), floor(2*nrow(iris2)/3))
> train <- iris2[indices,]
> test <- iris2[-indices,]
> testClass <- test[,"Species"]
> mySVM <- svm(Species ~., data=train, cost=100, gamma=1)
> myCM<-confusionMatrix(myPred,testClass)
> myCM
Confusion Matrix and Statistics
Reference
Prediction setosa versicolor virginica
setosa 15 0 0
versicolor 0 19 0
virginica 0 0 0
Overall Statistics
Accuracy : 1
95% CI : (0.8972, 1)
No Information Rate : 0.5588
P-Value [Acc > NIR] : 2.555e-09
Kappa : 1
Mcnemar's Test P-Value : NA
Statistics by Class:
Class: setosa Class: versicolor
Sensitivity 1.0000 1.0000
Specificity 1.0000 1.0000
Pos Pred Value 1.0000 1.0000
Neg Pred Value 1.0000 1.0000
Prevalence 0.4412 0.5588
Detection Rate 0.4412 0.5588
Detection Prevalence 0.4412 0.5588
Balanced Accuracy 1.0000 1.0000
Class: virginica
Sensitivity NA
Specificity 1
Pos Pred Value NA
Neg Pred Value NA
Prevalence 0
Detection Rate 0
Detection Prevalence 0
Balanced Accuracy NA
-------(p.294) ROC曲線計算------------------------------------------------
> install.packages("ROCR") 準備
Installing package into ‘C:/Users/ekaminuma/Documents/R/win-library/3.1’
(as ‘lib’ is unspecified)
Warning in install.packages :
package ‘ROCR’ is in use and will not be installed
> library(ROCR)
定性クラスを定量クラスへ変更
> tmp1<-as.numeric(myPred)
> tmp1
[1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
[32] 2 2 2
> tmp2<-as.numeric(testClass)
> tmp2
[1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
[32] 2 2 2
> out<-prediction(tmp1,tmp2)
> out
An object of class "prediction"
Slot "predictions": ←予測値
[[1]]
[1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
[32] 2 2 2
Slot "labels": ←正解値
[[1]]
[1] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
[32] 2 2 2
Levels: 1 < 2
Slot "cutoffs":
[[1]]
[1] Inf 2 1
Slot "fp":
[[1]]
[1] 0 0 15
Slot "tp":
[[1]]
[1] 0 19 19
Slot "tn":
[[1]]
[1] 15 15 0
Slot "fn":
[[1]]
[1] 19 0 0
Slot "n.pos":
[[1]]
[1] 19
Slot "n.neg":
[[1]]
[1] 15
Slot "n.pos.pred":
[[1]]
[1] 0 19 34
Slot "n.neg.pred":
[[1]]
[1] 34 15 0
perf<-performance(out,"tpr","fpr")
plot(perf)
参考：ROC曲線、AUCのわかり易い説明
http://cse.fra.affrc.go.jp/kiyo/home/pop/intro/Topics/entori/2014/1/8_Habitat_model_and_AUC_files/ROC-AUC.pdf
> data(ROCR.simple)
> with(ROCR.simple,plot(predictions,labels))
> summary(ROCR.simple)
Length Class Mode
predictions 200 -none- numeric
labels 200 -none- numeric
> ROCR.simple
$predictions
[1] 0.612547843 0.364270971 0.432136142 0.140291078 0.384895941
[6] 0.244415489 0.970641299 0.890172812 0.781781371 0.868751832
[11] 0.716680598 0.360168796 0.547983407 0.385240464 0.423739359
[16] 0.101699993 0.628095575 0.744769966 0.657732644 0.490119891
[21] 0.072369921 0.172741714 0.105722115 0.890078186 0.945548941
[26] 0.984667270 0.360180429 0.448687336 0.014823599 0.543533783
[31] 0.292368449 0.701561487 0.715459280 0.714985914 0.120604738
[36] 0.319672178 0.911723615 0.757325590 0.090988280 0.529402244
[41] 0.257402979 0.589909284 0.708412104 0.326672910 0.086546283
[46] 0.879459891 0.362693564 0.230157183 0.779771989 0.876086217
[51] 0.353281048 0.212014560 0.703293499 0.689075677 0.627012496
[56] 0.240911145 0.402801992 0.134794140 0.120473353 0.665444679
[61] 0.536339509 0.623494622 0.885179651 0.353777439 0.408939895
[66] 0.265686095 0.932159806 0.248500489 0.858876675 0.491735594
[71] 0.151350957 0.694457482 0.496513160 0.123504905 0.499788081
[76] 0.310718619 0.907651100 0.340078180 0.195097957 0.371936985
[81] 0.517308606 0.419560072 0.865639036 0.018527600 0.539086009
[86] 0.005422562 0.772728821 0.703885141 0.348213542 0.277656869
[91] 0.458674210 0.059045866 0.133257805 0.083685883 0.531958184
[96] 0.429650397 0.717845453 0.537091350 0.212404891 0.930846938
[101] 0.083048377 0.468610247 0.393378108 0.663367560 0.349540913
[106] 0.194398425 0.844415442 0.959417835 0.211378771 0.943432189
[111] 0.598162949 0.834803976 0.576836208 0.380396459 0.161874325
[116] 0.912325837 0.642933593 0.392173971 0.122284044 0.586857799
[121] 0.180631658 0.085993218 0.700501359 0.060413627 0.531464015
[126] 0.084254795 0.448484671 0.938583020 0.531006532 0.785213140
[131] 0.905121019 0.748438143 0.605235403 0.842974300 0.835981859
[136] 0.364288579 0.492596896 0.488179708 0.259278968 0.991096434
[141] 0.757364019 0.288258273 0.773336236 0.040906997 0.110241034
[146] 0.760726142 0.984599159 0.253271061 0.697235328 0.620501132
[151] 0.814586047 0.300973098 0.378092079 0.016694412 0.698826511
[156] 0.658692553 0.470206008 0.501489336 0.239143340 0.050999138
[161] 0.088450984 0.107031842 0.746588080 0.480100183 0.336592126
[166] 0.579511087 0.118555284 0.233160827 0.461150807 0.370549294
[171] 0.770178504 0.537336015 0.463227453 0.790240205 0.883431431
[176] 0.745110673 0.007746305 0.012653524 0.868331219 0.439399995
[181] 0.540221346 0.567043171 0.035815400 0.806543942 0.248707470
[186] 0.696702150 0.081439129 0.336315317 0.126480399 0.636728451
[191] 0.030235062 0.268138293 0.983494405 0.728536415 0.739554341
[196] 0.522384507 0.858970526 0.383807972 0.606960209 0.138387070
$labels
[1] 1 1 0 0 0 1 1 1 1 0 1 0 1 0 0 0 1 1 1 0 0 0 0 1 0 1 0 0 1 1
[31] 0 1 1 1 0 0 1 1 0 1 0 1 0 1 0 1 0 1 0 1 1 0 1 0 1 0 0 0 0 1
[61] 1 1 1 0 0 0 1 0 1 0 0 1 0 0 0 0 0 0 0 0 1 0 1 0 0 1 1 0 0 1
[91] 0 0 1 0 1 0 1 1 0 1 0 0 0 1 0 0 1 0 0 1 1 1 0 0 0 1 1 0 0 1
[121] 0 0 1 0 1 0 0 1 1 1 1 1 0 1 1 0 0 0 0 1 1 0 1 0 1 0 1 1 1 1
[151] 1 0 0 0 1 1 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 1 0 1 1 1 0 1 1 0
[181] 1 1 0 1 0 0 0 1 0 0 0 1 0 1 1 0 1 0 1 0
> pred <- prediction(ROCR.simple$predictions, ROCR.simple$labels)
> perf_auc = performance(pred,'auc')
> perf_auc
An object of class "performance"
Slot "x.name":
[1] "None"
Slot "y.name":
[1] "Area under the ROC curve"
Slot "alpha.name":
[1] "none"
Slot "x.values":
list()
Slot "y.values":
[[1]]
[1] 0.8341875
Slot "alpha.values":
list()
> auc <- [email protected][[1]]
> auc
[1] 0.8341875
>
> out <- performance(pred,"tpr","fpr")
> plot(out)
> abline(0,1)
> out
An object of class "performance"
Slot "x.name":
[1] "False positive rate"
Slot "y.name":
[1] "True positive rate"
Slot "alpha.name":
[1] "Cutoff"
Slot "x.values":
[[1]]
[1] 0.000000000 0.000000000 0.000000000
[4] 0.000000000 0.009345794 0.009345794
[7] 0.018691589 0.028037383 0.028037383
：
[196] 0.981308411 0.990654206 0.990654206
[199] 0.990654206 1.000000000 1.000000000
Slot "y.values":
[[1]]
[1] 0.00000000 0.01075269 0.02150538 0.03225806
[5] 0.03225806 0.04301075 0.04301075 0.04301075
[9] 0.05376344 0.06451613 0.07526882 0.08602151
：
[185] 0.96774194 0.96774194 0.96774194 0.96774194
[189] 0.96774194 0.96774194 0.96774194 0.96774194
[193] 0.96774194 0.96774194 0.96774194 0.96774194
[197] 0.96774194 0.97849462 0.98924731 0.98924731
[201] 1.00000000
Slot "alpha.values":
[[1]]
[1] Inf 0.991096434 0.984667270
[4] 0.984599159 0.983494405 0.970641299
[7] 0.959417835 0.945548941 0.943432189
[10] 0.938583020 0.932159806 0.930846938
：
[193] 0.040906997 0.035815400 0.030235062
[196] 0.018527600 0.016694412 0.014823599
[199] 0.012653524 0.007746305 0.005422562