require("tidyverse")
## Loading required package: tidyverse
## Loading tidyverse: ggplot2
## Loading tidyverse: tibble
## Loading tidyverse: tidyr
## Loading tidyverse: readr
## Loading tidyverse: purrr
## Loading tidyverse: dplyr
## Conflicts with tidy packages ----------------------------------------------
## filter(): dplyr, stats
## lag():    dplyr, stats
require("irr") #kappa2
## Loading required package: irr
## Loading required package: lpSolve
Value of K Strength of agreement
< 0.20 Poor
0.21 - 0.40 Fair
0.41 - 0.60 Moderate
0.61 - 0.80 Good
0.81 - 1.00 Very good

1 References

  • Altman DG (1991) Practical statistics for medical research. London: Chapman and Hall.
  • Cohen J (1960) A coefficient of agreement for nominal scales. Educational and Psychological Measurement 20:37-46.

2 Dataset

kappa.lv <- read.csv("kappa_lv.csv", header = T, stringsAsFactors = F)

2.1 Rename levels

kappa.lv[kappa.lv=="A"] <- 1
kappa.lv[kappa.lv=="B"] <- 2
kappa.lv[kappa.lv=="C"] <- 2
kappa.lv[kappa.lv=="97"] <- 3
kappa.lv[kappa.lv=="99"] <- 0

2.2 Reformat the dataset

kappa.lv.wide <- kappa.lv %>% 
  unite("observer_time", observer, time) %>% 
  gather("tooth", "value", -c(observer_time, child)) %>% 
  unite("child_tooth", child, tooth ) %>% 
  spread(observer_time, value)
rm(kappa.lv)

3 Kappa

Convert all dataframe to factors

kappa.lv.wide[] <- lapply(kappa.lv.wide, factor)

Check levels

levels(kappa.lv.wide$`1_1`)
## [1] "0" "1" "2" "3"

Use only Cohen’s Kappa weighted

3.1 1

observer.1 <- kappa.lv.wide %>% 
  select(starts_with("1_"))
kappam.fleiss(observer.1,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 3920 
##    Raters = 2 
##     Kappa = 0.731 
## 
##         z = 52 
##   p-value = 0 
## 
##    Kappa      z p.value
## 0  0.749 46.889   0.000
## 1  0.694 43.436   0.000
## 2  0.788 49.360   0.000
## 3  1.000 62.610   0.000
kappa2(observer.1, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 3920 
##    Raters = 2 
##     Kappa = 0.806 
## 
##         z = 50.5 
##   p-value = 0

3.2 2

observer.2 <- kappa.lv.wide %>% 
  select(starts_with("2_"))
kappam.fleiss(observer.2,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 6160 
##    Raters = 2 
##     Kappa = 0.872 
## 
##         z = 74.9 
##   p-value = 0 
## 
##     Kappa      z p.value
## 0   0.874 68.619   0.000
## 1   0.866 67.969   0.000
## 2   0.912 71.603   0.000
## 98  0.666 52.292   0.000
kappa2(observer.2, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 6160 
##    Raters = 2 
##     Kappa = 0.876 
## 
##         z = 68.8 
##   p-value = 0

3.3 3

observer.3 <- kappa.lv.wide %>% 
  select(starts_with("3_"))
kappam.fleiss(observer.3,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 3640 
##    Raters = 2 
##     Kappa = 0.866 
## 
##         z = 63.9 
##   p-value = 0 
## 
##    Kappa      z p.value
## 0  0.865 52.168   0.000
## 1  0.833 50.268   0.000
## 2  0.941 56.801   0.000
kappa2(observer.3, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 3640 
##    Raters = 2 
##     Kappa = 0.901 
## 
##         z = 54.4 
##   p-value = 0

3.4 4

observer.4 <- kappa.lv.wide %>% 
  select(starts_with("4_"))
kappam.fleiss(observer.4,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 3500 
##    Raters = 2 
##     Kappa = 0.938 
## 
##         z = 57.3 
##   p-value = 0 
## 
##    Kappa      z p.value
## 0  0.938 55.485   0.000
## 1  0.937 55.447   0.000
## 2  0.965 57.112   0.000
kappa2(observer.4, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 3500 
##    Raters = 2 
##     Kappa = 0.941 
## 
##         z = 55.6 
##   p-value = 0

3.5 5

observer.5 <- kappa.lv.wide %>% 
  select(starts_with("5_"))
kappam.fleiss(observer.5,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 6020 
##    Raters = 2 
##     Kappa = 0.96 
## 
##         z = 84 
##   p-value = 0 
## 
##     Kappa      z p.value
## 0   0.964 74.796   0.000
## 1   0.953 73.966   0.000
## 2   0.972 75.407   0.000
## 98  1.000 77.589   0.000
kappa2(observer.5, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 6020 
##    Raters = 2 
##     Kappa = 0.974 
## 
##         z = 75.6 
##   p-value = 0

3.6 6

observer.6 <- kappa.lv.wide %>% 
  select(starts_with("6_"))
kappam.fleiss(observer.6,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 3360 
##    Raters = 2 
##     Kappa = 0.926 
## 
##         z = 55.2 
##   p-value = 0 
## 
##    Kappa      z p.value
## 0  0.925 53.636   0.000
## 1  0.923 53.523   0.000
## 2  1.000 57.966   0.000
kappa2(observer.6, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 3360 
##    Raters = 2 
##     Kappa = 0.932 
## 
##         z = 54.1 
##   p-value = 0

3.7 7

observer.7 <- kappa.lv.wide %>% 
  select(starts_with("7_"))
kappam.fleiss(observer.7,  detail  = T)
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 3920 
##    Raters = 2 
##     Kappa = 0.924 
## 
##         z = 63.7 
##   p-value = 0 
## 
##    Kappa      z p.value
## 0  0.924 57.843   0.000
## 1  0.913 57.146   0.000
## 2  1.000 62.610   0.000
kappa2(observer.7, "squared")
##  Cohen's Kappa for 2 Raters (Weights: squared)
## 
##  Subjects = 3920 
##    Raters = 2 
##     Kappa = 0.946 
## 
##         z = 59.2 
##   p-value = 0