Import Data, check for missing data and outliers

library(rio)
library(gtheory)
## Loading required package: lme4
## Loading required package: Matrix
## 
## Attaching package: 'lme4'
## The following object is masked from 'package:rio':
## 
##     factorize
library(psych)
library(lme4)
Rating <- read.csv("/Users/Lorraine/Desktop/Rating.csv")
View(Rating)
sum(is.na(Rating)) # No missing data
## [1] 0
boxplot.stats(Rating$score)$out # No outlier
## integer(0)

G-Study

varComp <- gstudy(Rating, score ~ (1|ID) + (1|rater))
## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl =
## control$checkConv, : Model failed to converge with max|grad| = 0.00437103
## (tol = 0.002, component 1)
varComp$components
##     source       var percent n
## 1       ID 0.6333828    58.5 1
## 2    rater 0.1934205    17.9 1
## 3 Residual 0.2562312    23.7 1

The variance component is 0.63 for persons, 0.19 for raters, and 0.26 for residual and persons x raters interaction.

D-Study

dout <- dstudy(varComp, colname.objects = "ID", data = Rating, colname.scores = "score")
dout$components
##     source        var percent n
## 1       ID 0.63338284    84.9 1
## 2    rater 0.04835514     6.5 4
## 3 Residual 0.06405780     8.6 4
dout$generalizability
## [1] 0.908153

The generalizability coefficient for relative rankings of 4 raters is 0.91

D-Study for 1 rater

r1 <- dstudy(varComp, colname.objects = "ID")
## Warning in dstudy.gstudy(varComp, colname.objects = "ID"): Did you intend to calculate intraclass correlation for a single observation (i.e., n = 1 for all sources of variation)?
## You must specify the data and which column contains scores if you wish to conduct a D study based on observed scores.
r1$components
##     source       var percent n
## 1       ID 0.6333828    58.5 1
## 2    rater 0.1934205    17.9 1
## 3 Residual 0.2562312    23.7 1
r1$var.universe
## [1] 0.6333828
r1$var.error.rel
## [1] 0.2562312
r1$generalizability
## [1] 0.7119749

The generalizability coefficient for 1 rater is 0.71

D-Study for 2 raters

k <- 2
r2 <- r1$var.universe/(r1$var.universe + (r1$var.error.rel/k))
r2
## [1] 0.8317586

The generalizability coefficient for 2 raters is 0.83

D-Study for 3 raters

k <- 3
r3 <- r1$var.universe/(r1$var.universe + (r1$var.error.rel/k))
r3
## [1] 0.8811753

The generalizability coefficient for 3 raters is 0.88

How many raters would be needed to achieve reliability of 0.8 or better? What about 0.90?

At least 2 raters would be needed to achieve reliability of 0.8 or better and at least 4 raters would be needed for a reliability of 0.9 or better.