First we need to install and load any packages

pacman::p_load(tidyverse, ggplot2, irr, readxl, GGally, dplyr)

Import Data

data <- read_excel("StatementIOR_240403.xlsx")
View(data)

Make new dataframes

Make one dataframe for Domains and another for Constructs (all 78 statements)

domains <- data[, c("CynDom", "LesDom", "RobDom")] #need IOR on this df
constructs <- data[, c("CynCon", "LesCon", "RobCon")] #need IOR on this df

Make new data frames for each of the One Welfare components (26 statements each)

human_df <- filter(data, Category == "Human")
animal_df <- filter(data, Category == "Animal")
planet_df <- filter(data, Category == "Planet")

At this point, we have the full dataframe (‘data’) consisting of all 78 statements. We also have the ‘domains’ and ‘constructs’ dataframes which also consist of all 78 statements and each only contain the columns that we want to use for the IOR for domains and constructs, respectively.

I also have three smaller dataframes: animal_df, human_df, and planet_df, which each contain all the original columns, but only the 26 relevant statements each. I now need to take each of these dataframes and make the domain and construct dataframes for each.

Animal domains and constructs

animal_dom <- animal_df[, c("CynDom", "LesDom", "RobDom")] #need IOR on this df
animal_con <- animal_df[, c("CynCon", "LesCon", "RobCon")] #need IOR on this df

Human domains and constructs

human_dom <- human_df[, c("CynDom", "LesDom", "RobDom")] #need IOR on this df
human_con <- human_df[, c("CynCon", "LesCon", "RobCon")] #need IOR on this df

Planet domains and constructs

planet_dom <- planet_df[, c("CynDom", "LesDom", "RobDom")] #need IOR on this df
planet_con <- planet_df[, c("CynCon", "LesCon", "RobCon")] #need IOR on this df

Now to run Fleiss Kappa IOR tests

First for the domain dataframe

domainsIOR <- kappam.fleiss(domains) 

print(domainsIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 3 
##     Kappa = 0.691 
## 
##         z = 32 
##   p-value = 0

repeat for constructs dataframe

constructIOR <- kappam.fleiss(constructs) 

print(constructIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 3 
##     Kappa = 0.412 
## 
##         z = 31.4 
##   p-value = 0

Now for ‘animals’ statements only

animaldomainIOR <- kappam.fleiss(animal_dom) 

print(animaldomainIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.734 
## 
##         z = 17 
##   p-value = 0
animalconstructIOR <- kappam.fleiss(animal_con) 

print(animalconstructIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.556 
## 
##         z = 17.7 
##   p-value = 0

Now for ‘human’ statements only

humandomainIOR <- kappam.fleiss(human_dom) 

print(humandomainIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.612 
## 
##         z = 14.6 
##   p-value = 0
humanconstructIOR <- kappam.fleiss(human_con) 

print(humanconstructIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.285 
## 
##         z = 11.2 
##   p-value = 0

Now for ‘planet’ statements only

planetdomainIOR <- kappam.fleiss(planet_dom) 

print(planetdomainIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.695 
## 
##         z = 17.4 
##   p-value = 0
planetconstructIOR <- kappam.fleiss(planet_con) 

print(planetconstructIOR) 
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 26 
##    Raters = 3 
##     Kappa = 0.363 
## 
##         z = 13.8 
##   p-value = 0

Pairwise Kappa: Let’s see how each pair of observers did

First for domains (all 78 statements)

# Create subsets of the data
LesCynDom <- data[, c("CynDom", "LesDom")] 
LesRobDom <- data[, c("RobDom", "LesDom")] 
CynRobDom <- data[, c("CynDom", "RobDom")] 

# Compute Fleiss' Kappa for each pair of observers
kappa_LesCynDom <- kappam.fleiss(LesCynDom)
kappa_LesRobDom <- kappam.fleiss(LesRobDom)
kappa_CynRobDom <- kappam.fleiss(CynRobDom)

# Print the results
print(kappa_LesCynDom) #This will compare Lesley and Cynthia at domain level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.702 
## 
##         z = 19.3 
##   p-value = 0
print(kappa_LesRobDom) #This will compare Lesley and Robin at domain level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.658 
## 
##         z = 17.6 
##   p-value = 0
print(kappa_CynRobDom) #This will compare Robin and Cynthia at domain level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.711 
## 
##         z = 18 
##   p-value = 0

Now for constructs (all 78 statements)

# Create subsets of the data
LesCynCon <- data[, c("CynCon", "LesCon")] 
LesRobCon <- data[, c("RobCon", "LesCon")] 
CynRobCon <- data[, c("CynCon", "RobCon")] 

# Compute Fleiss' Kappa for each pair of observers
kappa_LesCynCon <- kappam.fleiss(LesCynCon)
kappa_LesRobCon <- kappam.fleiss(LesRobCon)
kappa_CynRobCon <- kappam.fleiss(CynRobCon)

# Print the results
print(kappa_LesCynCon) #This will compare Lesley and Cynthia at construct level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.479 
## 
##         z = 20.8 
##   p-value = 0
print(kappa_LesRobCon) #This will compare Lesley and Robin at construct level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.384 
## 
##         z = 16.1 
##   p-value = 0
print(kappa_CynRobCon) #This will compare Robin and Cynthia at construct level
##  Fleiss' Kappa for m Raters
## 
##  Subjects = 78 
##    Raters = 2 
##     Kappa = 0.371 
## 
##         z = 15.8 
##   p-value = 0