Simulations for Ordlesing
N Foldnes
6/25/2020
Design
We have two stop rules (less or more precise)
- Standard error below 0.4, or reached 20 items
- Standard error below 0.2, or reached 40 items
We have two start rules
- Assume initial skill is -1
- Assume initial skill is -2
These are fully crossed to yield four conditions.
Simulation procedure
In each condition, we simulate 10 000 test takers, whose true ability range uniformly from -2.2 to + 2.2
From the simulation we obtain information such as the estimated ability for each student, the number of items for each student, etcetera
Results
Low precision, initial skill -1
res11 <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start = startmin1, test = test,
stop = stop1, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res11)
## The plot was not captured!a <- res11$final.values.df
qplot(a$true.theta, a$estimated.theta)+geom_smooth()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
Low precision, initial skill -2
res21 <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start = startmin2, test = test,
stop = stop1, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res21)
## The plot was not captured!a <- res21$final.values.df
qplot(a$true.theta, a$estimated.theta)+geom_smooth()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
High precision, initial skill -1
res12 <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start = startmin1, test = test,
stop = stop2, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res12)
## The plot was not captured!a <- res12$final.values.df
qplot(a$true.theta, a$estimated.theta)+geom_smooth()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
High precision, initial skill -2
res22 <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start = startmin2, test = test,
stop = stop2, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res22)
## The plot was not captured!a <- res22$final.values.df
qplot(a$true.theta, a$estimated.theta)+geom_smooth()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'
Classification error
Let us try to classify at-risk students. We define at-risk by true score less than -1. Say we use the rule
\[Estimated.Ability - 2\cdot SE < -1\] to identify at-risk students.
Low precision, initial skill -1
a <- res11$final.values.df
a$atrisk.true <- a$true.theta < -1
a$atrisk <- a$estimated.theta-2*a$final.SE < -1
library(ROCit)## Warning: package 'ROCit' was built under R version 3.6.2table(a$atrisk.true, a$atrisk)##
## FALSE TRUE
## FALSE 5107 2166
## TRUE 2 2725ROCit_obj <- rocit(score=as.numeric(a$atrisk),class=as.numeric(a$atrisk.true))
plot(ROCit_obj)
Low precision, initial skill -2
a <- res21$final.values.df
a$atrisk.true <- a$true.theta < -1
a$atrisk <- a$estimated.theta-2*a$final.SE < -1
library(ROCit)
table(a$atrisk.true, a$atrisk)##
## FALSE TRUE
## FALSE 6198 1075
## TRUE 54 2673ROCit_obj <- rocit(score=as.numeric(a$atrisk),class=as.numeric(a$atrisk.true))
plot(ROCit_obj)
High precision, initial skill -1
a <- res12$final.values.df
a$atrisk.true <- a$true.theta < -1
a$atrisk <- a$estimated.theta-2*a$final.SE < -1
library(ROCit)
table(a$atrisk.true, a$atrisk)##
## FALSE TRUE
## FALSE 6384 889
## TRUE 5 2722ROCit_obj <- rocit(score=as.numeric(a$atrisk),class=as.numeric(a$atrisk.true))
plot(ROCit_obj)
High precision, initial skill -2
a <- res22$final.values.df
a$atrisk.true <- a$true.theta < -1
a$atrisk <- a$estimated.theta-2*a$final.SE < -1
library(ROCit)
table(a$atrisk.true, a$atrisk)##
## FALSE TRUE
## FALSE 6388 885
## TRUE 14 2713ROCit_obj <- rocit(score=as.numeric(a$atrisk),class=as.numeric(a$atrisk.true))
plot(ROCit_obj)
Variations with randomesque and Ultra High precision
High precision, initial skill -1, randomesque=5
newstart <- list(theta = -1, startSelect = "MFI",
randomesque =5)
newtest <- list(method = "BM", itemSelect = "MFI", randomesque=5)
res12r <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start =newstart, test = newtest,
stop = stop2, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res12r)
## The plot was not captured!Ultra High precision (StdERR=0.1), initial skill -1, randomesque=5, Length 25
newstop <- list(rule = c("precision","length"), thr = c(0.1, 25))
newstart <- list(theta = -1, startSelect = "MFI",
randomesque =5)
newtest <- list(method = "BM", itemSelect = "MFI", randomesque=5)
res12rr <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start =newstart, test = newtest,
stop = newstop, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res12rr)
## The plot was not captured!Ultra High precision (StdERR=0.1), initial skill -1, randomesque=5, Length 40
newstop <- list(rule = c("precision","length"), thr = c(0.1, 40))
newstart <- list(theta = -1, startSelect = "MFI",
randomesque =5)
newtest <- list(method = "BM", itemSelect = "MFI", randomesque=5)
res12rr <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start =newstart, test = newtest,
stop = newstop, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res12rr)
## The plot was not captured!Ultra High precision (StdERR=0.1), initial skill -1, randomesque=5, Length 60
newstop <- list(rule = c("precision","length"), thr = c(0.1, 60))
newstart <- list(theta = -1, startSelect = "MFI",
randomesque =5)
newtest <- list(method = "BM", itemSelect = "MFI", randomesque=5)
res12rr <- simulateRespondents(thetas = seq(-2.2, 2.2, length.out=10^4),
itemBank = bank,
start =newstart, test = newtest,
stop = newstop, final = final)## Simulation process: 0 %
## Simulation process: 10 %
## Simulation process: 20 %
## Simulation process: 30 %
## Simulation process: 40 %
## Simulation process: 50 %
## Simulation process: 60 %
## Simulation process: 70 %
## Simulation process: 80 %
## Simulation process: 90 %
## Simulation process: 100 %plot(res12rr)
## The plot was not captured!