The final item pool

After Oddny/Vibeke removed more than fifty items due to the timing contradiction, we are left with 324 items. The item parameters are as follows.

kable(round(itempool[, 1:2], 1))%>%  kable_styling(position = "center")
a b
ord_001_dag_2 1.5 -1.5
ord_003_dag_500 2.3 -0.8
ord_005_tak_2 2.1 -1.9
ord_006_tak_1 1.8 -2.0
ord_010_sak_1 1.7 -1.9
ord_012_sak_200 2.7 -0.9
ord_015_lam_500 1.5 -2.0
ord_018_rar_1 1.5 -2.0
ord_019_rar_500 2.1 -1.3
ord_020_rar_200 3.5 -0.6
ord_021_far_2 1.6 -2.2
ord_023_far_500 2.7 -1.2
ord_025_lat_2 2.5 -1.5
ord_027_lat_500 2.2 -1.2
ord_028_lat_200 1.8 -0.5
ord_029_hav_2 1.5 -1.7
ord_031_hav_500 1.6 -1.6
ord_035_lek_500 3.3 -1.3
ord_038_mel_1 2.0 -1.6
ord_042_pen_1 1.8 -1.7
ord_043_pen_500 2.5 -1.4
ord_049_ser_2 2.1 -2.1
ord_051_ser_500 1.7 -1.4
ord_053_les_2 1.9 -2.1
ord_057_fat_2 1.4 -2.0
ord_059_fat_500 2.0 -1.3
ord_060_fat_200 1.9 -0.9
ord_061_fin_2 1.8 -2.2
ord_067_rir_500 2.3 -1.4
ord_068_rir_200 1.6 -1.2
ord_071_ris_500 1.6 -2.2
ord_072_ris_200 2.4 -1.3
ord_073_lov_2 1.6 -2.1
ord_075_lov_500 1.3 -2.3
ord_080_bil_200 1.3 -0.6
ord_082_kos_1 1.7 -1.8
ord_083_kos_500 2.5 -1.3
ord_087_rop_500 3.9 -1.2
ord_088_rop_200 3.2 -0.7
ord_090_hun_1 1.3 -1.8
ord_091_hun_500 2.1 -1.2
ord_093_bur_2 2.1 -1.2
ord_095_bur_500 2.1 -0.8
ord_096_bur_200 4.0 -0.5
ord_097_dyr_2 1.6 -1.9
ord_099_dyr_500 2.2 -1.6
ord_103_sør_500 1.9 -1.3
ord_104_sør_200 3.0 -0.7
ord_105_dør_2 1.3 -2.0
ord_108_dør_200 1.8 -1.2
ord_111_søl_500 2.2 -1.2
ord_113_bål_2 1.5 -1.7
ord_115_bål_500 2.2 -0.9
ord_116_bål_200 3.8 -0.4
ord_117_bra_2 1.6 -2.0
ord_119_bra_500 1.9 -1.3
ord_120_bra_200 2.7 -1.0
ord_121_arm_2 1.9 -1.9
ord_123_arm_500 2.8 -1.3
ord_124_arm_200 2.9 -0.7
ord_125_sol_2 1.5 -2.2
ord_129_mus_2 1.7 -2.1
ord_131_mus_500 2.5 -1.2
ord_133_bok_2 1.7 -2.2
ord_137_syn_2 1.3 -2.3
ord_139_syn_500 2.9 -1.2
ord_141_dele_2 2.0 -1.5
ord_143_dele_500 3.0 -0.8
ord_144_dele_200 3.7 -0.4
ord_145_lege_2 1.9 -1.6
ord_147_lege_500 2.6 -1.0
ord_148_lege_200 2.6 -0.5
ord_150_gate_1 2.0 -1.6
ord_151_gate_500 1.9 -1.3
ord_153_hale_2 1.5 -2.5
ord_155_hale_500 2.0 -1.4
ord_156_hale_200 2.2 -0.7
ord_159_møte_500 2.3 -0.9
ord_160_møte_200 2.7 -0.5
ord_162_nyse_1 2.0 -1.5
ord_163_nyse_500 3.7 -0.8
ord_164_nyse_200 3.3 -0.5
ord_166_rope_1 1.8 -1.8
ord_169_våte_2 1.4 -2.1
ord_171_våte_500 2.7 -1.0
ord_174_kino_1 1.7 -1.5
ord_175_kino_500 2.8 -1.0
ord_176_kino_200 2.0 -0.7
ord_179_hjul_500 2.1 -1.3
ord_180_hjul_200 3.3 -0.6
ord_181_peis_2 2.0 -1.6
ord_183_peis_500 2.3 -1.1
ord_184_peis_200 2.9 -0.7
ord_186_sand_1 1.5 -1.6
ord_187_sand_500 2.9 -1.1
ord_188_sand_200 3.6 -0.5
ord_190_våge_1 2.6 -1.1
ord_191_våge_500 2.3 -0.8
ord_192_våge_200 3.2 -0.3
ord_193_kort_2 1.4 -2.0
ord_194_kort_1 1.7 -1.4
ord_195_kort_500 3.4 -0.8
ord_198_natt_1 2.3 -1.7
ord_199_natt_500 2.6 -1.3
ord_200_natt_200 2.6 -0.7
ord_206_leke_1 2.0 -1.4
ord_207_leke_500 1.9 -1.2
ord_208_leke_200 3.1 -0.4
ord_210_høne_1 1.3 -1.9
ord_211_høne_500 2.2 -0.9
ord_212_høne_200 2.8 -0.5
ord_213_tann_2 2.1 -1.8
ord_216_tann_200 2.4 -0.7
ord_217_lang_2 1.2 -2.2
ord_218_lang_1 3.1 -1.0
ord_220_lang_200 1.9 -0.7
ord_221_bart_2 1.9 -1.2
ord_223_bart_500 3.2 -0.6
ord_224_bart_200 2.9 -0.2
ord_226_bråk_1 1.4 -1.4
ord_227_bråk_500 2.0 -0.8
ord_228_bråk_200 2.5 -0.6
ord_231_brød_500 2.0 -1.5
ord_232_brød_200 3.6 -0.7
ord_233_hest_2 1.5 -2.0
ord_235_hest_500 2.8 -1.2
ord_236_hest_200 2.5 -0.9
ord_239_såpe_500 2.1 -1.0
ord_241_bord_2 1.3 -1.8
ord_242_bord_1 1.6 -1.2
ord_243_bord_500 4.4 -0.7
ord_246_rote_1 1.7 -1.6
ord_247_rote_500 2.1 -1.2
ord_248_rote_200 3.2 -0.7
ord_254_klam_1 1.4 -1.2
ord_255_klam_500 2.1 -0.5
ord_256_klam_200 3.1 -0.2
ord_257_fyre_2 1.4 -1.8
ord_259_fyre_500 2.3 -1.3
ord_260_fyre_200 1.9 -1.1
ord_262_slått_1 1.3 -1.0
ord_263_slått_500 1.7 -0.4
ord_266_frosk_1 1.6 -1.1
ord_267_frosk_500 2.0 -0.6
ord_268_frosk_200 3.6 -0.3
ord_271_sykle_500 2.9 -1.1
ord_272_sykle_200 2.3 -0.6
ord_275_lekser_500 1.6 -1.0
ord_278_klokke_1 1.6 -0.7
ord_279_klokke_500 2.2 -0.3
ord_280_klokke_200 3.4 -0.1
ord_282_synge_1 2.2 -1.1
ord_283_synge_500 3.2 -0.5
ord_284_synge_200 3.3 -0.4
ord_287_arver_500 1.6 -0.8
ord_290_skolen_1 1.4 -1.3
ord_296_hunden_200 2.7 0.2
ord_299_padle_500 2.6 -0.6
ord_300_padle_200 3.4 -0.3
ord_302_esker_1 1.2 -0.9
ord_303_esker_500 1.6 -0.3
ord_304_esker_200 1.6 0.1
ord_307_planke_500 2.0 -0.9
ord_310_niste_1 2.2 -0.8
ord_311_niste_500 2.5 -0.4
ord_312_niste_200 3.0 -0.2
ord_316_bamse_200 2.7 -0.5
ord_318_hjerne_1 2.2 -1.1
ord_320_hjerne_200 2.2 -0.3
ord_323_lappen_500 1.8 -0.8
ord_324_lappen_200 1.4 -0.8
ord_327_spruter_500 2.1 0.6
ord_330_bukser_1 1.9 -0.9
ord_331_bukser_500 2.6 -0.6
ord_332_bukser_200 2.8 -0.3
ord_335_genser_500 2.2 -0.5
ord_336_genser_200 2.4 -0.2
ord_338_bakke_1 1.6 -1.6
ord_340_bakke_200 2.0 -1.0
ord_342_hjelm_1 1.8 -0.5
ord_343_hjelm_500 1.6 -0.2
ord_344_hjelm_200 2.2 0.2
ord_348_bilde_200 2.3 0.3
ord_351_glade_500 2.4 -1.0
ord_352_glade_200 3.3 -0.5
ord_354_kosten_1 2.1 -1.2
ord_355_kosten_500 1.9 -0.9
ord_359_visper_500 2.0 0.1
ord_360_visper_200 2.6 0.5
ord_362_kornet_1 1.4 -0.2
ord_363_kornet_500 1.5 0.2
ord_367_silke_500 1.2 0.0
ord_371_slave_500 2.1 0.0
ord_372_slave_200 2.5 0.3
ord_375_teller_500 2.7 -0.7
ord_376_teller_200 2.9 -0.4
ord_378_brøler_1 1.7 -1.2
ord_379_brøler_500 3.1 -0.5
ord_380_brøler_200 2.8 -0.3
ord_383_gnager_500 1.7 -0.1
ord_384_gnager_200 1.6 0.4
ord_387_eksem_500 1.6 -0.6
ord_390_kloden_1 2.2 -0.5
ord_391_kloden_500 3.0 -0.3
ord_392_kloden_200 3.0 0.1
ord_395_flaske_500 2.5 0.3
ord_396_flaske_200 1.9 0.6
ord_399_styrke_500 2.1 0.0
ord_400_styrke_200 1.7 0.3
ord_408_skrape_200 1.4 0.8
ord_411_salve_500 1.7 0.2
ord_412_salve_200 1.9 0.5
ord_414_lykten_1 1.5 -1.2
ord_416_lykten_200 1.9 -0.6
ord_420_mynter_200 1.8 -0.8
ord_422_drops_1 1.5 -1.3
ord_423_drops_500 3.3 -0.8
ord_424_drops_200 3.1 -0.4
ord_427_våren_500 1.5 -1.2
ord_428_våren_200 3.1 -0.3
ord_430_dukken_1 1.3 -1.4
ord_432_dukken_200 2.0 -0.5
ord_435_sukker_500 3.2 -1.0
ord_436_sukker_200 2.2 -0.8
ord_442_luktet_1 1.2 -0.2
ord_443_luktet_500 1.6 0.0
ord_444_luktet_200 1.9 0.5
ord_452_dusten_200 1.5 -0.7
ord_456_skøyter_200 2.5 0.2
ord_458_tenker_1 1.8 -1.0
ord_459_tenker_500 4.0 -0.3
ord_460_tenker_200 3.7 -0.1
ord_462_krangle_1 2.1 -0.6
ord_464_krangle_200 3.5 -0.1
ord_467_trasket_500 2.5 0.1
ord_468_trasket_200 2.7 0.3
ord_470_vinduer_1 1.3 -1.0
ord_471_vinduer_500 2.1 -0.7
ord_472_vinduer_200 2.1 -0.2
ord_478_marsjerer_1 1.4 -0.7
ord_479_marsjerer_500 3.3 -0.2
ord_480_marsjerer_200 2.3 0.1
ord_482_kantarell_1 1.5 -1.0
ord_483_kantarell_500 2.0 -0.5
ord_484_kantarell_200 1.9 0.1
ord_486_tallerken_1 1.1 -1.3
ord_494_glitret_1 2.4 -0.5
ord_495_glitret_500 2.5 -0.3
ord_496_glitret_200 2.5 0.1
ord_499_praktisk_500 2.2 0.1
ord_500_praktisk_200 2.4 0.4
ord_503_speilet_500 2.0 -0.7
ord_504_speilet_200 2.7 -0.2
ord_516_kanonen_200 1.7 0.9
ord_519_fantastisk_500 1.4 -0.4
ord_520_fantastisk_200 1.8 -0.1
ord_523_transport_500 2.3 -0.7
ord_524_transport_200 2.0 -0.5
ord_530_prestene_1 1.7 -0.4
ord_531_prestene_500 1.8 0.0
ord_532_prestene_200 2.0 0.4
ord_538_instrument_1 2.4 -0.6
ord_539_instrument_500 2.5 -0.2
ord_542_balansere_1 2.5 -0.4
ord_544_balansere_200 2.4 0.4
ord_547_direkte_500 2.6 -0.3
ord_548_direkte_200 2.9 -0.1
ord_555_annerledes_500 2.5 -0.6
ord_556_annerledes_200 2.7 -0.1
ord_559_stativene_500 3.1 -0.1
ord_560_stativene_200 2.0 0.1
ord_563_stranden_500 2.8 -0.3
ord_564_stranden_200 2.7 -0.1
ord_570_strutsene_1 1.5 -0.3
ord_575_spraglete_500 2.2 0.3
ord_576_spraglete_200 2.0 0.6
ord_579_uhøflig_500 3.4 0.0
ord_580_uhøflig_200 2.8 0.2
ord_583_hemmelig_500 2.8 -0.8
ord_588_slåsskamp_200 1.9 -0.2
ord_591_eksplosjon_500 2.1 0.1
ord_592_eksplosjon_200 2.7 0.4
ord_599_pasning_500 2.6 -0.6
ord_600_pasning_200 2.9 -0.2
ord_606_ordentlig_1 2.3 -1.1
ord_607_ordentlig_500 2.7 -0.7
ord_608_ordentlig_200 2.6 -0.6
ord_611_konsentrasjon_500 2.0 -0.1
ord_622_kraftverk_1 3.1 -0.3
ord_623_kraftverk_500 3.0 0.0
ord_624_kraftverk_200 2.2 0.3
ord_626_overraskelsen_1 1.4 -0.8
ord_627_overraskelsen_500 1.4 -0.4
ord_628_overraskelsen_200 1.8 0.2
ord_634_kalenderne_1 2.4 -0.5
ord_638_systemene_1 2.6 -0.6
ord_639_systemene_500 2.8 -0.2
ord_640_systemene_200 2.5 0.1
ord_644_kikkertene_200 1.4 0.7
ord_650_skyskraperen_1 1.3 -0.2
ord_651_skyskraperen_500 1.6 0.3
ord_652_skyskraperen_200 1.5 0.9
ord_654_tørklær_1 1.9 -0.3
ord_655_tørklær_500 2.3 0.2
ord_656_tørklær_200 2.3 0.4
ord_658_forklærne_1 1.6 -0.8
ord_659_forklærne_500 2.7 -0.4
ord_660_forklærne_200 1.8 -0.1
ord_670_kvalmende_1 1.8 -0.7
ord_671_kvalmende_500 3.1 0.0
ord_672_kvalmende_200 2.9 0.2
ord_690_alarmanlegg_1 1.4 -0.7
ord_692_alarmanlegg_200 1.9 0.3
ord_699_avslappende_500 1.8 -0.7
ord_700_avslappende_200 1.8 -0.5
ord_708_kontraktene_200 1.3 1.2
ord_710_forvekslinger_1 1.4 0.2
ord_712_forvekslinger_200 1.9 0.5
ord_722_legetimene_1 1.3 0.1
ord_723_legetimene_500 2.0 0.3
ord_744_allsidighet_200 1.5 0.4
ord_748_fyrstikkeske_200 1.2 -0.5
ord_750_skylde_500 3.2 -0.8
ord_188_sand_200.1 2.2 -0.7

Scatterplot of discrimination vs difficulty

ggplot(itempool, aes(a,b))+geom_point()+xlab("Discrimination")+ylab("Difficulty")

Simulations with fixed test length 25 and \(n=10^4\) students

Previous investigations revealedsystematic bias in the tails with BM ability estimation:

Bias in the tails for BM estimation.

Preliminary simulations also revealed that with 25 items we will reach accurate ability estimation in the lower tails. So we fixed test length to 25. The ability estimation should be performed using either the default Bayes modal (BM) or Warm’s method (1989, Psychometrika).

n <- 10^4
truethetas <- seq(-2.2,2.2, length.out=n)
est.df <- data.frame(truetheta=truethetas)

NMAX <- 25
SEMTHR <- .01 # UNATTAINABLE SO LENGTH WILL BE 25 FOR ALL STUDENTS
est.df$NMAX <- NMAX; est.df$SE <- SEMTHR;

stop <- list(rule = c("precision","length"), thr = c(SEMTHR, NMAX))
res <- simulateRespondents(thetas = truethetas,
                           itemBank = bank,
                           start=start,
                           test = list(method = "WL", itemSelect = "MFI"), 
                           stop = stop)
## Simulation process:  0 %
## Simulation process:  10 %
## Simulation process:  20 %
## Simulation process:  30 %
## Simulation process:  40 %
## Simulation process:  50 %
## Simulation process:  60 %
## Simulation process:  70 %
## Simulation process:  80 %
## Simulation process:  90 %
## Simulation process:  100 %
cat(paste("WL, NMAX=", NMAX, "SE=", SEMTHR, "\n"))
## WL, NMAX= 25 SE= 0.01
plot(res)

## The plot was not captured!
cond.df <- data.frame(condTheta=res$condTheta, 
                      condBias=res$condBias,
                      condRMSE=res$condRMSE,
                      condnItems=res$condnItems,
                      condSE=res$condSE,
                      condMethod = "WL")

res <- simulateRespondents(thetas = truethetas,
                           itemBank = bank,
                           start=start,
                           test = list(method = "BM", itemSelect = "MFI"), 
                           stop = stop)
## Simulation process:  0 %
## Simulation process:  10 %
## Simulation process:  20 %
## Simulation process:  30 %
## Simulation process:  40 %
## Simulation process:  50 %
## Simulation process:  60 %
## Simulation process:  70 %
## Simulation process:  80 %
## Simulation process:  90 %
## Simulation process:  100 %
cat(paste("BM, NMAX=", NMAX, "SE=", SEMTHR, "\n"))
## BM, NMAX= 25 SE= 0.01
plot(res)

## The plot was not captured!
cond.df <- rbind(cond.df,data.frame(condTheta=res$condTheta, 
                      condBias=res$condBias,
                      condRMSE=res$condRMSE,
                      condnItems=res$condnItems,
                      condSE=res$condSE,
                      condMethod = "BM"))

Conditional Test length

ggplot(cond.df, aes(condTheta, condnItems, color=condMethod))+geom_line()+xlab("True ability")+ylab("Test length")+ggtitle("Test length")

Conditional Bias

ggplot(cond.df, aes(condTheta, condBias, color=condMethod))+geom_line()+xlab("True ability")+ylab("Bias")+
  ggtitle("BIAS")

Conditional RMSE

ggplot(cond.df, aes(condTheta, condRMSE, color=condMethod))+geom_line()+xlab("True ability")+ylab("RMSE")+
  ggtitle("RMSE
          ")

Conditional SE

ggplot(cond.df, aes(condTheta, condSE, color=condMethod))+geom_line()+xlab("True ability")+ylab("SE")+
  ggtitle("SE")

library("knitcitations")

Conclusion

Using the Bayes modal or the Warm ability estimator does not make any difference. Performance with fixed test length 25 yields a RMSE consistently below \(0.2\) up to the 75th ability percentile, which is deemed adequate for our purposes. We therefore end up with the following design