Multiple booklets 1PL items - tmatta/lsasim GitHub Wiki
We examined item parameter recovery under the following conditions: 1 (IRT model) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 3 (test booklets)
- One IRT model was included: Rasch model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value
- Two IRT R packages were evaluated:
TAM(version 2.4-9) andmirt(version 1.25) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration.
- Summary of item parameter recovery:
TAMandmirtdemonstrated a similar level of accuracy- b-parameter recovered well, with correlation ranging from 0.960 to 0.999, with bias ranging from -0.010 to 0, and with RMSE ranging from 0.055 to 0.333
- Sample sizes of 5000 consistently produced the most accurate results
- Four levels of test lengths performed very similarly
- When number of booklets increased, recovery accuracy slightly decreased
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL
#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a Rasch model
set.seed(4368) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_1pl = I,
thresholds = 1,
b_bounds = c(-2, 2))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
#------------------------------------------------------------------------------#
# Data simulation
#------------------------------------------------------------------------------#
set.seed(8088*(r+1))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment
= blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
b_par = item_pool$b)
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit Rasch model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = 'Rasch', verbose = F)
# fit Rasch model using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml(resp)
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b in mirt package
mirt_b <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b"]
# extract xsi.item (item difficulty) in TAM pacakge
tam_b <- tam.mod$item$xsi.item
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow = 1))
colnames(itempars) <- c("N", "I", "K", "rep")
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b <- cor( item_pool$b, mirt_b)
itempars$bias_mirt_b <- mean( mirt_b - item_pool$b )
itempars$RMSE_mirt_b <- sqrt(mean( ( mirt_b - item_pool$b)^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b <- cor( item_pool$b, tam_b)
itempars$bias_tam_b <- mean( tam_b - item_pool$b )
itempars$RMSE_tam_b <- sqrt(mean( ( tam_b - item_pool$b)^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b, bias_mirt_b, RMSE_mirt_b) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("Sample Size", "Test Length", "# of Booklets",
"corr_b", "bias_b", "RMSE_b")
round(mirt_recovery, 3)
## Sample Size Test Length # of Booklets corr_b bias_b RMSE_b
## 1 500 40 5 0.987 -0.004 0.182
## 2 1000 40 5 0.994 -0.006 0.126
## 3 5000 40 5 0.999 -0.001 0.055
## 4 500 60 5 0.988 -0.007 0.183
## 5 1000 60 5 0.994 -0.006 0.129
## 6 5000 60 5 0.999 -0.003 0.057
## 7 500 80 5 0.988 -0.003 0.180
## 8 1000 80 5 0.994 -0.008 0.127
## 9 5000 80 5 0.999 -0.003 0.056
## 10 500 100 5 0.988 -0.006 0.179
## 11 1000 100 5 0.994 -0.007 0.127
## 12 5000 100 5 0.999 -0.003 0.056
## 13 500 40 10 0.974 -0.008 0.262
## 14 1000 40 10 0.987 -0.010 0.184
## 15 5000 40 10 0.997 -0.005 0.081
## 16 500 60 10 0.974 -0.009 0.266
## 17 1000 60 10 0.987 -0.005 0.183
## 18 5000 60 10 0.997 -0.003 0.082
## 19 500 80 10 0.975 0.000 0.262
## 20 1000 80 10 0.987 -0.008 0.184
## 21 5000 80 10 0.997 -0.002 0.080
## 22 500 100 10 0.974 -0.008 0.261
## 23 1000 100 10 0.987 -0.007 0.179
## 24 5000 100 10 0.997 -0.002 0.080
## 25 500 40 15 0.960 -0.005 0.331
## 26 1000 40 15 0.981 -0.005 0.223
## 27 5000 40 15 0.996 -0.002 0.099
## 28 500 60 15 0.961 -0.007 0.332
## 29 1000 60 15 0.981 -0.007 0.226
## 30 5000 60 15 0.996 -0.001 0.099
## 31 500 80 15 0.960 -0.001 0.333
## 32 1000 80 15 0.981 -0.006 0.228
## 33 5000 80 15 0.996 -0.003 0.100
## 34 500 100 15 0.962 -0.001 0.319
## 35 1000 100 15 0.980 -0.009 0.229
## 36 5000 100 15 0.996 -0.004 0.100
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b, bias_tam_b, RMSE_tam_b) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("Sample Size", "Test Length", "# of Booklets",
"corr_b", "bias_b", "RMSE_b")
round(tam_recovery, 3)
## Sample Size Test Length # of Booklets corr_b bias_b RMSE_b
## 1 500 40 5 0.987 -0.005 0.182
## 2 1000 40 5 0.994 -0.006 0.126
## 3 5000 40 5 0.999 -0.002 0.055
## 4 500 60 5 0.988 -0.007 0.183
## 5 1000 60 5 0.994 -0.006 0.129
## 6 5000 60 5 0.999 -0.003 0.057
## 7 500 80 5 0.988 -0.004 0.180
## 8 1000 80 5 0.994 -0.008 0.127
## 9 5000 80 5 0.999 -0.003 0.056
## 10 500 100 5 0.988 -0.006 0.179
## 11 1000 100 5 0.994 -0.007 0.127
## 12 5000 100 5 0.999 -0.003 0.056
## 13 500 40 10 0.974 -0.008 0.262
## 14 1000 40 10 0.987 -0.010 0.184
## 15 5000 40 10 0.997 -0.005 0.081
## 16 500 60 10 0.974 -0.009 0.266
## 17 1000 60 10 0.987 -0.005 0.183
## 18 5000 60 10 0.997 -0.003 0.082
## 19 500 80 10 0.975 0.000 0.262
## 20 1000 80 10 0.987 -0.008 0.184
## 21 5000 80 10 0.997 -0.002 0.080
## 22 500 100 10 0.974 -0.008 0.261
## 23 1000 100 10 0.987 -0.008 0.179
## 24 5000 100 10 0.997 -0.002 0.080
## 25 500 40 15 0.960 -0.005 0.331
## 26 1000 40 15 0.981 -0.005 0.223
## 27 5000 40 15 0.996 -0.002 0.099
## 28 500 60 15 0.961 -0.007 0.332
## 29 1000 60 15 0.981 -0.007 0.226
## 30 5000 60 15 0.996 -0.001 0.099
## 31 500 80 15 0.960 -0.001 0.333
## 32 1000 80 15 0.981 -0.006 0.228
## 33 5000 80 15 0.996 -0.003 0.100
## 34 500 100 15 0.962 -0.001 0.319
## 35 1000 100 15 0.980 -0.009 0.229
## 36 5000 100 15 0.996 -0.004 0.100