PISA 2012 mathematics items - tmatta/lsasim GitHub Wiki
We examined item parameter recovery under the following conditions: 2 (IRT models) x 1 (IRT R package) x 5 (sample sizes)
- Two types of IRT models were used: Rasch items and partial credit (PC) items
- Item parameters were drawn from PISA 2012 mathematics items used in standard booklets
- There were 76 Rasch items and 8 PC items
- The total 84 items were administered in the 13 standard test booklets
- One IRT R packages was evaluated:
TAM(version 2.4-9) - Five sample sizes were used: 2000, 4000, 6000, 8000, and 10000
- Simulated samples were based on one ability level from distribution N(0, 1)
- One hundred replications were used for each condition for the calibration.
- Summary of item parameter recovery:
TAMrecovered item parameters well- For Rasch items:
- b-parameter recovered well, with correlation ranging from 0.997 to 0.999, with bias ranging from -0.003 to -0.001, and with RMSE ranging from 0.049 to 0.11
- For PC items:
- b1-parameter recovered well, with correlation ranging from 0.992 to 0.998, with bias ranging from -0.004 to 0.011, and with RMSE ranging from 0.064 to 0.144
- b2-parameter recovered well, with correlation ranging from 0.994 to 0.999, with bias ranging from -0.005 to 0.002, and with RMSE ranging from 0.083 to 0.181
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
# Set up conditions
N.cond <- c(2000, 4000, 6000, 8000, 10000) #number of sample sizes
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL
#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (r in 1:reps) { #number of replications
set.seed(8088*(r+1))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
pisa2012_math_block_mat <- as.matrix(pisa2012_math_block[, -c(1:2)])
pisa_blocks <- lsasim::block_design(item_parameters = pisa2012_math_item,
item_block_matrix = pisa2012_math_block_mat)
#assign blocks to booklets
pisa2012_math_book_mat <- as.matrix(pisa2012_math_booklet[, -1])
pisa_books <- lsasim::booklet_design(item_block_assignment =
pisa_blocks$block_assignment,
book_design = pisa2012_math_book_mat)
#assign booklets to subjects
subj_booklets <- lsasim::booklet_sample(n_subj = N,
book_item_design = pisa_books)
#subset items in standard booklets
subitems <- sort(unique(subj_booklets$item))
pisa_items <- pisa2012_math_item[subitems, ]
# generate item responses
pisa_ir <- lsasim::response_gen(subject = subj_booklets$subject,
item = subj_booklets$item,
theta = theta,
item_no = pisa_items$item,
b_par = pisa_items$b,
d_par = list(pisa_items$d1,
pisa_items$d2))
# extract item responses (excluding "subject" column)
resp <- pisa_ir[,1 :length(subitems)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit Rasch and PC models using TAM package
tam.mod <- NULL
tam.mod <- TAM::tam.mml(resp, irtmodel = "PCM2", control = list(maxiter = 200))
tam.mod.par <- tam.mod$item
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# convert TAM output into PCM parametrization
#--- Rasch items
tam_b <- tam.mod.par[tam.mod.par$B.Cat2.Dim1==0, "AXsi_.Cat1"]
#--- PC items
tam_b1 <- tam.mod.par[tam.mod.par$B.Cat2.Dim1==2, "AXsi_.Cat1"]
tam_b2 <- tam.mod.par[tam.mod.par$B.Cat2.Dim1==2, "AXsi_.Cat2"] -
tam.mod.par[tam.mod.par$B.Cat2.Dim1==2, "AXsi_.Cat1"]
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, r), nrow = 1))
colnames(itempars) <- c("N", "rep")
# retrieve generated (PISA) item parameters
#--- Rasch items
raschitems_b <- pisa_items[pisa_items$d1==0, "b"]
#--- PC items
pcmitems_b1 <- pisa_items[pisa_items$d1!=0, "b"] + pisa_items[pisa_items$d1!=0, "d1"]
pcmitems_b2 <- pisa_items[pisa_items$d1!=0, "b"] + pisa_items[pisa_items$d1!=0, "d2"]
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b <- cor( raschitems_b, tam_b)
itempars$bias_tam_b <- mean( tam_b - raschitems_b )
itempars$RMSE_tam_b <- sqrt(mean( ( tam_b - raschitems_b)^2 ))
itempars$corr_tam_b1 <- cor( pcmitems_b1, tam_b1)
itempars$bias_tam_b1 <- mean( tam_b1 - pcmitems_b1 )
itempars$RMSE_tam_b1 <- sqrt(mean( ( tam_b1 - pcmitems_b1)^2 ))
itempars$corr_tam_b2 <- cor( pcmitems_b2 , tam_b2)
itempars$bias_tam_b2 <- mean( tam_b2 - pcmitems_b2 )
itempars$RMSE_tam_b2 <- sqrt(mean( ( tam_b2 - pcmitems_b2 )^2 ))
# combine results
results <- rbind(results, itempars)
}
}
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
#--- Rasch items
tam_recovery_1PL <- aggregate(cbind(corr_tam_b, bias_tam_b, RMSE_tam_b) ~ N ,
data=results, mean, na.rm=TRUE)
names(tam_recovery_1PL) <- c("Sample Size",
"corr_b", "bias_b", "RMSE_b")
round(tam_recovery_1PL, 3)
## Sample Size corr_b bias_b RMSE_b
## 1 2000 0.997 -0.001 0.110
## 2 4000 0.998 -0.003 0.078
## 3 6000 0.999 -0.002 0.064
## 4 8000 0.999 -0.003 0.054
## 5 10000 0.999 -0.001 0.049
#--- PC items
tam_recovery_PC <- aggregate(cbind(corr_tam_b1, bias_tam_b1, RMSE_tam_b1,
corr_tam_b2, bias_tam_b2, RMSE_tam_b2) ~ N ,
data=results, mean, na.rm=TRUE)
names(tam_recovery_PC) <- c("Sample Size",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2")
round(tam_recovery_PC, 3)
## Sample Size corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2
## 1 2000 0.992 0.011 0.144 0.994 -0.005 0.181
## 2 4000 0.996 0.004 0.103 0.997 0.000 0.131
## 3 6000 0.997 -0.004 0.082 0.998 0.002 0.101
## 4 8000 0.998 -0.004 0.074 0.998 -0.001 0.095
## 5 10000 0.998 -0.002 0.064 0.999 -0.003 0.083