articleID <- "EXT_5-11-2015" # insert the article ID code here e.g., "10-3-2015_PS"
reportType <- "pilot" # specify whether this is the 'pilot' report or 'fisal' report
pilotNames <- "Andrew Nam" # insert the pilot's name here e.g., "Tom Hardwicke". If there are multiple cpilots enter both names in a character string e.g., "Tom Hardwicke, Bob Dylan"
copilotNames <- "Lauren Borchers" # # insert the co-pilot's name here e.g., "Michael Frank". If there are multiple co-pilots enter both names in a character string e.g., "Tom Hardwicke, Bob Dylan"
pilotTTC <- "240" # insert the pilot's estimated time to complete (in minutes, fine to approximate) e.g., 120
copilotTTC <- "240" # insert the co-pilot's estimated time to complete (in minutes, fine to approximate) e.g., 120
pilotStartDate <- "11/01/18" # insert the pilot's start date in US format e.g., as.Date("01/25/18", format = "%m/%d/%y")
copilotStartDate <- "11/02/18" # insert the co-pilot's start date in US format e.g., as.Date("01/25/18", format = "%m/%d/%y")
completionDate <- "11/04/19" # copilot insert the date of final report completion (after any necessary rounds of author assistance) in US format e.g., as.Date("01/25/18", format = "%m/%d/%y")The original authors conducted the study with 269 individuals on Amazon Mechanical Turk, two of which were excluded for not writing the tasked essay. The participants were further filtered using an initial attention check, then asked to write for 5 to 10 minutes about an event based on their condition. Participants were divided into 4 groups across 2 conditions: authentic vs. inauthentic (type of behavior) and general vs. unrelated to lying (type of event). Next, participants were asked to complete 7-point Likert scale measures: 10 questions about moral self-regard, 3 about feelings of impurity, and 4 about feelings of alienation. They were also asked to recall the original writing prompt and provide demographic information.
Impurity and moral self-regard. Similar 2 × 2 ANOVAs using impurity and moral self-regard as dependent measures also revealed only a significant main effect of type of behavior. Participants in the inauthentic-behavior condition reported greater feelings of impurity (M = 3.56, SD = 1.86, 95% CI = [3.30, 3.85]) and lower moral self-regard (M = 2.90, SD = 1.50, 95% CI = [2.61, 3.16]) than did participants in the authentic-behavior condition (impurity: M = 1.51, SD = 1.29, 95% CI = [1.25, 1.78]; moral self-regard: M = 4.99, SD = 1.68, 95% CI = [4.72, 5.26]), F(1, 263) = 111.06, p < .001, ηp 2 = .30, and F(1, 263) = 115.25, p < .001, ηp2 = .31, respectively.
# load packages
library(tidyverse) # for data munging
library(knitr) # for kable table formating
library(haven) # import and export 'SPSS', 'Stata' and 'SAS' Files
library(readxl) # import excel files
library(CARPSreports) # custom report functions
library(sjstats)# Prepare report object. This will be updated automatically by the reproCheck function each time values are compared
reportObject <- data.frame(dummyRow = TRUE, reportedValue = NA, obtainedValue = NA, valueType = NA, percentageError = NA, comparisonOutcome = NA, eyeballCheck = NA)data <- read_spss("data/data_Experiment_1.sav")data <- data %>%
mutate(subid = 1:nrow(data))data <- data %>%
filter(failed_essay == 0) %>%
select(subid, authenticity, general_type, moral_self_regard, impurity)desc = data %>%
group_by(authenticity) %>%
summarise_at(vars(moral_self_regard, impurity), funs(mean(.), sd(.)))
quiet <- reproCheck(reportedValue = '3.56', obtainedValue = desc[desc$authenticity == 0,]$impurity_mean, valueType = 'mean')## [1] "MATCH for mean. The reported value (3.56) and the obtained value (3.56) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '2.90', obtainedValue = desc[desc$authenticity == 0,]$moral_self_regard_mean, valueType = 'mean')## [1] "MATCH for mean. The reported value (2.9) and the obtained value (2.9) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.51', obtainedValue = desc[desc$authenticity == 1,]$impurity_mean, valueType = 'mean')## [1] "MATCH for mean. The reported value (1.51) and the obtained value (1.51) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '4.99', obtainedValue = desc[desc$authenticity == 1,]$moral_self_regard_mean, valueType = 'mean')## [1] "MATCH for mean. The reported value (4.99) and the obtained value (4.99) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.86', obtainedValue = desc[desc$authenticity == 0,]$impurity_sd, valueType = 'sd')## [1] "MATCH for sd. The reported value (1.86) and the obtained value (1.86) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.50', obtainedValue = desc[desc$authenticity == 0,]$moral_self_regard_sd, valueType = 'sd')## [1] "MATCH for sd. The reported value (1.5) and the obtained value (1.5) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.29', obtainedValue = desc[desc$authenticity == 1,]$impurity_sd, valueType = 'sd')## [1] "MATCH for sd. The reported value (1.29) and the obtained value (1.29) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.68', obtainedValue = desc[desc$authenticity == 1,]$moral_self_regard_sd, valueType = 'sd')## [1] "MATCH for sd. The reported value (1.68) and the obtained value (1.68) differed by 0%. Note that the obtained value was rounded to 2 decimal places to match the reported value."fit = aov(impurity ~ authenticity + general_type + authenticity*general_type, data=data)
df <- summary(fit)[[1]]
fstat = df[c(startsWith(row.names(df), "authenticity ")),][["F value"]]
pval = df[c(startsWith(row.names(df), "authenticity ")),][["Pr(>F)"]]
quiet <- reproCheck(reportedValue = '111.06', fstat, valueType = 'F')## [1] "MINOR_ERROR for F. The reported value (111.06) and the obtained value (110.67) differed by 0.35%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '<.001', obtainedValue = pval, valueType = 'p', eyeballCheck=TRUE)## [1] "MATCH for p. Eyeball comparison only."df = eta_sq(fit)
eta = df[df$term == "authenticity",]$etasq
quiet <- reproCheck(reportedValue = '.30', obtainedValue = eta, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (0.3) and the obtained value (0.29) differed by 3.33%. Note that the obtained value was rounded to 2 decimal places to match the reported value."interval = confint(lm(impurity ~ authenticity, data=data))
int_min = interval[c(startsWith(row.names(interval), "(Intercept)")),][["2.5 %"]]
int_max = interval[c(startsWith(row.names(interval), "(Intercept)")),][["97.5 %"]]
quiet <- reproCheck(reportedValue = '3.30', obtainedValue = int_min, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (3.3) and the obtained value (3.29) differed by 0.3%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '3.85', obtainedValue = int_max, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (3.85) and the obtained value (3.84) differed by 0.26%. Note that the obtained value was rounded to 2 decimal places to match the reported value."interval = confint(lm(moral_self_regard ~ authenticity, data=data))
int_min = interval[c(startsWith(row.names(interval), "(Intercept)")),][["2.5 %"]]
int_max = interval[c(startsWith(row.names(interval), "(Intercept)")),][["97.5 %"]]
quiet <- reproCheck(reportedValue = '2.61', obtainedValue = int_min, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (2.61) and the obtained value (2.62) differed by 0.38%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '3.16', obtainedValue = int_max, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (3.16) and the obtained value (3.17) differed by 0.32%. Note that the obtained value was rounded to 2 decimal places to match the reported value."interval = confint(lm(impurity ~ authenticity, data=data))
inauthenticity_mean = desc[desc$authenticity == 0,]$impurity_mean
int_min = inauthenticity_mean + interval[c(startsWith(row.names(interval), "authenticity")),][["2.5 %"]]
int_max = inauthenticity_mean + interval[c(startsWith(row.names(interval), "authenticity")),][["97.5 %"]]
quiet <- reproCheck(reportedValue = '1.25', obtainedValue = int_min, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (1.25) and the obtained value (1.13) differed by 9.6%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '1.78', obtainedValue = int_max, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (1.78) and the obtained value (1.9) differed by 6.74%. Note that the obtained value was rounded to 2 decimal places to match the reported value."interval = confint(lm(moral_self_regard ~ authenticity, data=data))
inauthenticity_mean = desc[desc$authenticity == 0,]$moral_self_regard_mean
int_min = inauthenticity_mean + interval[c(startsWith(row.names(interval), "authenticity")),][["2.5 %"]]
int_max = inauthenticity_mean + interval[c(startsWith(row.names(interval), "authenticity")),][["97.5 %"]]
quiet <- reproCheck(reportedValue = '4.72', obtainedValue = int_min, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (4.72) and the obtained value (4.6) differed by 2.54%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '5.26', obtainedValue = int_max, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (5.26) and the obtained value (5.37) differed by 2.09%. Note that the obtained value was rounded to 2 decimal places to match the reported value."fit = aov(moral_self_regard ~ authenticity + general_type + authenticity*general_type, data=data)
df <- summary(fit)[[1]]
fstat = df[c(startsWith(row.names(df), "authenticity ")),][["F value"]]
pval = df[c(startsWith(row.names(df), "authenticity ")),][["Pr(>F)"]]
quiet <- reproCheck(reportedValue = '115.25', fstat, valueType = 'F')## [1] "MINOR_ERROR for F. The reported value (115.25) and the obtained value (114.4) differed by 0.74%. Note that the obtained value was rounded to 2 decimal places to match the reported value."quiet <- reproCheck(reportedValue = '<.001', obtainedValue = pval, valueType = 'p', eyeballCheck=TRUE)## [1] "MATCH for p. Eyeball comparison only."df = eta_sq(fit)
eta = df[df$term == "authenticity",]$etasq
quiet <- reproCheck(reportedValue = '.31', obtainedValue = eta, valueType = 'other')## [1] "MINOR_ERROR for other. The reported value (0.31) and the obtained value (0.3) differed by 3.23%. Note that the obtained value was rounded to 2 decimal places to match the reported value."The reproducibility of the above study was successful for all means and standard deviations but failed to reproduce the exact confidence intervals with half of the intervals being constructed with < 1% difference and a maximum difference of 9.6%. The exact inferential statistics also failed to reproduce with the F-stat having a difference of less than 1% and the p-value having a difference of 3.23%.
Author_Assistance = FALSE # was author assistance provided? (if so, enter TRUE)
Insufficient_Information_Errors <- 0 # how many discrete insufficient information issues did you encounter?
# Assess the causal locus (discrete reproducibility issues) of any reproducibility errors. Note that there doesn't necessarily have to be a one-to-one correspondance between discrete reproducibility issues and reproducibility errors. For example, it could be that the original article neglects to mention that a Greenhouse-Geisser correct was applied to ANOVA outcomes. This might result in multiple reproducibility errors, but there is a single causal locus (discrete reproducibility issue).
locus_typo <- 0 # how many discrete issues did you encounter that related to typographical errors?
locus_specification <- 0 # how many discrete issues did you encounter that related to incomplete, incorrect, or unclear specification of the original analyses?
locus_analysis <- 0 # how many discrete issues did you encounter that related to errors in the authors' original analyses?
locus_data <- 0 # how many discrete issues did you encounter that related to errors in the data files shared by the authors?
locus_unidentified <- 2 # how many discrete issues were there for which you could not identify the cause
# How many of the above issues were resolved through author assistance?
locus_typo_resolved <- 0 # how many discrete issues did you encounter that related to typographical errors?
locus_specification_resolved <- 0 # how many discrete issues did you encounter that related to incomplete, incorrect, or unclear specification of the original analyses?
locus_analysis_resolved <- 0 # how many discrete issues did you encounter that related to errors in the authors' original analyses?
locus_data_resolved <- 0 # how many discrete issues did you encounter that related to errors in the data files shared by the authors?
locus_unidentified_resolved <- 0 # how many discrete issues were there for which you could not identify the cause
Affects_Conclusion <- FALSE # Do any reproducibility issues encounter appear to affect the conclusions made in the original article? TRUE, FALSE, or NA. This is a subjective judgement, but you should taking into account multiple factors, such as the presence/absence of decision errors, the number of target outcomes that could not be reproduced, the type of outcomes that could or could not be reproduced, the difference in magnitude of effect sizes, and the predictions of the specific hypothesis under scrutiny.reportObject <- reportObject %>%
filter(dummyRow == FALSE) %>% # remove the dummy row
select(-dummyRow) %>% # remove dummy row designation
mutate(articleID = articleID) %>% # add variables to report
select(articleID, everything()) # make articleID first column
# decide on final outcome
if(any(reportObject$comparisonOutcome %in% c("MAJOR_ERROR", "DECISION_ERROR")) | Insufficient_Information_Errors > 0){
finalOutcome <- "Failure without author assistance"
if(Author_Assistance == T){
finalOutcome <- "Failure despite author assistance"
}
}else{
finalOutcome <- "Success without author assistance"
if(Author_Assistance == T){
finalOutcome <- "Success with author assistance"
}
}
# collate report extra details
reportExtras <- data.frame(articleID, pilotNames, copilotNames, pilotTTC, copilotTTC, pilotStartDate, copilotStartDate, completionDate, Author_Assistance, finalOutcome, Insufficient_Information_Errors, locus_typo, locus_specification, locus_analysis, locus_data, locus_unidentified, locus_typo_resolved, locus_specification_resolved, locus_analysis_resolved, locus_data_resolved, locus_unidentified_resolved)
# save report objects
if(reportType == "pilot"){
write_csv(reportObject, "pilotReportDetailed.csv")
write_csv(reportExtras, "pilotReportExtras.csv")
}
if(reportType == "final"){
write_csv(reportObject, "finalReportDetailed.csv")
write_csv(reportExtras, "finalReportExtras.csv")
}devtools::session_info()## ─ Session info ──────────────────────────────────────────────────────────
## setting value
## version R version 3.5.1 (2018-07-02)
## os macOS 10.14
## system x86_64, darwin15.6.0
## ui X11
## language (EN)
## collate en_US.UTF-8
## ctype en_US.UTF-8
## tz America/Los_Angeles
## date 2018-11-04
##
## ─ Packages ──────────────────────────────────────────────────────────────
## package * version date lib
## assertthat 0.2.0 2017-04-11 [1]
## backports 1.1.2 2017-12-13 [1]
## base64enc 0.1-3 2015-07-28 [1]
## bayesplot 1.6.0 2018-08-02 [1]
## bindr 0.1.1 2018-03-13 [1]
## bindrcpp * 0.2.2 2018-03-29 [1]
## broom 0.5.0 2018-07-17 [1]
## callr 3.0.0 2018-08-24 [1]
## CARPSreports * 0.1 2018-11-01 [1]
## cellranger 1.1.0 2016-07-27 [1]
## cli 1.0.1 2018-09-25 [1]
## codetools 0.2-15 2016-10-05 [1]
## coin 1.2-2 2017-11-28 [1]
## colorspace 1.3-2 2016-12-14 [1]
## crayon 1.3.4 2017-09-16 [1]
## data.table 1.11.8 2018-09-30 [1]
## desc 1.2.0 2018-05-01 [1]
## devtools 2.0.1 2018-10-26 [1]
## digest 0.6.18 2018-10-10 [1]
## dplyr * 0.7.7 2018-10-16 [1]
## emmeans 1.3.0 2018-10-26 [1]
## estimability 1.3 2018-02-11 [1]
## evaluate 0.12 2018-10-09 [1]
## forcats * 0.3.0 2018-02-19 [1]
## fs 1.2.6 2018-08-23 [1]
## ggplot2 * 3.1.0 2018-10-25 [1]
## ggridges 0.5.1 2018-09-27 [1]
## glmmTMB 0.2.2.0 2018-07-03 [1]
## glue 1.3.0 2018-07-17 [1]
## gtable 0.2.0 2016-02-26 [1]
## haven * 1.1.2 2018-06-27 [1]
## hms 0.4.2 2018-03-10 [1]
## htmltools 0.3.6 2017-04-28 [1]
## httr 1.3.1 2017-08-20 [1]
## jsonlite 1.5 2017-06-01 [1]
## knitr * 1.20 2018-02-20 [1]
## lattice 0.20-35 2017-03-25 [1]
## lazyeval 0.2.1 2017-10-29 [1]
## lme4 1.1-18-1 2018-08-17 [1]
## lubridate 1.7.4 2018-04-11 [1]
## magrittr 1.5 2014-11-22 [1]
## MASS 7.3-50 2018-04-30 [1]
## Matrix 1.2-14 2018-04-13 [1]
## memoise 1.1.0 2017-04-21 [1]
## minqa 1.2.4 2014-10-09 [1]
## modelr 0.1.2 2018-05-11 [1]
## modeltools 0.2-22 2018-07-16 [1]
## multcomp 1.4-8 2017-11-08 [1]
## munsell 0.5.0 2018-06-12 [1]
## mvtnorm 1.0-8 2018-05-31 [1]
## nlme 3.1-137 2018-04-07 [1]
## nloptr 1.2.1 2018-10-03 [1]
## pillar 1.3.0 2018-07-14 [1]
## pkgbuild 1.0.2 2018-10-16 [1]
## pkgconfig 2.0.2 2018-08-16 [1]
## pkgload 1.0.2 2018-10-29 [1]
## plyr 1.8.4 2016-06-08 [1]
## prediction 0.3.6 2018-05-22 [1]
## prettyunits 1.0.2 2015-07-13 [1]
## processx 3.2.0 2018-08-16 [1]
## ps 1.2.0 2018-10-16 [1]
## purrr * 0.2.5 2018-05-29 [1]
## pwr 1.2-2 2018-03-03 [1]
## R6 2.3.0 2018-10-04 [1]
## Rcpp 0.12.19 2018-10-01 [1]
## readr * 1.1.1 2017-05-16 [1]
## readxl * 1.1.0 2018-04-20 [1]
## remotes 2.0.2 2018-10-30 [1]
## rlang 0.3.0.1 2018-10-25 [1]
## rmarkdown 1.10 2018-06-11 [1]
## rprojroot 1.3-2 2018-01-03 [1]
## rstudioapi 0.8 2018-10-02 [1]
## rvest 0.3.2 2016-06-17 [1]
## sandwich 2.5-0 2018-08-17 [1]
## scales 1.0.0 2018-08-09 [1]
## sessioninfo 1.1.0 2018-09-25 [1]
## sjlabelled 1.0.14 2018-09-12 [1]
## sjmisc 2.7.5 2018-09-13 [1]
## sjstats * 0.17.1 2018-10-02 [1]
## snakecase 0.9.2 2018-08-14 [1]
## stringdist 0.9.5.1 2018-06-08 [1]
## stringi 1.2.4 2018-07-20 [1]
## stringr * 1.3.1 2018-05-10 [1]
## survival 2.42-3 2018-04-16 [1]
## TH.data 1.0-9 2018-07-10 [1]
## tibble * 1.4.2 2018-01-22 [1]
## tidyr * 0.8.2 2018-10-28 [1]
## tidyselect 0.2.5 2018-10-11 [1]
## tidyverse * 1.2.1 2017-11-14 [1]
## TMB 1.7.14 2018-06-23 [1]
## usethis 1.4.0 2018-08-14 [1]
## withr 2.1.2 2018-03-15 [1]
## xml2 1.2.0 2018-01-24 [1]
## xtable 1.8-3 2018-08-29 [1]
## yaml 2.2.0 2018-07-25 [1]
## zoo 1.8-4 2018-09-19 [1]
## source
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## Github (METRICS-CARPS/CARPSreports@89db4a9)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.1)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
## CRAN (R 3.5.0)
##
## [1] /Library/Frameworks/R.framework/Versions/3.5/Resources/library