Step 1: Simulate Small A/B Test

n_small <- 50
alpha <- 0.05
p_A <- 0.5
p_B <- 0.65
effect_size <- ES.h(p_B, p_A)

group <- rep(c("A", "B"), each = n_small / 2)
conversion <- c(rbinom(n_small/2, 1, p_A), rbinom(n_small/2, 1, p_B))
data_small <- data.frame(group, conversion)
head(data_small)
##   group conversion
## 1     A          1
## 2     A          1
## 3     A          1
## 4     A          1
## 5     A          0
## 6     A          0

Step 2: Analyze Small Sample

summary_small <- data_small %>%
  group_by(group) %>%
  summarise(conversion_rate = mean(conversion))
test_small <- prop.test(table(data_small$group, data_small$conversion))
test_small
## 
##  2-sample test for equality of proportions with continuity correction
## 
## data:  table(data_small$group, data_small$conversion)
## X-squared = 0.72115, df = 1, p-value = 0.3958
## alternative hypothesis: two.sided
## 95 percent confidence interval:
##  -0.1533851  0.4733851
## sample estimates:
## prop 1 prop 2 
##   0.56   0.40

Step 3: Power of Small Sample

power_small <- pwr.2p.test(h = effect_size, n = n_small/2, sig.level = alpha)$power
#Adjust Sample Size to 80% Power
target_power <- 0.80
sample_required <- ceiling(pwr.2p.test(h = effect_size, power = target_power, sig.level = alpha)$n * 2)


power_small
## [1] 0.1898896
sample_required
## [1] 339

Step 4:Simulate with Adequate Sample Size

group2 <- rep(c("A", "B"), each = sample_required / 2)
conversion2 <- c(rbinom(sample_required/2, 1, p_A), rbinom(sample_required/2, 1, p_B))
data_large <- data.frame(group = group2, conversion = conversion2)

Step 5:Analyze Large Sample

summary_large <- data_large %>%
  group_by(group) %>%
  summarise(conversion_rate = mean(conversion))
test_large <- prop.test(table(data_large$group, data_large$conversion))
power_large <- pwr.2p.test(h = effect_size, n = sample_required/2, sig.level = alpha)$power
power_large
## [1] 0.8009537

Summary Table

summary_df <- data.frame(
  Stage = c("Small Sample", "Adjusted Sample"),
  Sample_Size = c(n_small, sample_required),
  Conv_A = round(c(summary_small$conversion_rate[1], summary_large$conversion_rate[1]), 4),
  Conv_B = round(c(summary_small$conversion_rate[2], summary_large$conversion_rate[2]), 4),
  p_value = round(c(test_small$p.value, test_large$p.value), 4),
  Power = round(c(power_small, power_large), 4)
)
kable(summary_df, caption = "Comparison of A/B Test Results at Different Sample Sizes")
Comparison of A/B Test Results at Different Sample Sizes
Stage Sample_Size Conv_A Conv_B p_value Power
Small Sample 50 0.4400 0.6000 0.3958 0.1899
Adjusted Sample 339 0.5562 0.6746 0.0336 0.8010

🧪 Visualizations

data_small$sample_size <- "Small Sample"
data_large$sample_size <- "Large Sample"
data_all <- bind_rows(data_small, data_large)
ggbetweenstats(
  data = data_all,
  x = group,
  y = conversion,
  grouping.var = sample_size,
  title = "Comparison of Conversion Rates by Group and Sample Size",
  results.subtitle = TRUE
)

summary_plot_data <- data_all %>%
  group_by(group, sample_size) %>%
  summarise(
    mean_conversion = mean(conversion),
    se = sd(conversion) / sqrt(n()),
    .groups = "drop"
  )

ggplot(summary_plot_data, aes(x = group, y = mean_conversion, fill = sample_size)) +
  geom_col(position = position_dodge(0.8), width = 0.6) +
  geom_errorbar(aes(ymin = mean_conversion - 1.96 * se, ymax = mean_conversion + 1.96 * se),
                width = 0.2, position = position_dodge(0.8)) +
  labs(title = "Conversion Rates by Group and Sample Size", x = "Group", y = "Mean Conversion Rate") +
  theme_minimal(base_size = 14) +
  scale_fill_manual(values = c("Small Sample" = "#EFC000FF", "Large Sample" = "#0073C2FF")) +
  ylim(0, 1)

ggplot(data_all, aes(x = group, y = conversion, fill = sample_size)) +
  geom_bar(stat = "summary", fun = mean, position = position_dodge()) +
  geom_signif(comparisons = list(c("A", "B")), map_signif_level = TRUE, y_position = 1.05, tip_length = 0.01) +
  facet_wrap(~ sample_size) +
  labs(title = "Conversion Comparison with Significance Bars") +
  theme_minimal()

ggplot(data_all, aes(x = as.numeric(as.factor(group)), y = conversion)) +
  stat_poly_line() +
  stat_poly_eq(aes(label = after_stat(eq.label)), parse = TRUE) +
  facet_wrap(~ sample_size) +
  labs(title = "Polynomial Trend Line on Conversion Data", x = "Group (1 = A, 2 = B)", y = "Conversion") +
  theme_minimal()

✅ Conclusion

A small sample (n = 50) produced inconclusive results (p = 0.76, power = 18%) Adjusted sample (n = 339) revealed significant differences (p = 0.001, power = 80%) Power analysis is essential in experimental design to avoid Type II errors ## R Markdown