# Load required libraries
library(ggplot2)
library(dplyr)
##
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
##
## filter, lag
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
library(tidyr)
# 1. Construct the complete 2001–2026 dataset
pub_data <- data.frame(
Year = 2001:2026,
`Smart Search (N=45)` = c(1, 1, 1, 1, 2, 1, 1, 0, 1, 0, 1, 2, 1, 3, 3, 1, 0, 2, 2, 1, 5, 1, 6, 5, 3, 0),
`Boolean Search (N=17)` = c(0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3, 1, 4, 1, 1, 4),
check.names = FALSE
)
# 2. Reshape data into long format for ggplot2
pub_long <- pub_data %>%
pivot_longer(
cols = c(`Smart Search (N=45)`, `Boolean Search (N=17)`),
names_to = "Strategy",
values_to = "Count"
)
# 3. Generate publication-quality grouped bar chart
pub_plot <- ggplot(pub_long, aes(x = factor(Year), y = Count, fill = Strategy)) +
geom_col(position = position_dodge(preserve = "single", width = 0.8), width = 0.7) +
scale_fill_manual(
values = c("Smart Search (N=45)" = "#2B5C8F", "Boolean Search (N=17)" = "#D95F02")
) +
scale_y_continuous(
breaks = seq(0, 7, by = 1),
expand = expansion(mult = c(0, 0.05))
) +
labs(
title = "Publication Year Distribution by Search Strategy",
subtitle = "Comparison of Web of Science Smart Search vs. Structured Boolean Search",
x = "Publication Year",
y = "Record Count",
fill = "Search Strategy"
) +
theme_classic(base_size = 12) +
theme(
plot.title = element_text(face = "bold", size = 14, hjust = 0),
plot.subtitle = element_text(color = "grey30", size = 10, margin = margin(b = 10)),
axis.text.x = element_text(angle = 45, hjust = 1, vjust = 1, color = "black"),
axis.text.y = element_text(color = "black"),
axis.title = element_text(face = "bold"),
legend.position = "top",
legend.title = element_text(face = "bold"),
panel.grid.major.y = element_line(color = "grey90", linewidth = 0.3),
plot.margin = margin(t = 10, r = 15, b = 10, l = 10)
)
# Display graph
print(pub_plot)
###Cohen inter coder relaibility computation###
# Load required libraries
library(readxl)
library(irr)
## Loading required package: lpSolve
library(psych)
## Warning: package 'psych' was built under R version 4.5.3
##
## Attaching package: 'psych'
## The following objects are masked from 'package:ggplot2':
##
## %+%, alpha
# 1. Import and convert directly to a standard data frame
file_path <- "New Web of science coding decision.xlsx"
df <- as.data.frame(read_excel(file_path))
# 2. Select the two coder columns
coder_data <- df[, c("Coder_Sylvia", "Coder_Emmanuel")]
# 3. Method 1: Using 'irr' package
kappa_result <- kappa2(coder_data, weight = "unweighted")
print(kappa_result)
## Cohen's Kappa for 2 Raters (Weights: unweighted)
##
## Subjects = 62
## Raters = 2
## Kappa = 0.871
##
## z = 6.86
## p-value = 6.98e-12
# 4. Method 2: Using 'psych' package with as.matrix() for 95% CIs
kappa_ci <- cohen.kappa(as.matrix(coder_data))
print(kappa_ci)
## Call: cohen.kappa1(x = x, w = w, n.obs = n.obs, alpha = alpha, levels = levels,
## w.exp = w.exp)
##
## Cohen Kappa and Weighted Kappa correlation coefficients and confidence boundaries
## lower estimate upper
## unweighted kappa 0.75 0.87 0.99
## weighted kappa 0.75 0.87 0.99
##
## Number of subjects = 62
###R2a. Retrieval Effectiveness Result.#####
###How do Smart Search and Boolean Search differ in their overall ability to retrieve relevant documents?
library(readxl)
library(dplyr)
# 1. Load the dataset
file_path <- "New Web of science coding decision.xlsx"
df <- read_excel(file_path)
# 2. Categorize strategy and normalize titles
df <- df %>%
mutate(
Strategy = ifelse(grepl("^SM", Doc_ID), "Smart Search", "Boolean Search"),
Title_clean = tolower(trimws(Title))
)
# 3. Determine the Pooled Gold Standard (Total unique relevant articles across both searches)
gold_standard_n <- df %>%
group_by(Title_clean) %>%
summarise(is_relevant = max(Final_Consensus, na.rm = TRUE)) %>%
filter(is_relevant == 1) %>%
nrow()
cat("Total Pooled Gold Standard Relevant Articles:", gold_standard_n, "\n\n")
## Total Pooled Gold Standard Relevant Articles: 24
# 4. Compute Effectiveness Metrics per Search Strategy
effectiveness_summary <- df %>%
group_by(Strategy) %>%
summarise(
`Total Yield (N)` = n(),
`Relevant Retrieved (R)` = sum(Final_Consensus),
`Precision` = round(sum(Final_Consensus) / n(), 4),
`Recall` = round(sum(Final_Consensus) / gold_standard_n, 4)
)
print(effectiveness_summary)
## # A tibble: 2 × 5
## Strategy `Total Yield (N)` `Relevant Retrieved (R)` Precision Recall
## <chr> <int> <dbl> <dbl> <dbl>
## 1 Boolean Search 17 14 0.824 0.583
## 2 Smart Search 45 17 0.378 0.708
###RQ@b. Ranking effectiveness###
library(readxl)
library(dplyr)
# 1. Load dataset
file_path <- "New Web of science coding decision.xlsx"
df <- read_excel(file_path)
# 2. Assign search strategy and normalize titles
df <- df %>%
mutate(
Strategy = ifelse(grepl("^SM", Doc_ID), "Smart Search", "Boolean Search"),
Title_clean = tolower(trimws(Title))
)
# 3. Compute Pooled Gold Standard count (Total unique relevant items)
gold_standard_n <- df %>%
group_by(Title_clean) %>%
summarise(rel = max(Final_Consensus, na.rm = TRUE), .groups = "drop") %>%
filter(rel == 1) %>%
nrow()
# 4. Define core Information Retrieval (IR) metric functions
calc_p_at_k <- function(rel_vec, k) {
sum(rel_vec[1:min(k, length(rel_vec))]) / k
}
calc_dcg <- function(rel_vec, k) {
rel <- rel_vec[1:min(k, length(rel_vec))]
if (length(rel) == 0) return(0)
ranks <- 1:length(rel)
sum(rel / log2(ranks + 1))
}
calc_ndcg <- function(rel_vec, k, total_gold) {
dcg <- calc_dcg(rel_vec, k)
ideal_rel <- numeric(k)
ideal_rel[1:min(k, total_gold)] <- 1
idcg <- calc_dcg(ideal_rel, k)
if (idcg == 0) return(0)
return(dcg / idcg)
}
calc_map <- function(rel_vec, total_gold) {
rel_ranks <- which(rel_vec == 1)
if (length(rel_ranks) == 0) return(0)
precisions <- sapply(rel_ranks, function(r) sum(rel_vec[1:r]) / r)
sum(precisions) / total_gold
}
# 5. Compute comprehensive ranking metrics across cutoffs (10, 15, 17, 20)
ranking_summary <- df %>%
group_by(Strategy) %>%
arrange(Rank, .by_group = TRUE) %>%
summarise(
`Total Yield` = n(),
`P@10` = round(calc_p_at_k(Final_Consensus, 10), 4),
`P@15` = round(calc_p_at_k(Final_Consensus, 15), 4),
`P@17` = round(calc_p_at_k(Final_Consensus, 17), 4),
`P@20` = round(calc_p_at_k(Final_Consensus, 20), 4),
`MAP` = round(calc_map(Final_Consensus, gold_standard_n), 4),
`nDCG@10` = round(calc_ndcg(Final_Consensus, 10, gold_standard_n), 4),
`nDCG@15` = round(calc_ndcg(Final_Consensus, 15, gold_standard_n), 4),
`nDCG@17` = round(calc_ndcg(Final_Consensus, 17, gold_standard_n), 4),
`nDCG@20` = round(calc_ndcg(Final_Consensus, 20, gold_standard_n), 4),
.groups = "drop"
)
print(ranking_summary)
## # A tibble: 2 × 11
## Strategy `Total Yield` `P@10` `P@15` `P@17` `P@20` MAP `nDCG@10` `nDCG@15`
## <chr> <int> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 Boolean S… 17 1 0.933 0.824 0.7 0.581 1 0.956
## 2 Smart Sea… 45 0.6 0.6 0.529 0.5 0.444 0.700 0.676
## # ℹ 2 more variables: `nDCG@17` <dbl>, `nDCG@20` <dbl>
###Overlapping serach results######
library(ggplot2)
library(dplyr)
library(ggrepel) # Package to prevent label collision
# 1. Dataset setup
df_dumbbell <- data.frame(
Title = c(
"DRM & Accessibility of E-Resources",
"Comparison of Accessibility & Usability",
"Ensuring Accessibility of E-Resources",
"Assessment of DL Design Guidelines",
"BVI Users' Interactions with DLs",
"Enhancing Usability (Help Features)",
"Help-Seeking Situations (Mobile)"
),
BL_Rank = c(1, 2, 6, 8, 9, 12, 13),
SM_Rank = c(2, 1, 23, 4, 37, 3, 35)
) %>%
mutate(
Title = factor(Title, levels = rev(Title)),
Displaced = ifelse(SM_Rank > 20, "Displaced (>20)", "Top Rank (<=20)")
)
# 2. Plot with automatic label repulsion
ggplot(df_dumbbell) +
# Segment lines
geom_segment(
aes(x = BL_Rank, xend = SM_Rank, y = Title, yend = Title, color = Displaced),
linewidth = 1.2, alpha = 0.8
) +
# Dots
geom_point(aes(x = BL_Rank, y = Title), color = "#1f77b4", size = 4) +
geom_point(aes(x = SM_Rank, y = Title), color = "#ff7f0e", size = 4) +
# Repelled Labels (Prevents overlaps)
geom_text_repel(
aes(x = BL_Rank, y = Title, label = paste0("BL #", BL_Rank)),
color = "#1f77b4", fontface = "bold", size = 3.2,
direction = "y", nudge_y = 0.2, segment.color = NA
) +
geom_text_repel(
aes(x = SM_Rank, y = Title, label = paste0("SM #", SM_Rank)),
color = "#e65100", fontface = "bold", size = 3.2,
direction = "y", nudge_y = -0.2, segment.color = NA
) +
# Cutoff Lines & Text
geom_vline(xintercept = 17, linetype = "dashed", color = "gray50", alpha = 0.7) +
annotate("text", x = 17.5, y = 1.2, label = "Boolean Cutoff (N=17)",
angle = 90, color = "gray40", fontface = "bold", size = 3, hjust = 0) +
geom_vline(xintercept = 45, linetype = "dotted", color = "gray30", alpha = 0.7) +
annotate("text", x = 44.5, y = 1.2, label = "Smart Search Cutoff (N=45)",
angle = 90, color = "gray30", fontface = "bold", size = 3, hjust = 0) +
# Styling
scale_x_continuous(limits = c(-2, 48), breaks = seq(0, 45, by = 5)) +
scale_color_manual(values = c("Displaced (>20)" = "#d62728", "Top Rank (<=20)" = "#1f77b4")) +
theme_minimal(base_size = 11) +
theme(
legend.position = "none",
panel.grid.minor = element_blank(),
panel.grid.major.y = element_blank(),
plot.title = element_text(face = "bold", size = 12),
axis.text.y = element_text(face = "bold", color = "black")
) +
labs(
title = "Positional Displacement Gap of Overlapping Records",
x = "Rank Position (Lower Number = Higher Position)",
y = "Record Description"
)
####P@K plots ######
library(ggplot2)
library(dplyr)
library(gridExtra)
##
## Attaching package: 'gridExtra'
## The following object is masked from 'package:dplyr':
##
## combine
# 1. Compute P@k and Cumulative Yield in R
df_metrics <- df %>%
group_by(Strategy) %>%
arrange(Rank) %>%
mutate(
Cum_Relevant = cumsum(Final_Consensus),
Precision_at_k = Cum_Relevant / Rank
)
# 2. Precision-at-K Plot
p1 <- ggplot(df_metrics, aes(x = Rank, y = Precision_at_k, color = Strategy, group = Strategy)) +
geom_line(linewidth = 1.1) +
geom_point(size = 2) +
geom_vline(xintercept = 17, linetype = "dashed", color = "gray50") +
scale_color_manual(values = c("Boolean Search" = "#1f77b4", "Smart Search" = "#ff7f0e")) +
scale_y_continuous(limits = c(0, 1.05), breaks = seq(0, 1, 0.2)) +
theme_minimal() +
labs(title = "Precision Decay Across Rank Cutoffs (P@k)", x = "Rank Cutoff (k)", y = "Precision@k")
# 3. Cumulative Discovery Plot
p2 <- ggplot(df_metrics, aes(x = Rank, y = Cum_Relevant, color = Strategy, group = Strategy)) +
geom_line(linewidth = 1.1) +
geom_point(size = 2) +
geom_hline(yintercept = 24, linetype = "dotted", color = "red3") +
scale_color_manual(values = c("Boolean Search" = "#1f77b4", "Smart Search" = "#ff7f0e")) +
scale_y_continuous(limits = c(0, 25), breaks = seq(0, 25, 5)) +
theme_minimal() +
labs(title = "Cumulative Relevance Discovery Curve", x = "Rank Position", y = "Cumulative Relevant Found")
# Display plots side-by-side
grid.arrange(p1, p2, ncol = 2)
