Multiple Patterns Text Matching & N-grams Frequency

Ph.D. Candidate in Public Policy
tul54@psu.edu

Background & Data:  
  It’s an ongoing research project of a systematic literature review. Text data were article titles and abstracts that contain key terms of interest downloaded from Web of Science. In this demonstration, I will show how to extract the combined frequency table of individual words and n-grams. I mostly rely R base commands. I use WordDistance built-in stop words list. I also use the str_view_all command in the stringr package to highlight the matched patterns in a long text.  
  In text analysis, it may be necessary to create a project-specific stop words list.  

  Packages Preparation

library(readr) # import .csv
library(dbplyr) # %in%
library(DT) # show tables in R markdown
library(WordDistance) # a package currently available for IOS (still testing for Windows and Linux), contains a relatively comprehensive stop words list
data("stopwords") # a list of stop words
library(writexl) # export multiple excel (.xlsx) worksheets
library(stringr) # str_view_all highlights all searched terms a string

CGN <- read_csv("CGN.csv")
PADM <- CGN[grepl("public administration",CGN$`WoS Categories`, ignore.case = T)==TRUE,]
#names(CGN)

1 Subsetting Texts by Keywords

Index a list of abstracts by keywords.

PADM_text <- paste(PADM$`Article Title`, PADM$Abstract)
PADM$text <- PADM_text
ID.collab <- PADM$ID[grepl("collaborative governance", PADM$text, ignore.case = T)==T]
ID.netgov <- PADM$ID[grepl("governance network|network(|ed) governance", PADM$text, ignore.case = T)==T] # the previous excel did not capture the spelling variation of "networked governance"
ID.complex <- PADM$ID[grepl("complex(|ity) (theory|network|governance)|complex(|ity)(| adaptive) system", PADM$text, ignore.case = T)==T] # search multiple patterns
ID.self <- PADM$ID[grepl("self(| |-)govern|self(| |-)organi", PADM$text, ignore.case = T)==T]
ID.collabnet <- PADM$ID[grepl("collaborative network", PADM$text, ignore.case = T)==T]
ID.collabPAPM <- PADM$ID[grepl("collaborative public (administration|management)", PADM$text, ignore.case = T)==T]

It is important to search as many spelling variations as possible. When searching multiple terms/spelling variations in a string, use (), |, .* to translate your will to R codes. For example, the pattern "governance network|network(|ed) governance" matches “governance network(s)”, “network governance”, “networked governance”.  
 setdiff() shows the discrepancies of text searching. If setdiff returns numeric(0), it means the results are the same for both coding methods.

2 Complexity-related Concepts

2.1 Raw Frequency

Counting all occurrences of a pattern, and then aggregate all occurrences.

samplestr = PADM$text[PADM$ID%in%ID.complex] # A vector of texts
samplestr <- gsub("[[:punct:]]", "", samplestr) # remove punctuation
samplestr <- gsub("[[:digit:]]", "", samplestr) # remove numbers
samplestr  <- samplestr[samplestr!=""] # remove extra space
samplestr <- tolower(samplestr) # set to lower case b4 removing stopwords
# remove stop words AFTER splitting the entire string
split_samplestr <- strsplit(samplestr, fixed = F, perl = T, split = " ") # split into single-word units
for (i in 1: length(split_samplestr)) 
  {
  split_samplestr[[i]] <- split_samplestr[[i]][!split_samplestr[[i]]%in%stopwords]
  split_samplestr[[i]] <- split_samplestr[[i]][split_samplestr[[i]]!=""]
  names(split_samplestr[[i]]) <- seq_along(split_samplestr[[i]])
  }
str_length <- rep(NA, length(split_samplestr)) # must be OUTSIDE the {}
for (i in 1: length(split_samplestr)) 
  {  
  str_length[i] <- length(split_samplestr[[i]])
  }
# Make a matrix with the number of columns corresponding to the longest string
bigrams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-1)
for (i in 1: nrow(bigrams)) 
  {
  for(j in 1: ncol(bigrams)) 
    {
    bigrams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1])
  }
}
bigrams <- unlist(bigrams[grepl("NA", bigrams)==F]) # many columns are NAs b/c the ncols = max(string length)
bigrams <- as.data.frame(bigrams)
bigrams$Freq <- rep(1, nrow(bigrams))
bigrams <- aggregate(bigrams$Freq, by = list(bigrams$bigrams), sum)
names(bigrams) <- c("Term","Freq")
bigrams <- bigrams[bigrams$Freq>1,]
trigrams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-2)
for (i in 1: nrow(trigrams)) 
  {
  for(j in 1: ncol(trigrams)) 
    {
    trigrams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1], split_samplestr[[i]][j+2])
  }
}
trigrams <- unlist(trigrams[grepl("NA", trigrams)==F])
trigrams <- as.data.frame(trigrams)
trigrams$Freq <- rep(1, nrow(trigrams))
trigrams <- aggregate(trigrams$Freq, by = list(trigrams$trigrams), sum)
names(trigrams) <- c("Term","Freq")
trigrams <- trigrams[trigrams$Freq>1,]
four_grams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-3)
for (i in 1: nrow(four_grams)) 
  {
  for(j in 1: ncol(four_grams)) 
    {
    four_grams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1], split_samplestr[[i]][j+2], split_samplestr[[i]][j+3])
  }
}
four_grams <- unlist(four_grams[grepl("NA", four_grams)==F])
four_grams <- as.data.frame(four_grams)
four_grams$Freq <- rep(1, nrow(four_grams))
four_grams <- aggregate(four_grams$Freq, by = list(four_grams$four_grams), sum)
names(four_grams) <- c("Term","Freq")
four_grams <- four_grams[four_grams$Freq>1,]
word_Freq <- data.frame(unlist(split_samplestr), rep(1, length(unlist(split_samplestr))))
word_Freq <- aggregate(word_Freq[,2], by = list(word_Freq[,1]), sum)
names(word_Freq) <- c("Term","Freq")
Term_Freq <- rbind(bigrams, trigrams, four_grams, word_Freq)
Term_Freq <- Term_Freq[rev(order(Term_Freq$Freq)),]
Term_Freq <- aggregate(Term_Freq$Freq, by = list(Term_Freq$Term), sum)
names(Term_Freq) <- c("Term","Freq")
Term_Freq <- Term_Freq[rev(order(Term_Freq$Freq)),]
rownames(Term_Freq) <- seq(1: nrow(Term_Freq))
Complex_Term_Freq_Raw <- Term_Freq[Term_Freq$Freq>2,]
datatable(Complex_Term_Freq_Raw)

2.2 Adjusted Frequency

If a pattern occurs multiple times in the same text, count only once.

# Bigram
bigrams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-1)
for (i in 1: nrow(bigrams)) {
  for(j in 1: ncol(bigrams)) {
    bigrams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1])
  }
}
bigrams_list <- list()
for (i in 1:nrow(bigrams)) {
bigrams_list[[i]]<- unlist(bigrams[i,][grepl("NA", bigrams[i,])==F])
bigrams_list[[i]] <- unique(bigrams_list[[i]])
}
bigrams <- unlist(bigrams_list)
bigrams <- as.data.frame(bigrams)
bigrams$Freq <- rep(1, nrow(bigrams))
bigrams <- aggregate(bigrams$Freq, by = list(bigrams$bigrams), sum)
names(bigrams) <- c("Term","Freq")
bigrams <- bigrams[bigrams$Freq>1,]
## Trigram
trigrams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-2)
for (i in 1: nrow(trigrams)) {
  for(j in 1: ncol(trigrams)) {
    trigrams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1],split_samplestr[[i]][j+2])
  }
}
trigrams_list <- list()
for (i in 1:nrow(trigrams)) {
  trigrams_list[[i]]<- unlist(trigrams[i,][grepl("NA", trigrams[i,])==F])
  trigrams_list[[i]] <- unique(trigrams_list[[i]])
}
trigrams <- unlist(trigrams_list)
trigrams <- as.data.frame(trigrams)
trigrams$Freq <- rep(1, nrow(trigrams))
trigrams <- aggregate(trigrams$Freq, by = list(trigrams$trigrams), sum)
names(trigrams) <- c("Term","Freq")
trigrams <- trigrams[trigrams$Freq>1,]
# Four_grams
four_grams <- matrix(list(c("","")), nrow = length(split_samplestr), ncol = max(str_length)-3)
for (i in 1: nrow(four_grams)) {
  for(j in 1: ncol(four_grams)) {
    four_grams[i,j] <- paste(split_samplestr[[i]][j], split_samplestr[[i]][j+1],split_samplestr[[i]][j+2],split_samplestr[[i]][j+3])
  }
}
four_grams_list <- list()
for (i in 1:nrow(four_grams)) {
  four_grams_list[[i]]<- unlist(four_grams[i,][grepl("NA", four_grams[i,])==F])
  four_grams_list[[i]] <- unique(four_grams_list[[i]])
}
four_grams <- unlist(four_grams_list)
four_grams <- as.data.frame(four_grams)
four_grams$Freq <- rep(1, nrow(four_grams))
four_grams <- aggregate(four_grams$Freq, by = list(four_grams$four_grams), sum)
names(four_grams) <- c("Term","Freq")
four_grams <- four_grams[four_grams$Freq>1,]
# Single Words
unique_split_samplestr <- list()
for (i in 1: length(split_samplestr)) 
  {
unique_split_samplestr[[i]] <- unique(unlist(split_samplestr[[i]]))
}
word_Freq <- data.frame(unlist(unique_split_samplestr), rep(1, length(unlist(unique_split_samplestr))))
word_Freq <- aggregate(word_Freq[,2], by = list(word_Freq[,1]), sum)
names(word_Freq) <- c("Term","Freq")
# Combine Single Words, Bigrams, Trigrams, and Four_grams
Term_Freq <- rbind(bigrams, trigrams, four_grams, word_Freq)
Term_Freq <- Term_Freq[rev(order(Term_Freq$Freq)),]
Term_Freq <- aggregate(Term_Freq$Freq, by = list(Term_Freq$Term), sum)
names(Term_Freq) <- c("Term","Freq")
Term_Freq <- Term_Freq[rev(order(Term_Freq$Freq)),]
Complex_Term_Freq_Adjusted <- Term_Freq[Term_Freq$Freq>1,]
rownames(Complex_Term_Freq_Adjusted) <- seq(1: nrow(Complex_Term_Freq_Adjusted))

datatable(Complex_Term_Freq_Adjusted)

3 Collaborative Governance

3.1 Raw Frequency

Counting all occurrences of a pattern, and then aggregate all occurrences.

3.2 Adjusted Frequency

If a pattern occurs multiple times in the same text, count only once.

4 Network Governance

network(ed) governance/governance network(s)

4.1 Raw Frequency

4.2 Adjusted Frequency

If a pattern occurs multiple times in the same text, count only once.

5 Self-organi(z|s)(ing|ation)|Self-govern(ance|ing)

5.1 Raw Frequency

Counting all occurrences of a pattern, and then aggregate all occurrences.

5.2 Adjusted Frequency

If a pattern occurs multiple times in the same text, count only once.

Write Multiple Sheets to .xlsx

write_xlsx(list("Complex_Term_Raw" = Complex_Term_Freq_Raw, "Complex_Term_Adjusted" = Complex_Term_Freq_Adjusted, "Collaborative_Term_Raw" = Collab_Term_Freq_Raw, "Collaborative_Term_Adjusted" = Collab_Term_Freq_Adjusted, "NetGov_Term_Raw" = netgov_Term_Freq_Raw, "NetGov_Term_Adjusted" = netgov_Term_Freq_Adjusted, "Self_Term_Raw" = self_Term_Freq_Raw, "Self_Term_Adjusted" = self_Term_Freq_Adjusted), "Term_Freq_CGN.xlsx")