Paquetes
library(pacman)
p_load("dplyr", "stringr", "ggplot2", "wordcloud","rmdformats","vembedr", "xfun")Análisis de conteo de palabras para conocer su frecuencia
Video de youtube de la charla:
embed_url("https://youtu.be/__YUt8-VNcI")- Este es un ejercicio de la materia de Estadistica aplicada de la clase de las 9:00 a.m. (LMV) del departamento de matemáticas de ITSON.
Procesamiento del lenguaje natural
Entendiendo el lenguaje
Funcion
FreqCategory <- function(value) {
strCategory <- ifelse(value <=5, " 5",
ifelse(value <=10, " 10",
ifelse(value <=20, " 20",
ifelse(value <=50, " 50",
ifelse(value <=100, " 100",
ifelse(value <=500, " 500",
ifelse(value <=1000, " 1,000",
">1,000")))))))
strCategory
}Datos de texto
setwd("~/itson/Semestre 4/estadisticas/r")
hackathon <- readLines("Hackathon.txt")
head(hackathon)## [1] "solo aqui probando algo"
## [2] ""
## [3] "hola guillermo y paola buenas tardes"
## [4] ""
## [5] "estamos dando unos minutos a que lleguen"
## [6] ""Conteo de lineas (renglones)
#longitud del vector
intLineCount <-length(hackathon)
intLineCount## [1] 1960Conteo de palabras por linea
lstUNPrfLines <- str_split(hackathon," ")
# palabras por linea
vciUNPrfWperL <- unlist(lapply(lstUNPrfLines, length))
# imprimir media de palabras por linea
mean(vciUNPrfWperL)## [1] 3.831122Conteo de palabras
# deslistar para obtener un vector de palabras
vcsUNPrfWords <- unlist(lstUNPrfLines)
# recuento total de palabras = longitud del vector
intWordCount <- length(vcsUNPrfWords)
# imprimir
intWordCount## [1] 7509Mostrar primeras 100 palabras
head(vcsUNPrfWords, 100)## [1] "solo" "aqui" "probando" "algo"
## [5] "" "hola" "guillermo" "y"
## [9] "paola" "buenas" "tardes" ""
## [13] "estamos" "dando" "unos" "minutos"
## [17] "a" "que" "lleguen" ""
## [21] "los" "demas" "participantes" "para"
## [25] "arrancar" "" "gracias" "a"
## [29] "todos" "los" "que" "se"
## [33] "estan" "uniendo" "" "vamos"
## [37] "a" "dar" "dos" "minutos"
## [41] "mas" "para" "que" ""
## [45] "lleguen" "los" "demas" "compañeros"
## [49] "y" "" "arrancamos" ""
## [53] "muy" "bien" "pues" "bienvenidos"
## [57] "y" "bienvenidas" "" "los"
## [61] "que" "están" "llegando" "seguimos"
## [65] "con" "" "nuestra" "dinámica"
## [69] "de" "charlas" "y" "talleres"
## [73] "" "en" "esta" "ocasión"
## [77] "pues" "ya" "vieron" "tenemos"
## [81] "a" "" "nuestro" "invitado"
## [85] "" "raúl" "que" "nos"
## [89] "va" "a" "platicar" "sobre"
## [93] "pitch" "" "muchas" "gracias"
## [97] "raúl" "por" "acompañarnos" "les"Limpieza de palabras
# lower case
vcsUNPrfWords <- str_to_lower(vcsUNPrfWords)
# remove numbers
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:digit:]]", "")
# remove punctuation
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:punct:]]", "")
# remove white spaces
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:space:]]", "")
# remove special chars
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[~@#$%&-_=<>]", "")
# remove empty vectors
vcsUNPrfWords <- vcsUNPrfWords[vcsUNPrfWords != ""]
# hack & remove $
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="$", "")
# head
head(vcsUNPrfWords,100)## [1] "solo" "aqui" "probando" "algo"
## [5] "hola" "guillermo" "y" "paola"
## [9] "buenas" "tardes" "estamos" "dando"
## [13] "unos" "minutos" "a" "que"
## [17] "lleguen" "los" "demas" "participantes"
## [21] "para" "arrancar" "gracias" "a"
## [25] "todos" "los" "que" "se"
## [29] "estan" "uniendo" "vamos" "a"
## [33] "dar" "dos" "minutos" "mas"
## [37] "para" "que" "lleguen" "los"
## [41] "demas" "compaã±eros" "y" "arrancamos"
## [45] "muy" "bien" "pues" "bienvenidos"
## [49] "y" "bienvenidas" "los" "que"
## [53] "estãn" "llegando" "seguimos" "con"
## [57] "nuestra" "dinãmica" "de" "charlas"
## [61] "y" "talleres" "en" "esta"
## [65] "ocasiã³n" "pues" "ya" "vieron"
## [69] "tenemos" "a" "nuestro" "invitado"
## [73] "raãºl" "que" "nos" "va"
## [77] "a" "platicar" "sobre" "pitch"
## [81] "muchas" "gracias" "raãºl" "por"
## [85] "acompaã±arnos" "les" "cuento" "un"
## [89] "poco" "de" "ã©l" "raãºl"
## [93] "es" "licenciado" "en" "administraciã³n"
## [97] "con" "maestrãa" "en" "marketing"Data frame con palabras normales
# make data frame
dfrUNPrfWords <- data.frame(vcsUNPrfWords)
colnames(dfrUNPrfWords) <- c("Words")
dfrUNPrfWords$Words <- as.character(dfrUNPrfWords$Words)
# normal word count
head(dfrUNPrfWords,10)## Words
## 1 solo
## 2 aqui
## 3 probando
## 4 algo
## 5 hola
## 6 guillermo
## 7 y
## 8 paola
## 9 buenas
## 10 tardesConteo de palabras “normales”
# resumiendo los datos
dfrUNPrfFreq <- dfrUNPrfWords %>%
group_by(Words) %>%
summarise(Freq=n()) %>%
arrange(desc(Freq))
head(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 que 383
## 2 de 236
## 3 a 183
## 4 y 166
## 5 es 161
## 6 pues 128Primera nube de palabras normales
# nube de palabras
wordcloud(dfrUNPrfFreq$Words[1:100], dfrUNPrfFreq$Freq[1:100], random.order=F, max.words=100, colors=brewer.pal(8, "Dark2"))
Data frame de palabras realmente significativas
En esta sección vamos a quitar las “stop words”
# significant words only
# remove all words with len <= 2
dfrUNPrfWords <- filter(dfrUNPrfWords, str_length(Words)>2)
# remover las "stop words" o palabras comunes como conjunciones
vcsCmnWords <- c("de","que","en","y","la","a","el","es","una","un","pues","no","para","los","se","las","como","con","más","por","lo","hay","del","o","entonces","este","está","nos","pero","también","creo","porque","también","yo","ya","esta","si","me","al","son","tiene","donde","bueno","ha","sobre","ejemplo","bien","gracias","ser","eso","todo","uso","ver","tener","esto","estos","muchas","cómo","cuando","sea","tenemos","su","tienen","así","desde","han","parte","ahí","les","tal","qué","estar")
dfrUNPrfWords <- filter(dfrUNPrfWords, !(Words %in% vcsCmnWords))
# remover las palabras no significativas para este contexto
vcsBadWords <- c("decir","muy","están")
dfrUNPrfWords <- filter(dfrUNPrfWords, !(Words %in% vcsBadWords))
# show
head(dfrUNPrfWords)## Words
## 1 solo
## 2 aqui
## 3 probando
## 4 algo
## 5 hola
## 6 guillermoConteo de palabras significativas
dfrUNPrfFreq <- dfrUNPrfWords %>%
group_by(Words) %>%
summarise(Freq=n()) %>%
arrange(desc(Freq))
head(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 ustedes 72
## 2 mãs 49
## 3 hacer 30
## 4 pitch 29
## 5 vamos 29
## 6 supuesto 25“cola” de palabras significativas
tail(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 volar 1
## 2 volverlo 1
## 3 vuelve 1
## 4 week 1
## 5 wild 1
## 6 zoom 1Eliminar palabras dispersas
# palabras con una frecuencia absoluta menor a 5
dfrUNPrfFreq <- filter(dfrUNPrfFreq, Freq>5)
tail(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 mil 6
## 2 nada 6
## 3 objetivo 6
## 4 realmente 6
## 5 sin 6
## 6 vas 6Conteo final de palabras
# total word count = length of vector
intWordCountFinal <- length(dfrUNPrfFreq$Words)
# print
intWordCountFinal## [1] 101Categorización por frecuencias
# add FrequencyCategory colum
dfrUNPrfFreq <- mutate(dfrUNPrfFreq, Fcat=FreqCategory(dfrUNPrfFreq$Freq))
# new data frame for Frequency Of Categorized Frequencies ...
dfrUNPrfFocf <- dfrUNPrfFreq %>% group_by(Fcat) %>% summarise(Rfrq=n())
#
dfrUNPrfFocf$Fcat <- factor(dfrUNPrfFocf$Fcat, levels=dfrUNPrfFocf$Fcat, ordered=T)
# head
head(dfrUNPrfFocf,10)## # A tibble: 4 x 2
## Fcat Rfrq
## <ord> <int>
## 1 " 10" 62
## 2 " 20" 29
## 3 " 50" 9
## 4 " 100" 1Nueva nube de palabras
wordcloud(dfrUNPrfFreq$Words[1:50], dfrUNPrfFreq$Freq[1:50], random.order=F, max.words=100, colors=brewer.pal(8, "Dark2"))
## gráfica de barras de palabras
ggplot(slice(dfrUNPrfFreq,1:30), aes(x=reorder(Words,-Freq),y=Freq)) +
geom_bar(stat="identity", fill=rainbow(30)) +
ylab("Frequency") +
xlab("Words") +
ggtitle("Primeras 30 palabras con mayor frecuencia") +
theme(plot.title=element_text(size=rel(1.5), colour="blue")) +
coord_flip()
## Gráfica de frecuencia
ggplot(dfrUNPrfFocf, aes(Fcat,Rfrq))+
geom_bar(stat="identity", width=0.8, fill=rainbow(length(dfrUNPrfFocf$Fcat))) +
xlab("Words With Frequency Less Than") + ylab("Frequency") +
theme(axis.text.x=element_text(angle=60, hjust=1, vjust=1),axis.text.y=element_text(angle=60, hjust=1, vjust=1),plot.title=element_text(size=rel(1.5), colour="blue")) +
ggtitle("Frequency Of Word Count")
Longitud de palabras
dfrUNPrfChrs <- data.frame(Chars=nchar(dfrUNPrfFreq$Words))
#intRowCount <- nrow(table(dfrUNPrfChrs))
ggplot(dfrUNPrfChrs, aes(x=Chars)) +
geom_histogram(binwidth=1, fill='blue') +
geom_vline(xintercept=mean(nchar(dfrUNPrfFreq$Words)), color='black', size=1.5, alpha=.5) +
xlab("Word Length (Chars)") + ylab("Number Of Words (Frequency)")