Paquetes.
library(pacman)
p_load("dplyr", "stringr", "ggplot2", "wordcloud","rmdformats","vembedr", "xfun")Análisis de conteo de palabras para conocer su frecuencia.
Video de youtube de la charla.
embed_url("https://www.youtube.com/watch?v=5G81FfEtf0Q")- Este es un ejercicio de la materia de estadística aplicada de la clase de las 9:00 a.m. (LMV) del departamento de matemáticas de ITSON.
Función.
FreqCategory <- function(value) {
strCategory <- ifelse(value <=5, " 5",
ifelse(value <=10, " 10",
ifelse(value <=20, " 20",
ifelse(value <=50, " 50",
ifelse(value <=100, " 100",
ifelse(value <=500, " 500",
ifelse(value <=1000, " 1,000",
">1,000")))))))
}Datos de texto.
setwd("~/EA9am")
cancer <- readLines("cancer.txt", encoding = "UTF-8")
head(cancer)## [1] "estamos en vivo"
## [2] ""
## [3] "muy buenas tardes a todos y todas mi"
## [4] ""
## [5] "nombre es lourdes velasco y soy parte"
## [6] ""Conteo de lineas (renglones).
#Longitud del vector.
intLineCount <- length(cancer)
intLineCount## [1] 2806Conteo de palabras por linea.
# separar
lstUNPrfLines <- str_split(cancer," ")
# palabras por linea
vciUNPrfWperL <- unlist(lapply(lstUNPrfLines, length))
# imprimir media de palabras por linea
mean(vciUNPrfWperL)## [1] 3.654669Conteo de palabras.
# deslistar para obtener un vector de palabras
vcsUNPrfWords <- unlist(lstUNPrfLines)
# recuento total de palabras = longitud del vector
intWordCount <- length(vcsUNPrfWords)
# imprimir
intWordCount## [1] 10255Mostrar palabras.
head(vcsUNPrfWords, 100)## [1] "estamos" "en" "vivo" ""
## [5] "muy" "buenas" "tardes" "a"
## [9] "todos" "y" "todas" "mi"
## [13] "" "nombre" "es" "lourdes"
## [17] "velasco" "y" "soy" "parte"
## [21] "" "del" "equipo" "de"
## [25] "hacerla" "calixto" "trabajando" ""
## [29] "en" "el" "componente" "de"
## [33] "emprendimiento" "" "muchas" "gracias"
## [37] "por" "acompañarnos" "a" "esta"
## [41] "" "charla" "sobre" "inteligencia"
## [45] "artificial" "les" "" "comparto"
## [49] "que" "al" "finalizar" "la"
## [53] "sesión" "" "tendremos" "unos"
## [57] "minutos" "para" "preguntas" "y"
## [61] "" "respuestas" "" "les"
## [65] "pedimos" "de" "favor" "las"
## [69] "escriban" "en" "la" ""
## [73] "parte" "de" "ley" "y"
## [77] "en" "la" "sección" "de"
## [81] "" "comentarios" "de" "la"
## [85] "transmisión" "en" "vivo" "de"
## [89] "" "facebook" "y" "bueno"
## [93] "pues" "antes" "de" "iniciar"
## [97] "" "queremos" "mostrarles" "un"Limpieza de palabras.
vcsUNPrfWords <- str_to_lower(vcsUNPrfWords)
# remove numbers
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:digit:]]", "")
# remove punctuation
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:punct:]]", "")
# remove white spaces
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[[:space:]]", "")
# remove special chars
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="[~@#$%&-_=<>]", "")
# remove empty vectors
vcsUNPrfWords <- vcsUNPrfWords[vcsUNPrfWords != ""]
# hack & remove $
vcsUNPrfWords <- str_replace_all(vcsUNPrfWords, pattern="$", "")
# head
head(vcsUNPrfWords,100)## [1] "estamos" "en" "vivo" "muy"
## [5] "buenas" "tardes" "a" "todos"
## [9] "y" "todas" "mi" "nombre"
## [13] "es" "lourdes" "velasco" "y"
## [17] "soy" "parte" "del" "equipo"
## [21] "de" "hacerla" "calixto" "trabajando"
## [25] "en" "el" "componente" "de"
## [29] "emprendimiento" "muchas" "gracias" "por"
## [33] "acompañarnos" "a" "esta" "charla"
## [37] "sobre" "inteligencia" "artificial" "les"
## [41] "comparto" "que" "al" "finalizar"
## [45] "la" "sesión" "tendremos" "unos"
## [49] "minutos" "para" "preguntas" "y"
## [53] "respuestas" "les" "pedimos" "de"
## [57] "favor" "las" "escriban" "en"
## [61] "la" "parte" "de" "ley"
## [65] "y" "en" "la" "sección"
## [69] "de" "comentarios" "de" "la"
## [73] "transmisión" "en" "vivo" "de"
## [77] "facebook" "y" "bueno" "pues"
## [81] "antes" "de" "iniciar" "queremos"
## [85] "mostrarles" "un" "vídeo" "sobre"
## [89] "la" "iniciativa" "hacerla" "jalisco"
## [93] "perla" "en" "jalisco" "es"
## [97] "una" "iniciativa" "liderada" "por"Data frame de palabras normales.
dfrUNPrfWords <- data.frame(vcsUNPrfWords)
colnames(dfrUNPrfWords) <- c("Words")
dfrUNPrfWords$Words <- as.character(dfrUNPrfWords$Words)
# normal word count
head(dfrUNPrfWords,10)## Words
## 1 estamos
## 2 en
## 3 vivo
## 4 muy
## 5 buenas
## 6 tardes
## 7 a
## 8 todos
## 9 y
## 10 todasConteo de palabras “normales”.
# resumiendo los datos
dfrUNPrfFreq <- dfrUNPrfWords %>%
group_by(Words) %>%
summarise(Freq=n()) %>%
arrange(desc(Freq))
head(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 de 499
## 2 que 420
## 3 a 311
## 4 la 229
## 5 es 220
## 6 y 199Nube de palabras normales.
wordcloud(dfrUNPrfFreq$Words[1:100], dfrUNPrfFreq$Freq[1:100], random.order=F, max.words=100, colors=brewer.pal(8, "Dark2"))
## Data frame de palabras realmente significantes.
# significant words only
# remove all words with len <= 2
dfrUNPrfWords <- filter(dfrUNPrfWords, str_length(Words)>2)
# remover las "stop words" o palabras comunes como conjunciones
vcsCmnWords <- c("de","que","en","y","la","a","el","es","una","un","pues","no","para","los","se","las","como","con","más","por","lo","hay","del","o","entonces","este","está","nos","pero","también","creo","porque","también","yo","ya","esta","si","me","al","son","tiene","donde","bueno","ha","sobre","ejemplo","bien","gracias","ser","eso","todo","uso","ver","tener","esto","estos","muchas","cómo","cuando","sea","tenemos","su","tienen","así","desde","han","parte","ahí","les","tal","qué","estar", "facebook", "live", "ustedes", "estamos", "vivo", "más", "van", "fue", "entre", "varios", "utilizar", "estas", "algo", "uno", "otros")
dfrUNPrfWords <- filter(dfrUNPrfWords, !(Words %in% vcsCmnWords))
# remover las palabras no significativas para este contexto
vcsBadWords <- c("decir","muy","están", "buenas", "tardes", "todos", "todas", "vamos", "nosotros", "algoritmo", "david", "muchos", "nuestro", "podemos", "queremos", "ahora" )
dfrUNPrfWords <- filter(dfrUNPrfWords, !(Words %in% vcsBadWords))
# show
head(dfrUNPrfWords)## Words
## 1 nombre
## 2 lourdes
## 3 velasco
## 4 soy
## 5 equipo
## 6 hacerlaConteo de palabras significativas.
dfrUNPrfFreq <- dfrUNPrfWords %>%
group_by(Words) %>%
summarise(Freq=n()) %>%
arrange(desc(Freq))
head(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 algoritmos 69
## 2 datos 53
## 3 cáncer 37
## 4 clase 36
## 5 segmentación 33
## 6 bancos 30“Cola” de palabras significaticas.
tail(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 wiki 1
## 2 wil 1
## 3 wisconsin 1
## 4 yahvé 1
## 5 zacatones 1
## 6 zona 1Eliminar palabras dispersas.
# palabras con una frecuencia absoluta menor a 5
dfrUNPrfFreq <- filter(dfrUNPrfFreq, Freq>5)
tail(dfrUNPrfFreq)## # A tibble: 6 x 2
## Words Freq
## <chr> <int>
## 1 optimización 6
## 2 original 6
## 3 píxeles 6
## 4 pregunta 6
## 5 presentan 6
## 6 primero 6Conteo final de palabras.
# total word count = length of vector
intWordCountFinal <- length(dfrUNPrfFreq$Words)
# print
intWordCountFinal## [1] 129Categorización por frecuencias.
# add FrequencyCategory colum
dfrUNPrfFreq <- mutate(dfrUNPrfFreq, Fcat=FreqCategory(dfrUNPrfFreq$Freq))
# new data frame for Frequency Of Categorized Frequencies ...
dfrUNPrfFocf <- dfrUNPrfFreq %>% group_by(Fcat) %>% summarise(Rfrq=n())
#
dfrUNPrfFocf$Fcat <- factor(dfrUNPrfFocf$Fcat, levels=dfrUNPrfFocf$Fcat, ordered=T)
# head
head(dfrUNPrfFocf,10)## # A tibble: 4 x 2
## Fcat Rfrq
## <ord> <int>
## 1 " 10" 85
## 2 " 20" 26
## 3 " 50" 16
## 4 " 100" 2Nube de palabras significativas.
wordcloud(dfrUNPrfFreq$Words[1:50], dfrUNPrfFreq$Freq[1:50], random.order=F, max.words=100, colors=brewer.pal(8, "Dark2"))
Gráfica de barras de palabras.
ggplot(slice(dfrUNPrfFreq,1:30), aes(x=reorder(Words,-Freq),y=Freq)) +
geom_bar(stat="identity", fill=rainbow(30)) +
ylab("Frequency") +
xlab("Words") +
ggtitle("Primeras 30 palabras con mayor frecuencia") +
theme(plot.title=element_text(size=rel(1.5), colour="blue")) +
coord_flip()
Gráfica de frecuencia.
ggplot(dfrUNPrfFocf, aes(Fcat,Rfrq))+
geom_bar(stat="identity", width=0.8, fill=rainbow(length(dfrUNPrfFocf$Fcat))) +
xlab("Words With Frequency Less Than") + ylab("Frequency") +
theme(axis.text.x=element_text(angle=60, hjust=1, vjust=1),axis.text.y=element_text(angle=60, hjust=1, vjust=1),plot.title=element_text(size=rel(1.5), colour="blue")) +
ggtitle("Frequency Of Word Count")
Longitud de palabras.
dfrUNPrfChrs <- data.frame(Chars=nchar(dfrUNPrfFreq$Words))
#intRowCount <- nrow(table(dfrUNPrfChrs))
ggplot(dfrUNPrfChrs, aes(x=Chars)) +
geom_histogram(binwidth=1, fill='blue') +
geom_vline(xintercept=mean(nchar(dfrUNPrfFreq$Words)), color='black', size=1.5, alpha=.5) +
xlab("Word Length (Chars)") + ylab("Number Of Words (Frequency)")