Scraping Berita Kemenag.com
Nama : Reza Arianti
NIM : G1501231039
library(tm)
library(dplyr)
library(wordcloud)
library(ggplot2)
library(lubridate)
library(tidytext)
library(tidyr)Langkah 1 Import Data
library(readr)
DATA <- read_csv("scraping_kemenag.csv")## Rows: 109 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (4): _id, titles, dates, links
## time (1): time_scraped
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.View(DATA)
glimpse(DATA)## Rows: 109
## Columns: 5
## $ `_id` <chr> "6656c5a705b4c9c5810bc931", "6656c5a705b4c9c5810bc932", "…
## $ time_scraped <time> 05:24:00, 05:24:00, 05:24:00, 05:24:00, 05:24:00, 05:24:…
## $ titles <chr> "Saktor Sektor Hulu, BPJPH Edukasi Sertifikasi Halal Jasa…
## $ dates <chr> "29 Mei 2024", "29 Mei 2024", "29 Mei 2024", "29 Mei 2024…
## $ links <chr> "/nasional/saktor-sektor-hulu-bpjph-edukasi-sertifikasi-h…Langkah 2 Menghapus Stop Words
# Load the data
headlines_df <- read.csv("scraping_kemenag.csv",encoding = "ISO-8859-1")
# Ensure correct encoding of the text
headlines_df$titles <- iconv(headlines_df$titles, from = "ISO-8859-1", to = "UTF-8")
# Create a text corpus
corpus <- Corpus(VectorSource(headlines_df$titles))
# Define custom stopwords for Indonesian
stopwords_id <- c("yang", "dan", "di", "ke", "dari", "ini", "itu", "untuk", "dengan", "pada", "adalah", "sebagai", "juga", "dalam", "tidak", "akan", "atau", "saya", "kami", "kita", "mereka", "anda", "ia")
# Clean the text
corpus <- corpus %>%
tm_map(content_transformer(tolower)) %>%
tm_map(removePunctuation) %>%
tm_map(removeNumbers) %>%
tm_map(removeWords, stopwords_id) %>%
tm_map(stripWhitespace)## Warning in tm_map.SimpleCorpus(., content_transformer(tolower)): transformation
## drops documents## Warning in tm_map.SimpleCorpus(., removePunctuation): transformation drops
## documents## Warning in tm_map.SimpleCorpus(., removeNumbers): transformation drops
## documents## Warning in tm_map.SimpleCorpus(., removeWords, stopwords_id): transformation
## drops documents## Warning in tm_map.SimpleCorpus(., stripWhitespace): transformation drops
## documents# Create a document-term matrix
dtm <- DocumentTermMatrix(corpus)
# Convert to matrix and get word frequencies
word_freq <- sort(colSums(as.matrix(dtm)), decreasing = TRUE)
word_freq_df <- data.frame(word = names(word_freq), freq = word_freq)
# Display the top 10 most frequent words
head(word_freq_df, 10)## word freq
## haji haji 34
## jemaah jemaah 29
## kemenag kemenag 20
## petugas petugas 9
## visa visa 9
## program program 8
## lansia lansia 7
## menag menag 7
## guru guru 6
## saudi saudi 6Langkah 3 Membuat Word Cloud
# Generate the word cloud
set.seed(1234)
wordcloud(words = word_freq_df$word, freq = word_freq_df$freq, min.freq = 1,
max.words = 90, random.order = FALSE, rot.per = 0.35,
colors = brewer.pal(8, "Dark2"))
The word cloud prominently features words like Hajj, Ministry of Religious Affairs (Kemenag), visa, officers, programs, BPJPH (Halal Product Assurance Agency), services,teachers, Saudi, competencies, Indonesia, religion, certification, and departure. This suggests a focus on topics related to religious affairs and services, including Hajj preparations, certification processes, and educational programs for teachers, particularly in Saudi Arabia.
Langkah 4 Membuat Bigrams
# Tokenize the titles into bigrams
bigrams <- DATA %>%
unnest_tokens(bigram, titles, token = "ngrams", n = 2)
# Count the bigrams
bigram_counts <- bigrams %>%
count(bigram, sort = TRUE)
# Display the top 10 bigrams
top_bigrams<-head(bigram_counts, 10)
# Define colors
colors <- c("lightblue", "#055266", "pink")
# Create a bar chart of the top 10 bigrams
ggplot(top_bigrams, aes(x = reorder(bigram, n), y = n, fill = as.factor(n))) +
geom_bar(stat = "identity", color = "gold") +
coord_flip() +
scale_fill_manual(values = rep(colors, length.out = 10)) +
labs(title = "Top 10 Bigrams in Titles",
x = "Bigram",
y = "Count") +
theme_minimal() +
theme(plot.background = element_rect(fill = "darkgreen"),
legend.position = "none", # Hide the legend
axis.text.y = element_text(color = "white",size = 12), # Color the y-axis text (bigram names) orange
plot.title = element_text(color = "yellow", size = 16, face = "bold"), # Color the title pink
axis.title.x = element_text(color = "yellow",face = "bold"), # Color the x-axis title pink
axis.title.y = element_text(color = "yellow",face = "bold"), # Color the x-axis title pink
axis.text.x = element_text(color = "darkorange")) # Color the x-axis text (numbers) pink
The bigrams on the right side indicate pairs of words that frequently co-occur in the analyzed text. These include combinations like “jemaah haji”, “petugas haji”, “sertifikasi halal”, etc. Such combinations highlight specific themes and activities related to visa services for Hajj, competency programs for teachers, and halal certification processes managed by BPJPH
Langkah 5 Membuat Trigrams
# Load necessary libraries
library(dplyr)
library(tidytext)
library(ggplot2)
# Assume dataku is your dataframe containing the 'titles' column
# Tokenize the titles into trigrams
trigrams <- DATA %>%
unnest_tokens(trigram, titles, token = "ngrams", n = 3)
# Count the trigrams
trigram_counts <- trigrams %>%
count(trigram, sort = TRUE)
# Display the top 10 trigrams
top_trigrams <- head(trigram_counts, 10)
# Define colors
colors <- c("darkblue", "#055266", "darkred")
# Create a bar chart of the top 10 bigrams
ggplot(top_bigrams, aes(x = reorder(bigram, n), y = n, fill = as.factor(n))) +
geom_bar(stat = "identity", color = "gold") +
coord_flip() +
scale_fill_manual(values = rep(colors, length.out = 10)) +
labs(title = "Top 10 Trigrams in Titles",
x = "Bigram",
y = "Count") +
theme_minimal() +
theme(plot.background = element_rect(fill = "darkgreen"),
legend.position = "none", # Hide the legend
axis.text.y = element_text(color = "white",size = 12), # Color the y-axis text (bigram names) orange
plot.title = element_text(color = "lightpink", size = 16, face = "bold"), # Color the title pink
axis.title.x = element_text(color = "lightpink",face = "bold"), # Color the x-axis title pink
axis.title.y = element_text(color = "lightpink",face = "bold"), # Color the x-axis title pink
axis.text.x = element_text(color = "darkorange")) # Color the x-axis text (numbers) pink
The trigrams on the right side likely represent specific combinations of words that frequently appear together in the analyzed text. These combinations include phrases such as “jemaah haji”, “petuga haji” and “sertifikasi halal” (BPJPH halal certification), etc. Each trigram captures meaningful associations within the dataset,highlighting key topics and activities related to religious services, educational programs, and certification processes
Langkah 5 Membuat Network Plot
library(dplyr)
library(tidytext)
library(igraph)##
## Attaching package: 'igraph'## The following object is masked from 'package:tidyr':
##
## crossing## The following objects are masked from 'package:lubridate':
##
## %--%, union## The following objects are masked from 'package:dplyr':
##
## as_data_frame, groups, union## The following objects are masked from 'package:stats':
##
## decompose, spectrum## The following object is masked from 'package:base':
##
## unionlibrary(ggraph)
# Load the data
headlines_df <- read.csv("scraping_kemenag.csv", encoding = "ISO-8859-1")
# Ensure correct encoding of the text
headlines_df$title <- iconv(headlines_df$title, from = "ISO-8859-1", to = "UTF-8")
# Tokenize the titles into bigrams
bigrams <- headlines_df %>%
unnest_tokens(bigram, title, token = "ngrams", n = 2)
# Count the bigrams
bigram_counts <- bigrams %>%
count(bigram, sort = TRUE) %>%
filter(n > 1) # Filter out bigrams that appear only once
# Separate the bigrams into two columns
bigram_counts <- bigram_counts %>%
separate(bigram, into = c("word1", "word2"), sep = " ")
# Create a graph from the bigrams
bigram_graph <- graph_from_data_frame(bigram_counts)
# Plot the network graph
set.seed(1234)
ggraph(bigram_graph, layout = "fr") +
geom_edge_link(aes(edge_alpha = n, edge_width = n), color = "#39A23A", show.legend = FALSE) +
geom_node_point(color = "yellow", size = 5) +
geom_node_text(aes(label = name), vjust = 1, hjust = 1) +
theme_void() +
labs(title = "Network Plot of Bigrams in Article Titles")
A network plot of bigrams visually represents relationships between pairs of words that frequently appear together in the analyzed text. Nodes represent individual words, and edges (lines between nodes) indicate co-occurrences of these words as bigrams. The size of each node typically reflects its degree of connectivity or frequency in the text, while edge thickness or color intensity may denote the strength or frequency of co-occurrence between connected nodes. Such visualizations help identify key word associations and thematic clusters within the dataset, offering insights into prevalent topics or recurring phrases