knitr::opts_chunk$set(eval = TRUE, include=TRUE, message = FALSE, warning = FALSE)This is my first project about web scraping. Web scraping is collecting data from websites and organizing them.
I decided to scrape the data on Municipal class because it might take time to request from PSA. Fortunately, the municipal class data is available in their website.
library(tidyverse)
library(rvest)
page_num <- 1:60
url <-"https://psa.gov.ph/classification/psgc/?q=psgc/municipalities&page="
page <- paste(url, page_num, sep="")
myfunc <- function(page){
page %>% read_html() %>%
html_nodes("#classifytable") %>%
html_table()
}
temp <- lapply(page[1:5], myfunc)
municipaldata <- bind_rows(temp)head(municipaldata)## # A tibble: 6 × 5
## Municipality `10 Digit Code` `Correspondence Code` `Income Class`
## <chr> <int> <int> <chr>
## 1 Adams 102801000 12801000 5th
## 2 Bacarra 102802000 12802000 3rd
## 3 Badoc 102803000 12803000 3rd
## 4 Bangui 102804000 12804000 4th
## 5 Burgos 102806000 12806000 5th
## 6 Carasi 102807000 12807000 5th
## # ℹ 1 more variable: `Population(2020)` <chr>Aside from the Municipal Class, I also need the latitude and longitude because I wanted to compute the distance of a school from the municipality.
page_num2 <- 1:76
url2 <- "https://geokeo.com/database/town/ph"
page2 <- paste(url2, page_num2, sep="/")
myfunc2 <- function(page){
page %>% read_html() %>%
html_node(".table-bordered") %>%
html_table()
}
temp2 <- lapply(page2[1:5], myfunc2)
municipal_latlong <- bind_rows(temp2)head(municipal_latlong)## # A tibble: 6 × 6
## `#` Name Country Latitude Longitude `Other Language Names`
## <int> <chr> <chr> <dbl> <dbl> <chr>
## 1 1 Aborlan Philippines 9.44 119. "\"name\"=>\"Aborlan\""
## 2 2 Abra de Ilog Philippines 13.4 121. "\"ref\"=>\"175101000\", \"…
## 3 3 Abucay Philippines 14.7 121. "\"name\"=>\"Abucay\""
## 4 4 Abulug Philippines 18.4 121. "\"ref\"=>\"021501000\", \"…
## 5 5 Abuyog Philippines 10.7 125. "\"name\"=>\"Abuyog\""
## 6 6 Adams Philippines 18.5 121. "\"name\"=>\"Adams\""library(qdapRegex)
library(stringi)
page3 <- c("https://www.philatlas.com/lists/physical/list-islands-100-or-more-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-26-to-99-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-11-to-25-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-5-pt1-to-10-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-3-pt1-to-5-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-2-pt1-to-3-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-1-pt6-to-2-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-1-pt3-to-1-pt5-sqmi.html",
"https://www.philatlas.com/lists/physical/list-islands-1-to-1-pt2-sqmi.html")
#lguTable
page3[1] %>% read_html() %>% html_node("#lguTable") %>% html_table()## # A tibble: 30 × 4
## `Island name` Approximate area (in sq …¹ Approximate area (in…² `Province(s)`
## <chr> <chr> <chr> <chr>
## 1 Luzon 42,411.69 109,846.27 "30 province…
## 2 Mindanao 36,860.30 95,468.17 "22 province…
## 3 Negros 5,041.57 13,057.68 "Negros Occi…
## 4 Samar 4,943.16 12,802.79 "Eastern Sam…
## 5 Panay 4,559.70 11,809.64 "Aklan, Anti…
## 6 Palawan 4,513.68 11,690.44 "Palawan"
## 7 Mindoro 3,912.72 10,133.94 "Occidental …
## 8 Leyte 2,770.63 7,175.93 "Leyte, Sout…
## 9 Cebu 1,739.76 4,505.97 "Cebu"
## 10 Bohol 1,485.49 3,847.43 "Bohol"
## # ℹ 20 more rows
## # ℹ abbreviated names: ¹`Approximate area (in sq mi)`,
## # ²`Approximate area (in sq km)`myfunc2 <- function(page){
page %>% read_html() %>%
html_node("#lguTable") %>%
html_table()
}
#steps uncessary DO NOT RUN
#contentpage <-"https://www.philatlas.com/physical/islands/tara.html"
#mytext <- contentpage %>% read_html() %>% html_node(".articleContent+ .articleContent") %>% html_text2()
#mytext %>% str_extract_all("(?<=approximately).+ (?=Elevation)") %>% unlist() %>%
# str_sub(start = 1, end = -3) %>% as.data.frame()
temp_islands <- lapply(page3[1:9], myfunc2)
island_data <- bind_rows(temp_islands[-1])
colnames(island_data) <- c("island_name", "sq_miles", "sq_km", "provinces")
#exclude first page because data types are different with the rest
#character vs numeric
#anyway first page is not necessary but we have to clean it
big_island_data <- bind_rows(temp_islands[1])
colnames(big_island_data) <- c("island_name", "sq_miles", "sq_km", "provinces")
big_island_data$sq_miles <- as.numeric(big_island_data$sq_miles)
big_island_data$sq_km <- as.numeric(big_island_data$sq_km)library(ggmap)
big_island_data## # A tibble: 30 × 4
## island_name sq_miles sq_km provinces
## <chr> <dbl> <dbl> <chr>
## 1 Luzon NA NA "30 provincesAbra\r\nAlbay\r\nApayao\r\nAurora\r\…
## 2 Mindanao NA NA "22 provincesAgusan del Norte\r\nAgusan del Sur\r…
## 3 Negros NA NA "Negros Occidental, Negros Oriental"
## 4 Samar NA NA "Eastern Samar, Northern Samar, Samar"
## 5 Panay NA NA "Aklan, Antique, Capiz, Iloilo"
## 6 Palawan NA NA "Palawan"
## 7 Mindoro NA NA "Occidental Mindoro, Oriental Mindoro"
## 8 Leyte NA NA "Leyte, Southern Leyte"
## 9 Cebu NA NA "Cebu"
## 10 Bohol NA NA "Bohol"
## # ℹ 20 more rowsisland_data## # A tibble: 298 × 4
## island_name sq_miles sq_km provinces
## <chr> <dbl> <dbl> <chr>
## 1 Samal 98.3 255. Davao del Norte
## 2 Camiguin 96.1 249. Camiguin
## 3 Panaon 80.0 207. Southern Leyte
## 4 Calayan 79.0 205. Cagayan
## 5 Lubang 78.1 202. Occidental Mindoro
## 6 Alabat 76.1 197. Quezon
## 7 Olutanga 69.9 181. Zamboanga Sibugay
## 8 Camiguin 69.1 179. Cagayan
## 9 Bucas Grande 49.2 128. Surigao del Norte
## 10 Bugsuk 47.9 124. Palawan
## # ℹ 288 more rowsall_island_data <- rbind(big_island_data, island_data) %>% mutate(address=paste(island_name, provinces, sep=", "))
geoloc <- geocode(all_island_data$address)
all_island_data <- cbind(all_island_data, geoloc)
write.csv(all_island_data, file = "all_island_data.csv")
tail (all_island_data)## island_name sq_miles sq_km provinces
## 323 Malobotlobot 1.02 2.64 Palawan
## 324 Buri 1.01 2.62 Samar
## 325 Talavera 1.01 2.61 Surigao del Norte
## 326 Capnoyan 1.00 2.60 Palawan
## 327 Patian 1.00 2.60 Sulu
## 328 Great Santa Cruz 1.00 2.60 Zamboanga del Sur
## address lon lat
## 323 Malobotlobot, Palawan 119.6797 11.499722
## 324 Buri, Samar 124.9748 11.579519
## 325 Talavera, Surigao del Norte 125.7000 9.750000
## 326 Capnoyan, Palawan 120.8987 10.734357
## 327 Patian, Sulu 121.0880 5.853901
## 328 Great Santa Cruz, Zamboanga del Sur 122.0639 6.866581When data merging for purposes of linking island dataset with administrative data, the big islands (rows 1-10) can be omitted.