Scrape Surveys

Load Necessary Packages

#install.packages(c("rvest", "dplyr"))
library(rvest)
library(dplyr)

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

Read data from the Webpage for my Assigned Biosample

namePage <- paste0(
  "https://www.ncbi.nlm.nih.gov/biosample/?term=", 18169573)

testPage <- read_html(namePage)

tableText <- testPage %>% 
    html_node("table") %>%
    html_table()

names(tableText)<-c('Question','Response')
head(tableText)

## # A tibble: 6 × 2
##   Question            Response         
##   <chr>               <chr>            
## 1 body habitat        UBERON:feces     
## 2 body product        UBERON:feces     
## 3 tissue              UBERON:feces     
## 4 geographic location United Kingdom   
## 5 diet                not provided     
## 6 dominant hand       I am right handed

Scrape Multiple Biosamples into a List and Give Meaningful Names

sampleIDs <- 31280770:31280775
survey <- list()

# Loop for each of the biosamples
for (i in 1:length(sampleIDs)) {

  namePage <- paste0(
    "https://www.ncbi.nlm.nih.gov/biosample/?term=",
    sampleIDs[i]
  )

  testPage <- read_html(namePage)

  tableText <- testPage %>%
    html_node("table") %>%
    html_table()

  names(tableText) <- c("Question", "Response")

  survey[[i]] <- tableText
}

Summarize the Responses from a Single Question Present in all Samples with a Pie Chart

Picking a Question

commonQuestions <- Reduce(
    intersect,
    lapply(survey, function(x) x$Question)
)
commonQuestions

Creating a Pie Chart

responses <- c()

for (i in 1:length(survey)) {

  answer <- survey[[i]] %>%
    filter(Question == "flossing_frequency") %>%
    pull(Response)

  responses <- c(responses, answer)
}

responseCounts <- table(responses)

pie(
  responseCounts,
  main = "Flossing Frequency of American Gut Project Participants"
)