--- title: Checking annotation status subtitle: Bird song evolution author: <a href="https://marce10.github.io/">Marcelo Araya-Salas, PhD</a> date: "`r Sys.Date()`" toc: true toc-depth: 2 toc-location: left number-sections: true highlight-style: pygments format: html: df-print: kable code-fold: true code-tools: true code-copy: true embed-resources: true editor_options: chunk_output_type: console --- <!-- this code add line numbers to code blocks --> <!-- only works when code folding is not used in yaml (code_folding: show) --> ```{r set root directory, echo = FALSE} # set working directory :: opts_knit$ set (root.dir = ".." )``` ```{r add link to github repo, echo = FALSE, results='asis'} # print link to github repo if any if (file.exists ("./.git/config" )){<- readLines ("./.git/config" )<- grep ("url" , config, value = TRUE )<- gsub (" \\ turl = |.git$" , "" , url)cat (" \n Source code, data and annotation protocol found at [" , url, "](" , url, ")" , sep = "" )``` ```{r setup style, echo = FALSE, message = FALSE, warning=FALSE} # options to customize chunk outputs :: opts_chunk$ set (tidy.opts = list (width.cutoff = 65 ), tidy = TRUE ,message = FALSE ,warning = FALSE ``` <!-- skyblue box --> < div class = "alert alert-info" > # Purpose {.unnumbered .unlisted} - Double-check annotations</ div > # Load packages {.unnumbered .unlisted} ```{r load packages, eval = TRUE} # knitr is require for creating html/pdf/word reports # formatR is used for soft-wrapping code # install/ load packages :: load_packages (packages = c ("knitr" , "formatR" , "rprojroot" , "viridis" , "googlesheets4" , "warbleR" , "maRce10/Rraven" , "googledrive" , "maRce10/ohun" ))<- "/media/m/Expansion/Projects/Ongoing/bird_song_evolution/consolidated_sound_files/" # annotation_path <- "/home/m/Insync/marceloa27@gmail.com/Google Drive/bird_song_evolution/raven_selections/" <- "./data/processed/annotations/" # check annotations # warbleR_options(wav.path = "run/user/1000/gvfs/smb-share:server=cinnas.local,share=neurobiología/marcelo_araya/bird_song_evolution/audios/consolidated_sound_files") ``` # Consolidate sound files ```{r, eval = FALSE} <- consolidate (path = "/media/m/Campylopterus/bird_song_evolution/by_family" , dest.path = "/run/user/1000/gvfs/smb-share:server=cinnas.local,share=neurobiología/marcelo_araya/bird_song_evolution/consolidated_sound_files" , parallel = 10 )``` # Changes in annnotation files ```{r, read and compare snapshot, eval = TRUE} <- readRDS ("./data/processed/prev_raven_annotations_snapshot.RDS" )<- fileSnapshot (annotation_path, md5sum= TRUE , recursive = TRUE )<- changedFiles (before = prev_snapshot, after = new_snapshot)# changes$added # saveRDS(new_snapshot, "./data/processed/prev_raven_annotations_snapshot.RDS") ``` - `r length(changes$unchanged)` files unchanged - `r length(changes$added)` files added- `r length(changes$deleted)` files deleted- `r length(changes$changed)` files updated## New files by family ```{r} # if not readin from google docs :: gs4_deauth ()# googlesheets4::gs4_auth() <- read_sheet ('https://docs.google.com/spreadsheets/d/16ukhyf37hm13f1FXB2JQ-tCfHoxXy2qj_TX2xWvTVu8/edit#gid=148139271' )$ assigned_to[rec_data$ assigned_to == "NA" ] <- NA # remove data with no family<` <- rec_data[rec_data$ family != "NA" , ]# new files added <- as.data.frame (table (dirname (changes$ added)))names (new_file_fams) <- c ("Family" , "New files" )$ By <- sapply (new_file_fams$ Family, function (x) rec_data$ assigned_to[rec_data$ family == x][1 ])<- kableExtra:: kbl (row.names = FALSE ,escape = FALSE ,format = "html" ,digits = # # sub_fam_count_kbl <- # kableExtra::row_spec( # kable_input = sub_fam_count_kbl, # row = which(prop_analzyed == 1), # background = grDevices::adjustcolor("#6DCD59FF", alpha.f = 0.3) # ) <- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` # Descriptive stats ```{r descriptive stats, eval = FALSE} # prev_status <- readRDS("./data/processed/annotation_status_results.RDS") # names(prev_status) <- paste0("prev_", names(prev_status)) # # saveRDS(prev_status, "./data/processed/previous_annotation_status_results.RDS") <- imp_raven (path = annotation_path, all.data = TRUE , warbler.format = TRUE , name.from.file = TRUE , ext.case = "lower" , pb = TRUE , recursive = TRUE )# grep(".Table.1 (1)", anns$sound.files, fixed = TRUE, value = T) $ family <- dirname (anns$ selec.file)$ selec.file <- basename (anns$ selec.file)$ species <- gsub ("_" , " " , sapply (strsplit (anns$ selec.file, "-" ), "[" , 1 ))<- unique (c (anns$ selec.file, unlist (.Options$ Rraven)))<- data.frame (file = ann_files)$ species <- gsub ("_" , " " , sapply (strsplit (ann_files$ file, "-" ), "[" , 1 ))$ family <- sapply (ann_files$ species, function (x) anns$ family[anns$ species == x][1 ])<- anns[! duplicated (paste (anns$ sound.files, anns$ selec.file)), ]rownames (undup_anns) <- 1 : nrow (undup_anns)# get those in which the names inside the text file and in the txt file name don't match <- na.omit (undup_anns$ selec.file[! sapply (seq_len (nrow (undup_anns)), function (x) grepl (gsub (".wav" , "" , undup_anns$ sound.files[x]), undup_anns$ selec.file[x]))])# get those in which the name doesnt match the expected <- ann_files[! grepl ("Table.1.selections.txt" , ann_files$ file) | ann_files$ file %in% c (unlist (.Options$ Rraven), file_name_no_match), ]$ problem <- if (nrow (weird_files) > 0 ) "file name" else vector ()$ problem[weird_files$ file %in% unlist (.Options$ Rraven)] <- "Empty file" $ problem[weird_files$ file %in% file_name_no_match] <- "Species name in txt file name and Begin column dont match" $ problem[weird_files$ file %in% unlist (.Options$ Rraven)] <- "Empty file" # keep only those with "good" names <- ann_files[! ann_files$ file %in% weird_files$ file, ]# counts per family <- aggregate (species ~ family, data = rec_data, length)names (fam_count)[2 ] <- "total" $ annotated <- sapply (fam_count$ family, function (x)sum (ann_files$ family == x))$ prop.annotated <- round (fam_count$ annotated / fam_count$ total, 2 )$ assinged.to <- sapply (fam_count$ family, function (x)paste (unique (rec_data$ assigned_to[rec_data$ family == x]), collapse = "/" ))$ assinged.to[fam_count$ assinged.to == "NA" ] <- "not assigned" $ assinged.to <- sapply (weird_files$ family, function (x)paste (unique (rec_data$ assigned_to[rec_data$ family == x]), collapse = "/" )) # check_sound_files() <- anns[, c ("sound.files" , "selec" , "start" , "end" , "bottom.freq" , "top.freq" , "selec.file" , "family" , "species" , "element" , "song" )]<- anns[anns$ selec.file %in% ann_files$ file, ]# # # anns$family <- sapply(anns$species, function(x) rec_data$family[rec_data$species == x][1]) $ assinged.to <- sapply (anns$ f, function (x)paste (unique (rec_data$ assigned_to[rec_data$ family == x]), collapse = "/" ))all (ann_files$ species %in% rec_data$ species)<- check_sels (anns, parallel = 20 , pb= TRUE , fix.selec = TRUE , path = path_sound_files)<- info_sound_files (parallel = 20 , path = path_sound_files, skip.error = TRUE )$ species <- gsub ("_" , " " , sapply (strsplit (file_info$ sound.files, "-" ), "[" , 1 ))$ family <- sapply (file_info$ species, function (x) rec_data$ family[rec_data$ species == x][1 ])as.data.frame (table (cs$ check.res[cs$ check.res != "OK" ]))unique (cs$ sound.files[cs$ check.res == "sound file not found" ])# exp_raven(cs, path = "./data/processed", sound.file.path = "/media/m/Seagate Portable Drive/bird_song_recordings/consolidated_files",file.name = "combined_annotations_22-03-2023") saveRDS (list (cs = cs, anns = anns, rec_data = rec_data, ann_files = ann_files, weird_files = weird_files, fam_count = fam_count, file_info = file_info), "./data/processed/annotation_status_results.RDS" )``` ```{r} attach (readRDS ("./data/processed/annotation_status_results.RDS" ))``` - `r length(unique(anns$sound.files))` species from `r length(unique(ann_files$family))` families already annotated (`r paste0(round(nrow(ann_files) / nrow(rec_data), 2) * 100, "%")` of all available especies; `r round(sum(file_info$duration[file_info$sound.files %in% anns$sound.files]) / 3600, 2)` recording hours; `r nrow(anns)` annotations)- `r sum(fam_count$prop.annotated >= 0.9)` families with 90% of all available especies annotated (`r round(sum(fam_count$prop.annotated >= 0.9) / nrow(fam_count), 2) * 100` % of all families)- `r sum(!file_info$sound.files %in% anns$sound.files)` species have not been annotated (`r round(sum(!file_info$sound.files %in% anns$sound.files) / nrow(rec_data) *100, 2)` %; `r round(sum(file_info$duration[!file_info$sound.files %in% anns$sound.files]) / 3600, 0)` recording hours; `r round(92/ (351 + 92) * 100, 0)` % of the total recording hours)# Species per family ## Done `r sum(fam_count$prop.annotated == 1)` families, `r sum(fam_count$total[fam_count$prop.annotated == 1])` recordings, `r round((sum(file_info$duration[file_info$sound.files %in% anns$sound.files[anns$family %in% fam_count$family[fam_count$prop.annotated == 1]]])) / 3600, 2)` recording hours)```{r, eval = TRUE} <- fam_count[fam_count$ prop.annotated == 1 ,]<- sub_fam_count$ prop.annotated$ prop.annotated <- ifelse ($ prop.annotated > 0.9 ,:: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "green" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE ifelse ($ prop.annotated > 0.5 ,:: cell_spec ($ prop.annotated,"html" ,color = "black" ,background = "yellow" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE $ assinged.to <- ifelse ($ assinged.to == "not assigned" ,:: cell_spec ($ assinged.to,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec (sub_fam_count$ assinged.to, "html" , new_tab = TRUE )$ recordings <- :: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fbird%5Fsong%5Frecordings%2F" , sub_fam_count$ family), new_tab = TRUE )$ spectrograms <- ifelse (prop_analzyed > 0 ,:: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fannotated%5Fspectrograms%2F" , sub_fam_count$ family), new_tab = TRUE ), "" )<- kableExtra:: kbl (row.names = FALSE ,escape = FALSE ,format = "html" ,digits = # # sub_fam_count_kbl <- # kableExtra::row_spec( # kable_input = sub_fam_count_kbl, # row = which(prop_analzyed == 1), # background = grDevices::adjustcolor("#6DCD59FF", alpha.f = 0.3) # ) <- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` ## Almost done `r sum(fam_count$prop.annotated >= 0.9 & fam_count$prop.annotated < 1)` families, `r length(unique(anns$sound.files[anns$family %in% fam_count$family[fam_count$prop.annotated >= 0.9 & fam_count$prop.annotated < 1]]))` recordings, `r round((sum(file_info$duration[file_info$sound.files %in% anns$sound.files[anns$family %in% fam_count$family[fam_count$prop.annotated >= 0.9 & fam_count$prop.annotated < 1]]])) / 3600, 2)` recording hours)```{r, eval = TRUE} <- fam_count[fam_count$ prop.annotated >= 0.9 & fam_count$ prop.annotated < 1 ,]<- sub_fam_count$ prop.annotated$ prop.annotated <- ifelse ($ prop.annotated > 0.9 ,:: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "green" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE ifelse ($ prop.annotated > 0.5 ,:: cell_spec ($ prop.annotated,"html" ,color = "black" ,background = "yellow" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE $ assinged.to <- ifelse ($ assinged.to == "not assigned" ,:: cell_spec ($ assinged.to,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec (sub_fam_count$ assinged.to, "html" , new_tab = TRUE )$ recordings <- :: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fbird%5Fsong%5Frecordings%2F" , sub_fam_count$ family), new_tab = TRUE )$ spectrograms <- ifelse (prop_analzyed > 0 ,:: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fannotated%5Fspectrograms%2F" , sub_fam_count$ family), new_tab = TRUE ), "" )<- kableExtra:: kbl (row.names = FALSE ,escape = FALSE ,format = "html" ,digits = <- :: row_spec (kable_input = sub_fam_count_kbl,row = which (prop_analzyed == 1 ),background = grDevices:: adjustcolor ("#6DCD59FF" , alpha.f = 0.3 )<- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` ## Halfway `r sum(fam_count$prop.annotated >= 0.0001 & fam_count$prop.annotated < 0.9)` families, `r length(unique(anns$sound.files[anns$family %in% fam_count$family[fam_count$prop.annotated >= 0.0001 & fam_count$prop.annotated < 0.9]]))` recordings, `r round((sum(file_info$duration[file_info$sound.files %in% anns$sound.files[anns$family %in% fam_count$family[fam_count$prop.annotated >= 0.0001 & fam_count$prop.annotated < 0.9]]])) / 3600, 2)` recording hours)```{r, eval = TRUE} <- fam_count[fam_count$ prop.annotated >= 0.0001 & fam_count$ prop.annotated < 0.90 ,]<- sub_fam_count$ prop.annotated$ prop.annotated <- ifelse ($ prop.annotated > 0.9 ,:: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "green" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE ifelse ($ prop.annotated > 0.5 ,:: cell_spec ($ prop.annotated,"html" ,color = "black" ,background = "yellow" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE $ assinged.to <- ifelse ($ assinged.to == "not assigned" ,:: cell_spec ($ assinged.to,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec (sub_fam_count$ assinged.to, "html" , new_tab = TRUE )$ recordings <- :: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fbird%5Fsong%5Frecordings%2F" , sub_fam_count$ family), new_tab = TRUE )$ spectrograms <- ifelse (prop_analzyed > 0 ,:: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fannotated%5Fspectrograms%2F" , sub_fam_count$ family), new_tab = TRUE ), "" )<- kableExtra:: kbl (row.names = FALSE ,escape = FALSE ,format = "html" ,digits = <- :: row_spec (kable_input = sub_fam_count_kbl,row = which (prop_analzyed == 1 ),background = grDevices:: adjustcolor ("#6DCD59FF" , alpha.f = 0.3 )<- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` ## Missing `r sum(fam_count$prop.annotated == 0)` families)```{r, eval = TRUE} <- fam_count[fam_count$ prop.annotated == 0 ,]<- sub_fam_count$ prop.annotated$ prop.annotated <- ifelse ($ prop.annotated > 0.9 ,:: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "green" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE ifelse ($ prop.annotated > 0.5 ,:: cell_spec ($ prop.annotated,"html" ,color = "black" ,background = "yellow" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec ($ prop.annotated,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE $ assinged.to <- ifelse ($ assinged.to == "not assigned" ,:: cell_spec ($ assinged.to,"html" ,color = "white" ,background = "red" ,bold = TRUE ,font_size = 12 ,new_tab = TRUE :: cell_spec (sub_fam_count$ assinged.to, "html" , new_tab = TRUE )$ recordings <- :: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fbird%5Fsong%5Frecordings%2F" , sub_fam_count$ family), new_tab = TRUE )$ spectrograms <- ifelse (prop_analzyed > 0 ,:: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?ga=1&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments%2Fbird%5Fsong%5Fevolution%2Fannotated%5Fspectrograms%2F" , sub_fam_count$ family), new_tab = TRUE ), "" )<- kableExtra:: kbl (row.names = FALSE ,escape = FALSE ,format = "html" ,digits = <- :: row_spec (kable_input = sub_fam_count_kbl,row = which (prop_analzyed == 1 ),background = grDevices:: adjustcolor ("#6DCD59FF" , alpha.f = 0.3 )<- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` # Double-checking annotations - `r sum(weird_files$problem == "file name" | weird_files$problem == "Species name in txt file name and Begin column dont match")` weirdly named file(s) and `r sum(weird_files$problem == "Empty file")` empty file(s):```{r} ``` ```{r, eval = TRUE} <- cs[cs$ check.res == "OK" , ]``` ```{r, eval = FALSE} # re run manually <- read.csv ("./data/processed/problematic_annotations.csv" )write.csv (prev_prob, "./data/processed/prev_problematic_annotations.csv" , row.names = FALSE )<- cs[cs$ check.res != "OK" , ]# weird duration or frequency range <- cs[cs$ end - cs$ start > 10 & ! cs$ family %in% c ("Tinamidae" , "Accipitridae" , "Falconidae" ), ] if (nrow (long_anns) > 0 )$ check.res <- "longer than 10 s" <- cs[cs$ top.freq - cs$ bottom.freq > 10 , ] $ check.res <- "frequency range wider than 10 kHz" <- cs[is.na (cs$ song), ] $ check.res <- "NAs in 'song'" <- cs[is.na (cs$ element), ] $ check.res <- "NAs in 'element'" <- rbind (problematic_sels, long_anns, broad_anns, song_na, elem_na)<- problematic_sels[, c ("sound.files" ,"family" , "assinged.to" , "selec" , "check.res" )]$ spectrograms <- "" names (problematic_sels) <- c ("sound.file" , "family" , "assinged.to" , "selection" , "problem" , "spectrograms" )<- problematic_sels[order (problematic_sels$ family, problematic_sels$ sound.file, problematic_sels$ selection), ]<- gsub (" \\ .wav" , "" , problematic_sels$ sound.file)<- gsub ("_" , "%5F" , sf)<- gsub ("-" , "%2D" , sf)<- problematic_sels[order (problematic_sels$ sound.file, problematic_sels$ selection), ]# find overlapping songs <- song_analysis (X = anns, parallel = 14 )<- check_sels (song_anns, parallel = 14 , fix.selec = TRUE , path = path_sound_files)# song_anns <- song_anns[song_anns$check.res == "OK",] <- song_anns[song_anns$ end - song_anns$ start > 0 , ]<- overlapping_sels (song_anns[song_anns$ check.res == "OK" & ! is.na (song_anns$ song), ], parallel = 14 )<- ovlp_songs[! is.na (ovlp_songs$ ovlp.sels), ]$ family <- sapply (ovlp_songs$ sound.files, function (x) anns$ family[anns$ sound.files == x][1 ])$ problem <- "overlapping songs" $ assinged.to <- sapply (ovlp_songs$ sound.files, function (x) anns$ assinged.to[anns$ sound.files == x][1 ])<- aggregate (song ~ sound.files + family + assinged.to + problem, data = ovlp_songs, unique)$ family <- sapply (ovlp_song_probs$ sound.file, function (x) anns$ family[anns$ sound.files == x][1 ])$ selections <- sapply (seq_len (nrow (ovlp_song_probs)), function (x) paste (na.omit (anns$ selec[anns$ sound.files == ovlp_songs$ sound.files[x] & anns$ song == ovlp_songs$ song[x]]), collapse = ", " ))<- aggregate (selection ~ sound.file + family + assinged.to + problem, data = problematic_sels, unique)$ sel.count <- sapply (agg_prob_sels$ sound.file, function (x) sum (problematic_sels$ sound.file == x))$ sel.count <- sapply (ovlp_song_probs$ sound.file, function (x) sum (ovlp_songs$ sound.files == x))names (agg_prob_sels)[5 ] <- "selections" $ song <- NULL names (ovlp_song_probs)[1 ] <- "sound.file" <- rbind (agg_prob_sels, ovlp_song_probs)<- data.frame (lapply (agg_prob_sels, as.character), stringsAsFactors= FALSE )$ fixed <- "" # Copy problematic to folder in google drive #### MAKE SURE SPECTROGRAMS EXISTS FOR ALL ANNOTATED FILES (if not run spectrogram creator chunk below) # search the jpeg image files that match the species name and copy those files to a new folder <- warbleR::: pblapply_wrblr_int (unique (agg_prob_sels$ sound.file), function (x){<- list.files (path = ann_spec_path, pattern = gsub (".wav" , "" , x), recursive = TRUE , full.names = TRUE )file.copy (from = jpegs, file.path ("/home/m/Insync/marceloa27@gmail.com/Google Drive/bird_song_evolution/annotated_spectrograms_problematic/" , basename (jpegs)), overwrite = TRUE )<- googledrive:: drive_ls (path = "bird_song_evolution/annotated_spectrograms_problematic/" )# image file page 1 # agg_prob_sels$jpeg_p1 <- gsub(".wav", "--p1.jpeg",agg_prob_sels$sound.file) # google drive link $ gd_id <- sapply (agg_prob_sels$ sound.file, function (x){as.vector (problm_image_drive$ id[grep (gsub (".wav" , "--p" , x), problm_image_drive$ name)])[1 ]$ gd_link <- ifelse (! is.na (agg_prob_sels$ gd_id), paste0 ("https://drive.google.com/file/d/" , agg_prob_sels$ gd_id, "/view?usp=drive_link" ), NA )write.csv (agg_prob_sels, "./data/processed/problematic_annotations.csv" , row.names = FALSE )``` ```{r, eval = TRUE} <- read.csv ("./data/processed/problematic_annotations.csv" )<- agg_prob_sels[grep ("not found|read" , agg_prob_sels$ problem, invert = TRUE ), ]$ fixed <- ifelse (is.na (agg_prob_sels$ fixed), "" , agg_prob_sels$ fixed)$ spectrograms <- ifelse (! is.na (agg_prob_sels$ gd_link), :: cell_spec ("link" , "html" , link = agg_prob_sels$ gd_link, new_tab = TRUE ), "" )$ recording <- :: cell_spec ("link" , "html" , link = paste0 ("https://6f33fa7f78ea46e2aaca-my.sharepoint.com/personal/marcelo_araya_ucr_ac_cr/_layouts/15/onedrive.aspx?q=" ,gsub ("_" , "%5F" , sapply (strsplit (agg_prob_sels$ sound.file, "-" ), "[" , 1 )), "&view=7&id=%2Fpersonal%2Fmarcelo%5Faraya%5Fucr%5Fac%5Fcr%2FDocuments&searchScope=folder" ), new_tab = TRUE )``` [ this data sheet ](https://docs.google.com/spreadsheets/d/1guDbKEVPB-scVQidTOVoPEn1YbXASBM7_cs6CpEYWUM/edit?usp=sharing) .## Daniela - `r length(unique(agg_prob_sels$sound.file[agg_prob_sels$assinged.to %in% c("Daniela", "Daniela/Carlos", "Daniela/Paula")]))` files```{r, eval = TRUE} <- agg_prob_sels[order (agg_prob_sels$ family, agg_prob_sels$ sound.file, agg_prob_sels$ problem), ]<- agg_prob_sels[agg_prob_sels$ problem != "frequency range wider than 10 kHz" ,]<- kableExtra:: kbl ($ assinged.to %in% c ("Daniela" , "Daniela/Carlos" , "Daniela/Paula" ), c ("sound.file" , "family" , "problem" , "selections" , "spectrograms" , "recording" )],row.names = FALSE ,escape = FALSE ,format = "html" ,digits = 3 <- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` ## Paula - `r length(unique(agg_prob_sels$sound.file[agg_prob_sels$assinged.to %in% c("Paula", "Carlos/Paula")]))` files```{r, eval = TRUE} <- kableExtra:: kbl ($ assinged.to %in% c ("Paula" , "Carlos/Paula" ), c ("sound.file" , "family" , "problem" , "selections" , "spectrograms" , "recording" )],row.names = FALSE ,escape = FALSE ,format = "html" ,digits = 3 <- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` ## Carlos - `r length(unique(agg_prob_sels$sound.file[agg_prob_sels$assinged.to %in% c("Carlos")]))` files```{r, eval = TRUE} <- kableExtra:: kbl ($ assinged.to %in% c ("Carlos" ), c ("sound.file" , "family" , "problem" , "selections" , "spectrograms" , "recording" )],row.names = FALSE ,escape = FALSE ,format = "html" ,digits = 3 <- :: kable_styling (bootstrap_options = c ("striped" , "hover" , "condensed" , "responsive" ),full_width = FALSE ,font_size = 12 ``` # Created annotated spectrograms ```{r, eval = FALSE} warbleR_options (wav.path = path_sound_files)source ("~/Dropbox/R_package_testing/warbleR/R/full_spectrograms.R" )source ("~/Dropbox/R_package_testing/warbleR/R/internal_functions.R" )<- "./data/processed/annotated_spectrograms" # create folders for each family for (i in unique (rec_data$ family))if (! file.exists (file.path (ann_spec_path, i)))dir.create (file.path (ann_spec_path, i))<- gsub (".Table.1.selections.txt" , ".wav" , basename (c (changes$ added, changes$ changed)))<- unique (c (new_and_changed_files, prev_prob$ sound.file))<- warbleR::: pblapply_wrblr_int (new_and_changed_files, pbar = TRUE , cl = 1 , function (x)<- cs[cs$ sound.files == x, ]<- range (c (sub_anns$ bottom.freq, sub_anns$ top.freq))1 ] <- frq_range[1 ] - 1 if (frq_range[1 ] < 0 ) frq_range[1 ] <- 0 2 ] <- frq_range[2 ] + ((frq_range[2 ] - frq_range[1 ]) / 3 )if (frq_range[2 ] < 6 ) frq_range[2 ] <- 6 if (frq_range[2 ] > 22.05 ) frq_range[2 ] <- 22.05 $ selec <- paste (sub_anns$ selec, sub_anns$ element, sep = "-" )print (file.path (ann_spec_path, sub_anns$ family[1 ], x)) unlink (list.files (path = file.path (ann_spec_path, sub_anns$ family[1 ]), pattern = gsub (".wav" , "" , x)))try (full_spectrograms (X = sub_anns, flim = frq_range, sxrow = 4 , rows = 10 , ovlp = 50 , collevels = seq (- 100 , 0 , 5 ), parallel = 1 , overwrite = TRUE , dest.path = file.path (ann_spec_path, sub_anns$ family[1 ]), song = "song" , fast.spec = TRUE , horizontal = TRUE , pb = FALSE , only.annotated = TRUE ), silent = TRUE )sum (! file.exists (file.path (ann_spec_path, cs$ family[! duplicated (cs$ sound.files)], gsub (".wav" , "--p" ,cs$ sound.files[! duplicated (cs$ sound.files)]))))``` # Add new recordings ```{r, eval = FALSE} <- readxl:: read_excel ("./data/raw/Marcelo Files - no Production cut.xlsx" )<- readxl:: read_excel ("./data/raw/Marcelo Files - Production cuts.xlsx" )<- new_data_cuts[! new_data_cuts$ ` Scientific Name ` %in% rec_data$ species,] nrow (new_data_cuts)<- new_data_no_cuts[! new_data_no_cuts$ ` Parent Species ` %in% rec_data$ species,] nrow (new_data_no_cuts)head (new_data_no_cuts)$ source <- "not publication cut" $ source <- "publication cut" <- read.csv ("./data/raw/NEW_Clements-Checklist-v2022-October-2022.csv" )<- clm[clm$ category == "species" , ]$ genus <- sapply (clm.sp$ scientific.name, function (x) strsplit (x, " " )[[1 ]][1 ])$ genus <- sapply (new_data_cuts$ ` Scientific Name ` , function (x) strsplit (x, " " )[[1 ]][1 ])$ family <- sapply (seq_len (nrow (new_data_cuts)), function (x){ fam <- clm.sp$ family[clm.sp$ genus == new_data_cuts$ genus[x]][1 ]<- if (length (fam) == 0 ) NA else strsplit (fam, " \\ (" )[[1 ]][1 ]return (fam)$ genus <- sapply (new_data_no_cuts$ ` Parent Species ` , function (x) strsplit (x, " " )[[1 ]][1 ])$ family <- sapply (seq_len (nrow (new_data_no_cuts)), function (x){ fam <- clm.sp$ family[clm.sp$ genus == new_data_no_cuts$ genus[x]][1 ]<- if (length (fam) == 0 ) NA else strsplit (fam, " \\ (" )[[1 ]][1 ]return (fam)$ Orginal.Scientific.Name <- new_data_cuts$ Scientific.Name <- new_data_cuts$ ` Scientific Name ` $ Orginal.Scientific.Name <- new_data_no_cuts$ ` Parent Species ` $ ` Scientific Name ` <- new_data_cuts$ ` Scientific Name ` <- NULL $ Scientific.Name <- new_data_no_cuts$ ` Parent Species ` $ file_url <- paste0 ("https://macaulaylibrary.org/asset/" , new_data_no_cuts$ ` ML Catalog Number ` )$ file_url <- paste0 ("https://macaulaylibrary.org/asset/" , new_data_cuts$ ` ML Catalog Number ` )$ species_ebird_url <- paste0 ("https://ebird.org/species/" , new_data_no_cuts$ ` eBird Species Code ` )$ species_ebird_url <- paste0 ("https://ebird.org/species/" , new_data_cuts$ SpeciesCode)$ ` Common Name ` <- new_data_cuts$ ` English Name ` $ Behaviors <- NA names (new_data_no_cuts)names (new_data_cuts)$ Recordist <- paste (new_data_cuts$ FirstName, new_data_cuts$ LastName)$ Year <- new_data_cuts$ Month <- new_data_cuts$ Day <- new_data_cuts$ Locality <- new_data_cuts$ ` Loc ID ` <- new_data_cuts$ ` Country State County ` <- new_data_cuts$ ` Taxon Category ` <- NA <- intersect (names (new_data_no_cuts), names (new_data_cuts))<- rbind (new_data_no_cuts[, common_colums], new_data_cuts[, common_colums])$ sound.files <- paste0 (gsub (" " , "_" , second_batch$ Scientific.Name), "-ML" , second_batch$ ` ML Catalog Number ` , ".wav" )# ohun::feature_acoustic_data(path = "~/Downloads/combined/") # warbleR::info_sound_files(path = "~/Downloads/combined/") # this code is not organized (is a mess!) <- list.files (path = "~/Downloads/combined/" )table (substr (fls, nchar (fls) - 3 , nchar (fls)))<- second_batch[second_batch$ ` ML Catalog Number ` %in% gsub (".m4a|.mp3|.wav" , "" , fls),]nrow (second_batch)<- fls[gsub (".m4a|.mp3|.wav" , "" , fls) %in% second_batch$ ` ML Catalog Number ` ]table (substr (sub_fls, nchar (sub_fls) - 3 , nchar (sub_fls)))sum (! second_batch$ ` ML Catalog Number ` %in% gsub (".m4a|.mp3|.wav" , "" , fls))mp3_2_wav (samp.rate = 44.1 , bit.depth = 16 , path = "~/Downloads/combined/" , overwrite = TRUE , dest.path = "~/Downloads/combined/" )# delete mp3s # setwd("~/Downloads/combined/") for (i in grep ("m4a$" , fls, value = TRUE ))<- paste0 ("ffmpeg -i " , i, " " , gsub ("m4a$" , "wav" , i))system (cll)write.csv (second_batch, "./data/raw/sound_files_and_extended_metadata_second_batch.csv" , row.names = FALSE )<- second_batch[, c ("family" , "Scientific.Name" , "species_ebird_url" , "ML Catalog Number" , "sound.files" , "Common Name" , "Orginal.Scientific.Name" , "Behaviors" , "file_url" )]write.csv (second_batch, "./data/raw/sound_files_metadata_second_batch.csv" , row.names = FALSE )fix_wavs (samp.rate = 44.1 , bit.depth = 16 , path = "~/Downloads/combined/" )<- file.rename (from = file.path ("~/Downloads/combined/converted_sound_files" , paste0 (second_batch$ ` ML Catalog Number ` , ".wav" )), to = file.path ("~/Downloads/combined/converted_sound_files" , second_batch$ sound.files))nrow (second_batch)all (fr)for (i in na.omit (unique (second_batch$ family))){print (i) if (! dir.exists (file.path ("/run/user/1000/gvfs/smb-share:server=cinnas.local,share=neurobiología/marcelo_araya/bird_song_evolution/by_family" , i)))dir.create (file.path ("/run/user/1000/gvfs/smb-share:server=cinnas.local,share=neurobiología/marcelo_araya/bird_song_evolution/by_family" , i))<- file.path ("~/Downloads/combined/converted_sound_files" , na.omit (second_batch$ sound.files[second_batch$ family == i]))<- file.path ("/run/user/1000/gvfs/smb-share:server=cinnas.local,share=neurobiología/marcelo_araya/bird_song_evolution/by_family" , i, na.omit (second_batch$ sound.files[second_batch$ family == i]))<- file.copy (frm, to = tu)``` <!-- light green box --> < div class = "alert alert-success" > # Takeaways {.unnumbered .unlisted} - Doing good progress</ div > <!-- '---' adds a gray vertical line --> <!-- add packages used, system details and versions --> # Session information {.unnumbered .unlisted} ```{r session info, echo=F} sessionInfo ()```