library(tidyjson)
library(magrittr)
library(jsonlite)
library(dplyr)Each classification is an array. Depending on the workflow and how it’s changed, classification arrays may vary in structure within a single project. Also, empty arrays seem to be problematic. Depending on the type of project, you probably want to split the data into workflows and even limit workflow version prior to flattening.
sas <- read.csv("../data/questions-SAS-1000.csv", stringsAsFactors = F)
kitteh <- read.csv("../data/kitteh-zoo-classifications.csv", stringsAsFactors = F)
wilde <- read.csv("../data/points-wildebeest.csv", stringsAsFactors = F)
chicago <- read.csv("../data/chicago-wildlife-watch-classifications.csv", stringsAsFactors = F)sas$annotations[1] %>% prettify[
{
"task": "init",
"task_label": "Are there whales or dolphins in this photo?",
"value": "No"
}
]
wilde$annotations[2] %>% prettify()[
{
"task": "T1",
"task_label": "Mark every wildebeest with the **RED CIRCLE**. Remove unwanted marks with the **X** in the corner of the circle.",
"value": [
{
"x": 3913.710798235543,
"y": 2106.318257956449,
"tool": 0,
"frame": 0,
"details": [
],
"tool_label": "Wildebeest Marker"
},
{
"x": 2803.6400990996435,
"y": 2469.2312010502965,
"tool": 0,
"frame": 0,
"details": [
],
"tool_label": "Wildebeest Marker"
},
{
"x": 2867.6826394344066,
"y": 2163.2457784417584,
"tool": 0,
"frame": 0,
"details": [
],
"tool_label": "Wildebeest Marker"
},
{
"x": 1586.8318327391382,
"y": 1956.8835166825118,
"tool": 0,
"frame": 0,
"details": [
],
"tool_label": "Wildebeest Marker"
},
{
"x": 1437.3992386246903,
"y": 1778.9850151659198,
"tool": 0,
"frame": 0,
"details": [
],
"tool_label": "Wildebeest Marker"
}
]
}
]
kitteh$annotations[1] %>% prettify[
{
"task": "init",
"task_label": "How many cats are there in this image?",
"value": "1"
},
{
"task": "T1",
"task_label": "Mark each cat's face and tail. Draw an ellipse around each cat's face (not including the ears) and mark the tail tip with a point.",
"value": [
]
}
]
It’s much easier to parse/flatten the JSON when everything is in a standard format, so you probably want to split out your raw file based on the Workflow and even Task IDs. You also want to limit to only the workflow version(s) with actual data. This is because previous versions, especially those with empty data, may have different structures for the classification data, which is annoying and problematic.
Note: you may need to dig into your raw data a bit to identify which workflow and version you need. Some projects have many workflows and versions, others not so many.
fun_check_workflow <- function(data){
data %>% group_by(workflow_id, workflow_version) %>%
summarise(date = max(created_at), count = n()) %>%
print
}For example: This is the Snapshots at Sea classifications by workflow
sas %>% fun_check_workflow()vs. that of Wildebeest Marking Project
wilde %>% fun_check_workflow()Vs. Chicago Wildlife Watch
chicago %>% fun_check_workflow()With jsonlite, you can basically flatten all of the json data into a series of nested lists. This works really well for simple data, like questions, but marking tasks and more complex workflows get a bit complicated.
library(jsonlite)
#Basic Flattening Function
basic_flattening <- function(jdata) {
out <- list() #create list to hold everything
for (i in 1:dim(jdata)[1]) { #loop through each row of the dataset at a time
classification_id <- jdata$classification_id[i]
subject_id <- jdata$subject_ids[i]
split_anno <- fromJSON(txt = jdata$annotations[i], simplifyDataFrame = T)
out[[i]] <- cbind(classification_id, subject_id, split_anno)
}
do.call(what = rbind, args = out)
}Single questions flatten alright
flat_sas <- sas %>% basic_flattening()
str(flat_sas)'data.frame': 999 obs. of 5 variables:
$ classification_id: int 2896902 2896910 2896915 2896917 2896923 2896934 2896940 2896945 2896950 2896958 ...
$ subject_id : int 920838 920143 920327 920336 920414 919869 920709 920171 920992 920875 ...
$ task : chr "init" "init" "init" "init" ...
$ task_label : chr "Are there whales or dolphins in this photo?" "Are there whales or dolphins in this photo?" "Are there whales or dolphins in this photo?" "Are there whales or dolphins in this photo?" ...
$ value : chr "No" "Yes" "Yes" "No" ...But more complex questions produce embedded lists inside the “value” column.
flat_wilde <- wilde[1:10,] %>% basic_flattening()
str(flat_wilde, max.level = 2)'data.frame': 10 obs. of 5 variables:
$ classification_id: int 2001677 2001683 2001684 2002042 2004527 3866661 3866666 3867128 4440549 4440554
$ subject_id : int 483320 483321 483320 483321 483320 1042992 1042854 1042907 1042916 1042956
$ task : chr "T1" "T1" "T1" "T1" ...
$ task_label : chr "Mark every wildebeest with the *Red circle*. Removed unwanted marks with " "Mark every wildebeest with the **RED CIRCLE**. Remove unwanted marks with the **X** in the corner of the circle." "Mark every wildebeest with the **RED CIRCLE**. Remove unwanted marks with the **X** in the corner of the circle." "Mark every wildebeest with the **RED CIRCLE**. Remove unwanted marks with the **X** in the corner of the circle." ...
$ value :List of 10
..$ :'data.frame': 30 obs. of 6 variables:
..$ :'data.frame': 5 obs. of 6 variables:
..$ : list()
..$ :'data.frame': 22 obs. of 6 variables:
..$ : list()
..$ :'data.frame': 6 obs. of 6 variables:
..$ :'data.frame': 3 obs. of 6 variables:
..$ :'data.frame': 10 obs. of 6 variables:
..$ :'data.frame': 2 obs. of 6 variables:
..$ : list()flat_kitteh <- kitteh %>% basic_flattening()
str(flat_kitteh, max.level = 3)'data.frame': 6 obs. of 5 variables:
$ classification_id: int 2962830 2962830 2962838 2962838 9451458 9451458
$ subject_id : int 458021 458021 458022 458022 458041 458041
$ task : chr "init" "T1" "init" "T1" ...
$ task_label : chr "How many cats are there in this image?" "Mark each cat's face and tail. Draw an ellipse around each cat's face (not including the ears) and mark the tail tip with a poi"| __truncated__ "How many cats are there in this image?" "Mark each cat's face and tail. Draw an ellipse around each cat's face (not including the ears) and mark the tail tip with a poi"| __truncated__ ...
$ value :List of 6
..$ : chr "1"
..$ : chr
..$ : chr "1"
..$ : chr
..$ : chr "1"
..$ :'data.frame': 1 obs. of 9 variables:
.. ..$ x : num 326
.. ..$ y : int 137
.. ..$ rx : num 227
.. ..$ ry : num 114
.. ..$ tool : int 0
.. ..$ angle : num -39.6
.. ..$ frame : int 0
.. ..$ details :List of 1
.. ..$ tool_label: chr "Catface"