CH5 HW
5.2.4 Exercises
Find all flights that:
Had an arrival delay of two or more hours
filter(flights, arr_delay >= 120)## # A tibble: 10,200 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 811 630 101 1047 830
## 2 2013 1 1 848 1835 853 1001 1950
## 3 2013 1 1 957 733 144 1056 853
## 4 2013 1 1 1114 900 134 1447 1222
## 5 2013 1 1 1505 1310 115 1638 1431
## 6 2013 1 1 1525 1340 105 1831 1626
## 7 2013 1 1 1549 1445 64 1912 1656
## 8 2013 1 1 1558 1359 119 1718 1515
## 9 2013 1 1 1732 1630 62 2028 1825
## 10 2013 1 1 1803 1620 103 2008 1750
## # … with 10,190 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Flew to Houston (IAH or HOU)
filter(flights, dest %in% c("IAH", "HOU"))## # A tibble: 9,313 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 517 515 2 830 819
## 2 2013 1 1 533 529 4 850 830
## 3 2013 1 1 623 627 -4 933 932
## 4 2013 1 1 728 732 -4 1041 1038
## 5 2013 1 1 739 739 0 1104 1038
## 6 2013 1 1 908 908 0 1228 1219
## 7 2013 1 1 1028 1026 2 1350 1339
## 8 2013 1 1 1044 1045 -1 1352 1351
## 9 2013 1 1 1114 900 134 1447 1222
## 10 2013 1 1 1205 1200 5 1503 1505
## # … with 9,303 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Were operated by United, American, or Delta
filter(flights, carrier %in% c("AA", "DL", "UA"))## # A tibble: 139,504 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 517 515 2 830 819
## 2 2013 1 1 533 529 4 850 830
## 3 2013 1 1 542 540 2 923 850
## 4 2013 1 1 554 600 -6 812 837
## 5 2013 1 1 554 558 -4 740 728
## 6 2013 1 1 558 600 -2 753 745
## 7 2013 1 1 558 600 -2 924 917
## 8 2013 1 1 558 600 -2 923 937
## 9 2013 1 1 559 600 -1 941 910
## 10 2013 1 1 559 600 -1 854 902
## # … with 139,494 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Departed in summer (July, August, and September)
filter(flights, month %in% 7:9)## # A tibble: 86,326 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 7 1 1 2029 212 236 2359
## 2 2013 7 1 2 2359 3 344 344
## 3 2013 7 1 29 2245 104 151 1
## 4 2013 7 1 43 2130 193 322 14
## 5 2013 7 1 44 2150 174 300 100
## 6 2013 7 1 46 2051 235 304 2358
## 7 2013 7 1 48 2001 287 308 2305
## 8 2013 7 1 58 2155 183 335 43
## 9 2013 7 1 100 2146 194 327 30
## 10 2013 7 1 100 2245 135 337 135
## # … with 86,316 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Arrived more than two hours late, but didn’t leave late
filter(flights, arr_delay > 120, dep_delay <= 0)## # A tibble: 29 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 27 1419 1420 -1 1754 1550
## 2 2013 10 7 1350 1350 0 1736 1526
## 3 2013 10 7 1357 1359 -2 1858 1654
## 4 2013 10 16 657 700 -3 1258 1056
## 5 2013 11 1 658 700 -2 1329 1015
## 6 2013 3 18 1844 1847 -3 39 2219
## 7 2013 4 17 1635 1640 -5 2049 1845
## 8 2013 4 18 558 600 -2 1149 850
## 9 2013 4 18 655 700 -5 1213 950
## 10 2013 5 22 1827 1830 -3 2217 2010
## # … with 19 more rows, and 11 more variables: arr_delay <dbl>, carrier <chr>,
## # flight <int>, tailnum <chr>, origin <chr>, dest <chr>, air_time <dbl>,
## # distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Were delayed by at least an hour, but made up over 30 minutes in flight
filter(flights, dep_delay >= 60, (dep_delay - arr_delay > 30))## # A tibble: 1,844 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 2205 1720 285 46 2040
## 2 2013 1 1 2326 2130 116 131 18
## 3 2013 1 3 1503 1221 162 1803 1555
## 4 2013 1 3 1839 1700 99 2056 1950
## 5 2013 1 3 1850 1745 65 2148 2120
## 6 2013 1 3 1941 1759 102 2246 2139
## 7 2013 1 3 1950 1845 65 2228 2227
## 8 2013 1 3 2015 1915 60 2135 2111
## 9 2013 1 3 2257 2000 177 45 2224
## 10 2013 1 4 1917 1700 137 2135 1950
## # … with 1,834 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Departed between midnight and 6am (inclusive)
filter(flights, dep_time >= 2400 | dep_time <= 600)## # A tibble: 9,373 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 517 515 2 830 819
## 2 2013 1 1 533 529 4 850 830
## 3 2013 1 1 542 540 2 923 850
## 4 2013 1 1 544 545 -1 1004 1022
## 5 2013 1 1 554 600 -6 812 837
## 6 2013 1 1 554 558 -4 740 728
## 7 2013 1 1 555 600 -5 913 854
## 8 2013 1 1 557 600 -3 709 723
## 9 2013 1 1 557 600 -3 838 846
## 10 2013 1 1 558 600 -2 753 745
## # … with 9,363 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Another useful dplyr filtering helper is between(). What does it do? Can you use it to simplify the code needed to answer the previous challenges?
filter(flights, between(dep_time, 601, 2359))## # A tibble: 319,148 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 1 601 600 1 844 850
## 2 2013 1 1 602 610 -8 812 820
## 3 2013 1 1 602 605 -3 821 805
## 4 2013 1 1 606 610 -4 858 910
## 5 2013 1 1 606 610 -4 837 845
## 6 2013 1 1 607 607 0 858 915
## 7 2013 1 1 608 600 8 807 735
## 8 2013 1 1 611 600 11 945 931
## 9 2013 1 1 613 610 3 925 921
## 10 2013 1 1 615 615 0 1039 1100
## # … with 319,138 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>How many flights have a missing dep_time? What other variables are missing? What might these rows represent?
sum(is.na(flights$dep_time))## [1] 8255map_dbl(flights, ~ sum(is.na(.x)))## year month day dep_time sched_dep_time
## 0 0 0 8255 0
## dep_delay arr_time sched_arr_time arr_delay carrier
## 8255 8713 0 9430 0
## flight tailnum origin dest air_time
## 0 2512 0 0 9430
## distance hour minute time_hour
## 0 0 0 0Why is NA ^ 0 not missing? Why is NA | TRUE not missing? Why is FALSE & NA not missing? Can you figure out the general rule? (NA * 0 is a tricky counterexample!)
Because anything that is ^ 0 equals 1. Because NA | TRUE is saying whether one of the two is TRUE and the second one is. Because at least one of the two expressions can be tested: FALSE & NA. In NA & NA neither can be tested and the results is NA & NA.
The general rule is that whenever there is a logical expressions, if one can be tested, then the result shouldn’t be NA. And any operation that the results is determined, regardless of the number, the inputting NA does not affect the result.
5.3.1 Exercises
How could you use arrange() to sort all missing values to the start? (Hint: use is.na()).
df <- tibble(x = c(5, 2, NA),
y = c(2, NA, 2))
rowSums(df)## [1] 7 NA NAarrange(df, desc(is.na(x)))## # A tibble: 3 x 2
## x y
## <dbl> <dbl>
## 1 NA 2
## 2 5 2
## 3 2 NAarrange(df, -(is.na(x)))## # A tibble: 3 x 2
## x y
## <dbl> <dbl>
## 1 NA 2
## 2 5 2
## 3 2 NA5.4.1 Exercises
Brainstorm as many ways as possible to select dep_time, dep_delay, arr_time, and arr_delay from flights.
vars <- c("dep_time", "dep_delay", "arr_time", "arr_delay")
select(flights, dep_time, dep_delay, arr_time, arr_delay)## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, starts_with("dep"), starts_with("arr"))## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, one_of(vars))## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect_(flights, .dots = vars)## Warning: select_() is deprecated.
## Please use select() instead
##
## The 'programming' vignette or the tidyeval book can help you
## to program with select() : https://tidyeval.tidyverse.org
## This warning is displayed once per session.## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect_(flights, "dep_time", "dep_delay", "arr_time", "arr_delay")## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, matches("dep"), matches("arr"), -matches("sched"), -carrier)## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, contains("dep"), contains("arr"), -contains("sched"), -carrier)## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, matches("^dep|^arr"))## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, matches("time$|delay$"), -contains("sched"), -contains("air"))## # A tibble: 336,776 x 4
## dep_time dep_delay arr_time arr_delay
## <int> <dbl> <int> <dbl>
## 1 517 2 830 11
## 2 533 4 850 20
## 3 542 2 923 33
## 4 544 -1 1004 -18
## 5 554 -6 812 -25
## 6 554 -4 740 12
## 7 555 -5 913 19
## 8 557 -3 709 -14
## 9 557 -3 838 -8
## 10 558 -2 753 8
## # … with 336,766 more rowsselect(flights, matches("^dep|arr_delay|time$"))## # A tibble: 336,776 x 7
## dep_time sched_dep_time dep_delay arr_time sched_arr_time arr_delay air_time
## <int> <int> <dbl> <int> <int> <dbl> <dbl>
## 1 517 515 2 830 819 11 227
## 2 533 529 4 850 830 20 227
## 3 542 540 2 923 850 33 160
## 4 544 545 -1 1004 1022 -18 183
## 5 554 600 -6 812 837 -25 116
## 6 554 558 -4 740 728 12 150
## 7 555 600 -5 913 854 19 158
## 8 557 600 -3 709 723 -14 53
## 9 557 600 -3 838 846 -8 140
## 10 558 600 -2 753 745 8 138
## # … with 336,766 more rowsWhat happens if you include the name of a variable multiple times in a select() call?
select(flights, dep_time, dep_time)## # A tibble: 336,776 x 1
## dep_time
## <int>
## 1 517
## 2 533
## 3 542
## 4 544
## 5 554
## 6 554
## 7 555
## 8 557
## 9 557
## 10 558
## # … with 336,766 more rowsNothing, it just returns it once.
What does the one_of() function do? Why might it be helpful in conjunction with this vector?
vars <- c("year", "month", "day", "dep_delay", "arr_delay")
select(flights, one_of(vars))## # A tibble: 336,776 x 5
## year month day dep_delay arr_delay
## <int> <int> <int> <dbl> <dbl>
## 1 2013 1 1 2 11
## 2 2013 1 1 4 20
## 3 2013 1 1 2 33
## 4 2013 1 1 -1 -18
## 5 2013 1 1 -6 -25
## 6 2013 1 1 -4 12
## 7 2013 1 1 -5 19
## 8 2013 1 1 -3 -14
## 9 2013 1 1 -3 -8
## 10 2013 1 1 -2 8
## # … with 336,766 more rowsIt works because select only accepts variable names without " " quotes. By including inside one_of() one can use character names.
Does the result of running the following code surprise you? How do the select helpers deal with case by default? How can you change that default?
By default, each select_helper function ignore the cases of the variables.
select(flights, contains("TIME"))## # A tibble: 336,776 x 6
## dep_time sched_dep_time arr_time sched_arr_time air_time time_hour
## <int> <int> <int> <int> <dbl> <dttm>
## 1 517 515 830 819 227 2013-01-01 05:00:00
## 2 533 529 850 830 227 2013-01-01 05:00:00
## 3 542 540 923 850 160 2013-01-01 05:00:00
## 4 544 545 1004 1022 183 2013-01-01 05:00:00
## 5 554 600 812 837 116 2013-01-01 06:00:00
## 6 554 558 740 728 150 2013-01-01 05:00:00
## 7 555 600 913 854 158 2013-01-01 06:00:00
## 8 557 600 709 723 53 2013-01-01 06:00:00
## 9 557 600 838 846 140 2013-01-01 06:00:00
## 10 558 600 753 745 138 2013-01-01 06:00:00
## # … with 336,766 more rowsWith this command you can treat each name as literal:
select(flights, contains("TIME", ignore.case = F))## # A tibble: 336,776 x 05.5.2 Exercises
Currently dep_time and sched_dep_time are convenient to look at, but hard to compute with because they’re not really continuous numbers. Convert them to a more convenient representation of number of minutes since midnight.
hours2mins <- function(x) {
x %/% 100 * 60 + x %% 100
}
# with integer division
mutate(flights,
dep_time = hours2mins(dep_time),
sched_dep_time = hours2mins(sched_dep_time))## # A tibble: 336,776 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <dbl> <dbl> <dbl> <int> <int>
## 1 2013 1 1 317 315 2 830 819
## 2 2013 1 1 333 329 4 850 830
## 3 2013 1 1 342 340 2 923 850
## 4 2013 1 1 344 345 -1 1004 1022
## 5 2013 1 1 354 360 -6 812 837
## 6 2013 1 1 354 358 -4 740 728
## 7 2013 1 1 355 360 -5 913 854
## 8 2013 1 1 357 360 -3 709 723
## 9 2013 1 1 357 360 -3 838 846
## 10 2013 1 1 358 360 -2 753 745
## # … with 336,766 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm># with rounding operations
mutate(flights,
dep_time = 60 * floor(dep_time/100) + (dep_time - floor(dep_time/100) * 100),
sched_dep_time = 60 * floor(sched_dep_time/100) + (sched_dep_time - floor(sched_dep_time/100) * 100))## # A tibble: 336,776 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <dbl> <dbl> <dbl> <int> <int>
## 1 2013 1 1 317 315 2 830 819
## 2 2013 1 1 333 329 4 850 830
## 3 2013 1 1 342 340 2 923 850
## 4 2013 1 1 344 345 -1 1004 1022
## 5 2013 1 1 354 360 -6 812 837
## 6 2013 1 1 354 358 -4 740 728
## 7 2013 1 1 355 360 -5 913 854
## 8 2013 1 1 357 360 -3 709 723
## 9 2013 1 1 357 360 -3 838 846
## 10 2013 1 1 358 360 -2 753 745
## # … with 336,766 more rows, and 11 more variables: arr_delay <dbl>,
## # carrier <chr>, flight <int>, tailnum <chr>, origin <chr>, dest <chr>,
## # air_time <dbl>, distance <dbl>, hour <dbl>, minute <dbl>, time_hour <dttm>Compare air_time with arr_time - dep_time. What do you expect to see? What do you see? What do you need to do to fix it?
flights %>%
mutate(dep_time = (dep_time %/% 100) * 60 + (dep_time %% 100),
sched_dep_time = (sched_dep_time %/% 100) * 60 + (sched_dep_time %% 100),
arr_time = (arr_time %/% 100) * 60 + (arr_time %% 100),
sched_arr_time = (sched_arr_time %/% 100) * 60 + (sched_arr_time %% 100)) %>%
transmute((arr_time - dep_time) %% (60*24) - air_time)## # A tibble: 336,776 x 1
## `(arr_time - dep_time)%%(60 * 24) - air_time`
## <dbl>
## 1 -34
## 2 -30
## 3 61
## 4 77
## 5 22
## 6 -44
## 7 40
## 8 19
## 9 21
## 10 -23
## # … with 336,766 more rowsCompare dep_time, sched_dep_time, and dep_delay. How would you expect those three numbers to be related?
hours2mins <- function(x) {
x %/% 100 * 60 + x %% 100
}
select(flights, contains("dep")) %>%
mutate(dep_time_two = hours2mins(dep_time) - hours2mins(sched_dep_time))## # A tibble: 336,776 x 4
## dep_time sched_dep_time dep_delay dep_time_two
## <int> <int> <dbl> <dbl>
## 1 517 515 2 2
## 2 533 529 4 4
## 3 542 540 2 2
## 4 544 545 -1 -1
## 5 554 600 -6 -6
## 6 554 558 -4 -4
## 7 555 600 -5 -5
## 8 557 600 -3 -3
## 9 557 600 -3 -3
## 10 558 600 -2 -2
## # … with 336,766 more rows# these two numbers don’t match because we aren’t accounting for flights
# where the departure time is the next day from the scheduled departure time.
select(flights, contains("dep")) %>%
mutate(dep_time_two = hours2mins(dep_time) - hours2mins(sched_dep_time)) %>%
filter(dep_delay != dep_time_two) %>%
mutate(dep_time_two = hours2mins(dep_time) - hours2mins(sched_dep_time - 2400))## # A tibble: 1,207 x 4
## dep_time sched_dep_time dep_delay dep_time_two
## <int> <int> <dbl> <dbl>
## 1 848 1835 853 853
## 2 42 2359 43 43
## 3 126 2250 156 156
## 4 32 2359 33 33
## 5 50 2145 185 185
## 6 235 2359 156 156
## 7 25 2359 26 26
## 8 106 2245 141 141
## 9 14 2359 15 15
## 10 37 2230 127 127
## # … with 1,197 more rowsFind the 10 most delayed flights using a ranking function. How do you want to handle ties? Carefully read the documentation for min_rank().
flights %>%
filter(min_rank(-(dep_delay)) %in% 1:10)## # A tibble: 10 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 9 641 900 1301 1242 1530
## 2 2013 1 10 1121 1635 1126 1239 1810
## 3 2013 12 5 756 1700 896 1058 2020
## 4 2013 3 17 2321 810 911 135 1020
## 5 2013 4 10 1100 1900 960 1342 2211
## 6 2013 6 15 1432 1935 1137 1607 2120
## 7 2013 6 27 959 1900 899 1236 2226
## 8 2013 7 22 845 1600 1005 1044 1815
## 9 2013 7 22 2257 759 898 121 1026
## 10 2013 9 20 1139 1845 1014 1457 2210
## # … with 11 more variables: arr_delay <dbl>, carrier <chr>, flight <int>,
## # tailnum <chr>, origin <chr>, dest <chr>, air_time <dbl>, distance <dbl>,
## # hour <dbl>, minute <dbl>, time_hour <dttm>flights %>%
top_n(10, dep_delay)## # A tibble: 10 x 19
## year month day dep_time sched_dep_time dep_delay arr_time sched_arr_time
## <int> <int> <int> <int> <int> <dbl> <int> <int>
## 1 2013 1 9 641 900 1301 1242 1530
## 2 2013 1 10 1121 1635 1126 1239 1810
## 3 2013 12 5 756 1700 896 1058 2020
## 4 2013 3 17 2321 810 911 135 1020
## 5 2013 4 10 1100 1900 960 1342 2211
## 6 2013 6 15 1432 1935 1137 1607 2120
## 7 2013 6 27 959 1900 899 1236 2226
## 8 2013 7 22 845 1600 1005 1044 1815
## 9 2013 7 22 2257 759 898 121 1026
## 10 2013 9 20 1139 1845 1014 1457 2210
## # … with 11 more variables: arr_delay <dbl>, carrier <chr>, flight <int>,
## # tailnum <chr>, origin <chr>, dest <chr>, air_time <dbl>, distance <dbl>,
## # hour <dbl>, minute <dbl>, time_hour <dttm>What does 1:3 + 1:10 return? Why? It wil return:
x <- c(2, 4, 6, 5, 7, 9, 8, 10, 12, 11)
p <- 1:3 + 1:10## Warning in 1:3 + 1:10: longer object length is not a multiple of shorter object
## lengthp == x## [1] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUEBecause 1:3 is reciclyed.
What trigonometric functions does R provide?
?Trig
5.6.7 Exercises
Brainstorm at least 5 different ways to assess the typical delay characteristics of a group of flights. Consider the following scenarios:
delay_char <-
flights %>%
group_by(flight) %>%
summarise(n = n(),
fifteen_early = mean(arr_delay == -15, na.rm = T),
fifteen_late = mean(arr_delay == 15, na.rm = T),
ten_always = mean(arr_delay == 10, na.rm = T),
thirty_early = mean(arr_delay == -30, na.rm = T),
thirty_late = mean(arr_delay == 30, na.rm = T),
percentage_on_time = mean(arr_delay == 0, na.rm = T),
twohours = mean(arr_delay > 120, na.rm = T)) %>%
map_if(is_double, round, 2) %>%
as_tibble()A flight is 15 minutes early 50% of the time, and 15 minutes late 50% of the time.
delay_char %>%
filter(fifteen_early == 0.5, fifteen_late == 0.5)## # A tibble: 0 x 9
## # … with 9 variables: flight <int>, n <int>, fifteen_early <dbl>,
## # fifteen_late <dbl>, ten_always <dbl>, thirty_early <dbl>,
## # thirty_late <dbl>, percentage_on_time <dbl>, twohours <dbl>A flight is always 10 minutes late.
delay_char %>%
filter(ten_always == 1)## # A tibble: 5 x 9
## flight n fifteen_early fifteen_late ten_always thirty_early thirty_late
## <int> <int> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 2254 1 0 0 1 0 0
## 2 3656 1 0 0 1 0 0
## 3 3785 2 0 0 1 0 0
## 4 3880 1 0 0 1 0 0
## 5 5854 1 0 0 1 0 0
## # … with 2 more variables: percentage_on_time <dbl>, twohours <dbl>A flight is 30 minutes early 50% of the time, and 30 minutes late 50% of the time.
delay_char %>%
filter(thirty_early == 0.5 & thirty_late == 0.5)## # A tibble: 0 x 9
## # … with 9 variables: flight <int>, n <int>, fifteen_early <dbl>,
## # fifteen_late <dbl>, ten_always <dbl>, thirty_early <dbl>,
## # thirty_late <dbl>, percentage_on_time <dbl>, twohours <dbl>99% of the time a flight is on time. 1% of the time it’s 2 hours late.
delay_char %>%
filter(percentage_on_time == 0.99 & twohours == 0.01)## # A tibble: 0 x 9
## # … with 9 variables: flight <int>, n <int>, fifteen_early <dbl>,
## # fifteen_late <dbl>, ten_always <dbl>, thirty_early <dbl>,
## # thirty_late <dbl>, percentage_on_time <dbl>, twohours <dbl>Which is more important: arrival delay or departure delay?
It depends
Come up with another approach that will give you the same output as:
not_cancelled <-
flights %>%
filter(!is.na(dep_delay), !is.na(arr_delay))
not_cancelled %>%
count(dest)## # A tibble: 104 x 2
## dest n
## <chr> <int>
## 1 ABQ 254
## 2 ACK 264
## 3 ALB 418
## 4 ANC 8
## 5 ATL 16837
## 6 AUS 2411
## 7 AVL 261
## 8 BDL 412
## 9 BGR 358
## 10 BHM 269
## # … with 94 more rows# and
not_cancelled %>%
count(tailnum, wt = distance)## # A tibble: 4,037 x 2
## tailnum n
## <chr> <dbl>
## 1 D942DN 3418
## 2 N0EGMQ 239143
## 3 N10156 109664
## 4 N102UW 25722
## 5 N103US 24619
## 6 N104UW 24616
## 7 N10575 139903
## 8 N105UW 23618
## 9 N107US 21677
## 10 N108UW 32070
## # … with 4,027 more rows# (without using count()).
#######################
not_cancelled %>%
group_by(dest) %>%
summarise(n = n())## # A tibble: 104 x 2
## dest n
## <chr> <int>
## 1 ABQ 254
## 2 ACK 264
## 3 ALB 418
## 4 ANC 8
## 5 ATL 16837
## 6 AUS 2411
## 7 AVL 261
## 8 BDL 412
## 9 BGR 358
## 10 BHM 269
## # … with 94 more rows# and
not_cancelled %>%
group_by(tailnum) %>%
tally(wt = distance)## # A tibble: 4,037 x 2
## tailnum n
## <chr> <dbl>
## 1 D942DN 3418
## 2 N0EGMQ 239143
## 3 N10156 109664
## 4 N102UW 25722
## 5 N103US 24619
## 6 N104UW 24616
## 7 N10575 139903
## 8 N105UW 23618
## 9 N107US 21677
## 10 N108UW 32070
## # … with 4,027 more rows# or
not_cancelled %>%
group_by(tailnum) %>%
summarize(n = sum(distance))## # A tibble: 4,037 x 2
## tailnum n
## <chr> <dbl>
## 1 D942DN 3418
## 2 N0EGMQ 239143
## 3 N10156 109664
## 4 N102UW 25722
## 5 N103US 24619
## 6 N104UW 24616
## 7 N10575 139903
## 8 N105UW 23618
## 9 N107US 21677
## 10 N108UW 32070
## # … with 4,027 more rowsOur definition of cancelled flights (is.na(dep_delay) | is.na(arr_delay) ) is slightly suboptimal. Why? Which is the most important column?
Because if a flight didn’t leave then it was cancelled. If the condition is.na(dep_delay) is met, then the flight was cancelled.
Look at the number of cancelled flights per day. Is there a pattern? Is the proportion of cancelled flights related to the average delay?
It looks like there is a positive relationship. The higher the average delay of the day, the higher the proportion of cancelled flights per day.
Which carrier has the worst delays?
flights %>%
group_by(carrier) %>%
summarise(dep_max = max(dep_delay, na.rm = T),
arr_max = max(arr_delay, na.rm = T)) %>%
arrange(desc(dep_max, arr_max)) %>%
filter(1:n() == 1)## # A tibble: 1 x 3
## carrier dep_max arr_max
## <chr> <dbl> <dbl>
## 1 HA 1301 1272Challenge: can you disentangle the effects of bad airports vs. bad carriers? Why/why not? (Hint: think about flights %>% group_by(carrier, dest) %>% summarise(n()))
It might be possible. For example, if we took the average departure delay for each carrier and then computed the deviations over the overall carrier mean from each airport mean, perhaps we can find something out. If the overall mean is, let’s say, 25 mins, and then each deviation is ± 1 or 2, then it could be that the airline is bad or that every single bad. I know, it might be more likely that the airline is bad, but we can’t be 100% sure. On the other hand, if the overall carrier mean is high and the deviations are all lower except for 1 or 2 airports then the effect is probably the airport effect.
flights %>%
summarise(n_car = n_distinct(carrier),
n_air = n_distinct(dest),
n_or = n_distinct(origin))## # A tibble: 1 x 3
## n_car n_air n_or
## <int> <int> <int>
## 1 16 105 3flights %>%
group_by(carrier) %>%
mutate(avg_carrier = mean(dep_delay, na.rm = T)) %>%
group_by(carrier, origin) %>%
mutate(origin_mean = mean(dep_delay, na.rm = T),
deviations = origin_mean - avg_carrier) %>%
summarise(deviations = mean(deviations), mean = mean(avg_carrier)) %>%
ggplot(aes(origin, deviations)) + geom_col() + facet_wrap(~ carrier)
Tearing out the effect is not straight forward but we can make some informed guesses. For example, whenever there are substantial deviations, they seem to be higher in EWR airport rather than in other airports. On the other hand, there are some airlines that look particular bad like 9E and MQ. And the same pattern is not found on the vast majority of other airlines, which would suggest it’s an airport issues rather than an airline issue.
flights %>%
group_by(carrier, dest) %>%
summarise(mean_departure = mean(dep_delay, na.rm = T),
mean_arrival = mean(arr_delay, na.rm = T))## # A tibble: 314 x 4
## # Groups: carrier [16]
## carrier dest mean_departure mean_arrival
## <chr> <chr> <dbl> <dbl>
## 1 9E ATL 0.965 0.857
## 2 9E AUS 19 -3.5
## 3 9E AVL -2.6 -12.1
## 4 9E BGR 34 NaN
## 5 9E BNA 19.1 9.29
## 6 9E BOS 14.8 5.66
## 7 9E BTV -4.5 -2.5
## 8 9E BUF 15.5 6.71
## 9 9E BWI 17.5 8.73
## 10 9E CAE -3.67 6
## # … with 304 more rowsFor each plane, count the number of flights before the first delay of greater than 1 hour.
flights %>%
mutate(dep_date = time_hour) %>%
group_by(tailnum) %>%
arrange(dep_date) %>%
mutate(cumulative = !cumany(arr_delay > 60)) %>%
filter(cumulative == T) %>%
tally(sort = TRUE)## # A tibble: 3,744 x 2
## tailnum n
## <chr> <int>
## 1 N705TW 97
## 2 N765US 97
## 3 N12125 94
## 4 N320AA 94
## 5 N13110 91
## 6 N3763D 82
## 7 N58101 82
## 8 N17122 81
## 9 N961UW 80
## 10 N950UW 79
## # … with 3,734 more rowsor
flights %>%
group_by(tailnum) %>%
arrange(time_hour) %>%
mutate(cum = arr_delay > 60,
cum_any = cumsum(cum)) %>%
filter(cum_any < 1) %>%
tally(sort = TRUE)## # A tibble: 3,744 x 2
## tailnum n
## <chr> <int>
## 1 N705TW 97
## 2 N765US 97
## 3 N12125 94
## 4 N320AA 94
## 5 N13110 91
## 6 N3763D 82
## 7 N58101 82
## 8 N17122 81
## 9 N961UW 80
## 10 N950UW 79
## # … with 3,734 more rowsWhat does the sort argument to count() do. When might you use it?
When you want to sort the cases based on the count.
flights %>%
count(flight, sort = T)## # A tibble: 3,844 x 2
## flight n
## <int> <int>
## 1 15 968
## 2 27 898
## 3 181 882
## 4 301 871
## 5 161 786
## 6 695 782
## 7 1109 716
## 8 745 711
## 9 359 709
## 10 1 701
## # … with 3,834 more rows5.7.1 Exercises
Refer back to the table of useful mutate and filtering functions. Describe how each operation changes when you combine it with grouping.
Which one?
Which plane (tailnum) has the worst on-time record?
flights %>%
filter(!is.na(arr_delay)) %>%
group_by(tailnum) %>%
summarise(prop_time = sum(arr_delay <= 30)/n(),
mean_arr = mean(arr_delay, na.rm = T),
fl = n()) %>%
arrange(desc(prop_time))## # A tibble: 4,037 x 4
## tailnum prop_time mean_arr fl
## <chr> <dbl> <dbl> <int>
## 1 N103US 1 -6.93 46
## 2 N1200K 1 -9.38 21
## 3 N121DE 1 15 2
## 4 N137DL 1 -5 1
## 5 N143DA 1 24 1
## 6 N14628 1 -6 1
## 7 N14629 1 -16.2 4
## 8 N1607B 1 -16 3
## 9 N1608 1 -11.3 3
## 10 N1610D 1 -14.5 2
## # … with 4,027 more rowsAll these flights are always late.
What time of day should you fly if you want to avoid delays as much as possible?
flights %>%
group_by(hour) %>%
filter(!is.na(dep_delay)) %>%
summarise( delay = mean( dep_delay > 0 , na.rm = T)) %>%
ggplot(aes(hour, delay, fill = delay)) + geom_col() 
# or
flights %>%
group_by(hour) %>%
summarize(m = mean(dep_delay, na.rm = TRUE),
sd = sd(dep_delay, na.rm = TRUE),
low_ci = m - 2*sd,
high_ci = m + 2*sd,
n = n()) %>%
ggplot(aes(hour, m, ymin = low_ci, ymax = high_ci)) +
geom_pointrange()## Warning: Removed 1 rows containing missing values (geom_pointrange).
Worst time to flight is in the early evening. Although that happens because more flights go out on that specific time also.
For each destination, compute the total minutes of delay. For each, flight, compute the proportion of the total delay for its destination.
flights %>%
group_by(dest) %>%
filter(!is.na(dep_delay)) %>%
summarise(tot_mins = sum(dep_delay[dep_delay > 0]))## # A tibble: 104 x 2
## dest tot_mins
## <chr> <dbl>
## 1 ABQ 4076
## 2 ACK 2603
## 3 ALB 10934
## 4 ANC 105
## 5 ATL 254414
## 6 AUS 36623
## 7 AVL 3092
## 8 BDL 8471
## 9 BGR 8170
## 10 BHM 8817
## # … with 94 more rowsflights %>%
filter(!is.na(dep_delay)) %>%
group_by(tailnum, dest) %>%
summarise(m = mean(dep_delay > 0), n = n()) %>%
arrange(desc(m))## # A tibble: 44,218 x 4
## # Groups: tailnum [4,037]
## tailnum dest m n
## <chr> <chr> <dbl> <int>
## 1 D942DN MCO 1 2
## 2 N10156 BDL 1 1
## 3 N10156 CLE 1 1
## 4 N10156 DCA 1 2
## 5 N10156 GSO 1 1
## 6 N10156 GSP 1 1
## 7 N10156 IAD 1 1
## 8 N10156 IND 1 2
## 9 N10156 MHT 1 1
## 10 N10156 MSN 1 1
## # … with 44,208 more rowsDelays are typically temporally correlated: even once the problem that caused the initial delay has been resolved, later flights are delayed to allow earlier flights to leave. Using lag() explore how the delay of a flight is related to the delay of the immediately preceding flight.
flights %>%
mutate(new_sched_dep_time = lubridate::make_datetime(year, month, day, hour, minute)) %>%
arrange(new_sched_dep_time) %>%
mutate(prev_time = lag(dep_delay)) %>%
# filter(between(dep_delay, 0, 300), between(prev_time, 0, 300)) %>% # play with this one
select(origin, new_sched_dep_time, dep_delay, prev_time) %>%
ggplot(aes(dep_delay, prev_time)) + geom_point(alpha = 1/10) +
geom_smooth()## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'## Warning: Removed 14167 rows containing non-finite values (stat_smooth).## Warning: Removed 14167 rows containing missing values (geom_point).
# or
flights %>%
select(year, month, day, hour, dest, dep_delay) %>%
group_by(dest) %>%
mutate(lag_delay = lag(dep_delay)) %>%
arrange(dest) %>%
filter(!is.na(lag_delay)) %>%
summarize(cor = cor(dep_delay, lag_delay, use = "complete.obs"),
n = n()) %>%
arrange(desc(cor)) %>%
filter(row_number(desc(cor)) %in% 1:10)## # A tibble: 10 x 3
## dest cor n
## <chr> <dbl> <int>
## 1 SBN 0.687 9
## 2 ORD 0.403 16641
## 3 HDN 0.365 13
## 4 ATL 0.351 16897
## 5 SFO 0.344 13229
## 6 BZN 0.325 34
## 7 BOS 0.323 15048
## 8 FLL 0.312 11933
## 9 BNA 0.302 6104
## 10 MDW 0.296 4043Although there is a lot of noise, you can see a sort of straight line going on there. There is also a correlation between the lagged values in many of the destionatinons. So correlation between flights is mostly in specific airports.
Look at each destination. Can you find flights that are suspiciously fast? (i.e. flights that represent a potential data entry error). Compute the air time a flight relative to the shortest flight to that destination. Which flights were most delayed in the air?
# (1)
flights %>%
group_by(dest) %>%
arrange(air_time) %>%
slice(1:5) %>%
select(tailnum, sched_dep_time, sched_arr_time, air_time) %>%
arrange(air_time)## Adding missing grouping variables: `dest`## # A tibble: 517 x 5
## # Groups: dest [105]
## dest tailnum sched_dep_time sched_arr_time air_time
## <chr> <chr> <int> <int> <dbl>
## 1 BDL N16911 1315 1411 20
## 2 BDL N12167 527 628 20
## 3 BDL N27200 851 954 21
## 4 BDL N13955 1315 1411 21
## 5 BDL N12160 1329 1426 21
## 6 BOS N947UW 1500 1608 21
## 7 PHL N13913 2129 2224 21
## 8 PHL N12921 2130 2225 21
## 9 PHL N8501F 1935 2056 21
## 10 PHL N22909 2129 2224 22
## # … with 507 more rows# (2)
flights %>%
group_by(dest) %>%
mutate(shortest = air_time - min(air_time, na.rm = T)) %>%
top_n(1, air_time) %>%
arrange(-air_time) %>%
select(tailnum, sched_dep_time, sched_arr_time, shortest)## Warning in min(air_time, na.rm = T): no non-missing arguments to min; returning
## Inf## Adding missing grouping variables: `dest`## # A tibble: 112 x 5
## # Groups: dest [104]
## dest tailnum sched_dep_time sched_arr_time shortest
## <chr> <chr> <int> <int> <dbl>
## 1 HNL N77066 1335 1836 133
## 2 SFO N703TW 1730 2110 195
## 3 LAX N178DN 1815 2146 165
## 4 ANC N572UA 1615 1953 46
## 5 SAN N794JB 1620 1934 134
## 6 SNA N16709 1819 2137 131
## 7 BUR N624JB 1730 2046 110
## 8 LAS N852UA 1729 2013 143
## 9 SJC N632JB 1830 2205 91
## 10 SEA N17245 1727 2040 119
## # … with 102 more rowsFind all destinations that are flown by at least two carriers. Use that information to rank the carriers.
flights %>%
group_by(dest) %>%
filter(n_distinct(carrier) > 2) %>%
group_by(carrier) %>%
summarise(n = n_distinct(dest)) %>%
arrange(-n)## # A tibble: 15 x 2
## carrier n
## <chr> <int>
## 1 DL 37
## 2 EV 36
## 3 UA 36
## 4 9E 35
## 5 B6 30
## 6 AA 17
## 7 MQ 17
## 8 WN 9
## 9 OO 5
## 10 US 5
## 11 VX 3
## 12 YV 3
## 13 FL 2
## 14 AS 1
## 15 F9 1