Cross-validation the Human Accomplishment dataset with Wikipedia data

Data and cleaning

#load pan
pan = read_csv("data/Pantheon_2016_12_03.csv")

Parsed with column specification:
cols(
  Rank = col_integer(),
  Name = col_character(),
  `Place of Birth*` = col_character(),
  `Birth Year` = col_integer(),
  Gender = col_character(),
  Occupation = col_character(),
  L = col_integer(),
  `L*` = col_double(),
  PV = col_character(),
  PVe = col_character(),
  PVne = col_character(),
  `sPV` = col_character(),
  HPI = col_double()
)

#sensible names
names(pan) = c("Rank", "Name", "Place of Birth*", "Birth Year", "Gender", "Occupation", 
"L", "Lstar", "PV", "PVe", "PVne", "sPV", "HPI")
#name variable
pan$FNLN = pan$Name
pan$Name = NULL
#for now, we remove the few duplicates
duplicated(pan$FNLN) %>% which

 [1]  2813  3476  8242  8591  8611  8659  8885  8948  9231  9597 10332 10899

pan %<>% dplyr::filter(!duplicated(FNLN))
#convert to real numerics
pan$PV %>% str_detect(" M") %>% proportion_true()

[1] 1

pan$sPV %>% str_detect(" K") %>% proportion_true()

[1] 1

#just remove the string part since they all have the same
pan$PV %<>% str_replace(" M", "") %>% as.numeric()
pan$PVe %<>% str_replace(" M", "") %>% as.numeric()
pan$PVne %<>% str_replace(" M", "") %>% as.numeric()
pan$sPV %<>% str_replace(" K", "") %>% as.numeric()
#merge HA into pan when exact match
#no this causes duplicates
#we have to use the inventories
sci = dplyr::filter(d, Inventory == "Science", !duplicated(FNLN)) %>% left_join(pan)

Joining, by = "FNLN"

#matches
#assert no duplicates
assert_that(!sci$FNLN %>% duplicated() %>% any)

[1] TRUE

#how many?
sci$HPI %>% count_NA(reverse = T)

[1] 338

sci$HPI %>% count_NA(reverse = T) %>% divide_by(nrow(sci))

[1] 0.25

EDA

index_vars = c("Index", "Rank", "L", "Lstar", "PV", "PVe", "PVne", "sPV", "HPI")
wtd.cors(sci[index_vars]) %>% write_clipboard()

      Index  Rank     L Lstar    PV   PVe  PVne   sPV   HPI
Index  1.00 -0.45  0.59 -0.12  0.47  0.45  0.48  0.29  0.51
Rank  -0.45  1.00 -0.73 -0.11 -0.45 -0.43 -0.44 -0.17 -0.93
L      0.59 -0.73  1.00  0.08  0.83  0.81  0.83  0.47  0.86
Lstar -0.12 -0.11  0.08  1.00 -0.03 -0.08  0.00 -0.03  0.09
PV     0.47 -0.45  0.83 -0.03  1.00  0.98  0.99  0.53  0.66
PVe    0.45 -0.43  0.81 -0.08  0.98  1.00  0.94  0.53  0.62
PVne   0.48 -0.44  0.83  0.00  0.99  0.94  1.00  0.52  0.67
sPV    0.29 -0.17  0.47 -0.03  0.53  0.53  0.52  1.00  0.25
HPI    0.51 -0.93  0.86  0.09  0.66  0.62  0.67  0.25  1.00

GG_scatter(sci, "Index", "L", case_names_vector = "FNLN") +
  xlab("Human Accomplishment index of scientists (Charles Murray)") +
  ylab("Number of Wikipedias that cover this person\n(Pantheon dataset)")
ggsave("figures/HA_L.png")

Saving 7.29 x 4.5 in image

GG_scatter(sci, "Index", "HPI", case_names_vector = "FNLN") +
  xlab("Human Accomplishment index of scientists (Charles Murray)") +
  ylab("Complex measure of historical importance based of Wikipedia\n(Pantheon dataset)")
ggsave("figures/HA_HPI.png")

Saving 7.29 x 4.5 in image

GG_scatter(sci, "Index", "PV", case_names_vector = "FNLN") +
  xlab("Human Accomplishment index of scientists (Charles Murray)") +
  ylab("Page views (millions), Wikipedia\n(Pantheon dataset)")
ggsave("figures/HA_PV.png")

Saving 7.29 x 4.5 in image

GG_scatter(sci, "PVe", "PVne", case_names_vector = "FNLN") +
  xlab("Page views (millions), English Wikipedia\n(Pantheon dataset)") +
  ylab("Page views (millions), Non-English Wikipedia\n(Pantheon dataset)")
ggsave("figures/PVe_PVne.png")

Saving 7.29 x 4.5 in image