I first get the data and create list of matrices.

t <- data.table(read_dta(paste0(path, "data/tmatrices_cz.dta")))
c <- data.table(read_dta(paste0(path, "data/cz_full_covariates.dta")))
c <- c[, .(cz, s_rank, e_rank_b)]
setnames(c, names(c), c("cz", "rmob", "amob"))
setkey(t, cz)
setkey(c, cz)
t <- c[t] # merge data
t <- t[complete.cases(t)] # remove missing data
nrow(t)
[1] 709
# loop to create matrices 
mm <- list()
for (i in 1:nrow(t)) {
      tm <- matrix(NA, 5, 5)
            for (h in 1:5) {
                  for (j in 1:5) {
                        tm[h, j] <- t[i, get(paste0("prob_p", h, "_k", j))]
                  }
            }
      mm[[i]] <- tm
}

Hopefulness function:

hopefulness <- function(mat) {
  t <- upper.triangle(mat)
  diag(t) <- 0
  return( sum(t[1:2,4:5])/sum(t) )
}
t[, hope := sapply(mm, hopefulness)]
hist(t$hope)

cor(t[, .(rmob, amob, hope)])
           rmob       amob       hope
rmob  1.0000000 -0.6844716 -0.7758684
amob -0.6844716  1.0000000  0.8947943
hope -0.7758684  0.8947943  1.0000000

Pretty high correlation (.78 and 0.89). If I consider only upper transitions (let’s say 3 to 5):

upperMob <- function(mat) {
  t <- upper.triangle(mat)
  diag(t) <- 0
  # return((t[1,5]*1/3 + t[1,4]*1/4 + t[2,5]*1/4 + t[2,4]* 1/6 )/ sum(t) ) # based on number of jumps
  return((sum(t[3,4:5]) + t[4,5])/ sum(t) )
}
t[, upper := sapply(mm, upperMob)]
hist(t$upper)

cor(t[, .(rmob, amob, upper)])
            rmob       amob     upper
rmob   1.0000000 -0.6844716 0.2784068
amob  -0.6844716  1.0000000 0.3560352
upper  0.2784068  0.3560352 1.0000000
cor(t[, .(hope, upper)])
          hope    upper
hope  1.000000 0.185908
upper 0.185908 1.000000

This means that Chetty’s mobility measures are mainly measuring lower level mobility. Any comments?

LS0tCnRpdGxlOiAiQ2hldHR5J3MgTW9iaWxpdHkgTWF0cmljZXMiCm91dHB1dDogaHRtbF9ub3RlYm9vawphdXRob3I6IFNlYmFzdGlhbiBEYXphCi0tLQoKSSBmaXJzdCBnZXQgdGhlIGRhdGEgYW5kIGNyZWF0ZSBsaXN0IG9mIG1hdHJpY2VzLgoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CnJtKGxpc3Q9bHMoYWxsPVRSVUUpKQpsaWJyYXJ5KGhhdmVuKQpsaWJyYXJ5KHNkYXphcikKbGlicmFyeShtYXRyaXhjYWxjKQpsaWJyYXJ5KGRhdGEudGFibGUpCnBhdGggPC0gIi9Vc2Vycy9zZGF6YS9Hb29nbGUgRHJpdmUvMDBEaXNzZXJ0YXRpb24vQ2hhcHRlcnMvMDIvIgpgYGAKCmBgYHtyfQp0IDwtIGRhdGEudGFibGUocmVhZF9kdGEocGFzdGUwKHBhdGgsICJkYXRhL3RtYXRyaWNlc19jei5kdGEiKSkpCmMgPC0gZGF0YS50YWJsZShyZWFkX2R0YShwYXN0ZTAocGF0aCwgImRhdGEvY3pfZnVsbF9jb3ZhcmlhdGVzLmR0YSIpKSkKCmMgPC0gY1ssIC4oY3osIHNfcmFuaywgZV9yYW5rX2IpXQpzZXRuYW1lcyhjLCBuYW1lcyhjKSwgYygiY3oiLCAicm1vYiIsICJhbW9iIikpCgpzZXRrZXkodCwgY3opCnNldGtleShjLCBjeikKCnQgPC0gY1t0XSAjIG1lcmdlIGRhdGEKdCA8LSB0W2NvbXBsZXRlLmNhc2VzKHQpXSAjIHJlbW92ZSBtaXNzaW5nIGRhdGEKbnJvdyh0KQoKIyBsb29wIHRvIGNyZWF0ZSBtYXRyaWNlcyAKbW0gPC0gbGlzdCgpCmZvciAoaSBpbiAxOm5yb3codCkpIHsKICAgICAgdG0gPC0gbWF0cml4KE5BLCA1LCA1KQogICAgICAgICAgICBmb3IgKGggaW4gMTo1KSB7CiAgICAgICAgICAgICAgICAgIGZvciAoaiBpbiAxOjUpIHsKICAgICAgICAgICAgICAgICAgICAgICAgdG1baCwgal0gPC0gdFtpLCBnZXQocGFzdGUwKCJwcm9iX3AiLCBoLCAiX2siLCBqKSldCiAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICBtbVtbaV1dIDwtIHRtCn0KCmBgYAoKSG9wZWZ1bG5lc3MgZnVuY3Rpb246CgpgYGB7cn0KaG9wZWZ1bG5lc3MgPC0gZnVuY3Rpb24obWF0KSB7CiAgdCA8LSB1cHBlci50cmlhbmdsZShtYXQpCiAgZGlhZyh0KSA8LSAwCiAgcmV0dXJuKCBzdW0odFsxOjIsNDo1XSkvc3VtKHQpICkKfQoKYGBgCgpgYGB7cn0KdFssIGhvcGUgOj0gc2FwcGx5KG1tLCBob3BlZnVsbmVzcyldCmhpc3QodCRob3BlKQpjb3IodFssIC4ocm1vYiwgYW1vYiwgaG9wZSldKQpgYGAKClByZXR0eSBoaWdoIGNvcnJlbGF0aW9uICguNzggYW5kIDAuODkpLiBJZiBJIGNvbnNpZGVyIG9ubHkgdXBwZXIgdHJhbnNpdGlvbnMgKGxldCdzIHNheSAzIHRvIDUpOiAKCmBgYHtyfQp1cHBlck1vYiA8LSBmdW5jdGlvbihtYXQpIHsKICB0IDwtIHVwcGVyLnRyaWFuZ2xlKG1hdCkKICBkaWFnKHQpIDwtIDAKICAjIHJldHVybigodFsxLDVdKjEvMyArIHRbMSw0XSoxLzQgKyB0WzIsNV0qMS80ICsgdFsyLDRdKiAxLzYgKS8gc3VtKHQpICkgIyBiYXNlZCBvbiBudW1iZXIgb2YganVtcHMKICByZXR1cm4oKHN1bSh0WzMsNDo1XSkgKyB0WzQsNV0pLyBzdW0odCkgKQp9CgpgYGAKCmBgYHtyfQp0WywgdXBwZXIgOj0gc2FwcGx5KG1tLCB1cHBlck1vYildCmhpc3QodCR1cHBlcikKY29yKHRbLCAuKHJtb2IsIGFtb2IsIHVwcGVyKV0pCmNvcih0WywgLihob3BlLCB1cHBlcildKQpgYGAKClRoaXMgbWVhbnMgdGhhdCBDaGV0dHkncyBtb2JpbGl0eSBtZWFzdXJlcyBhcmUgbWFpbmx5IG1lYXN1cmluZyBsb3dlciBsZXZlbCBtb2JpbGl0eS4gQW55IGNvbW1lbnRzPwoK