Packages

library(pracma)
library(tidyverse)
library(readxl)
library(glmnet)
library(knitr)
library(kableExtra)
library(plotly)

Reading the data

FN1.tbl <- read_excel("data/NL_3035_countpixelEUcropmap.xlsx",
                      sheet = "Total", range = "I4:K15", col_names = FALSE,
                      col_types = c("text", "text", "numeric")) %>%
  rename(crop = ...1, code = ...2, value = ...3)

FN2.tbl <- read_excel("data/ConfMatrix_NL.xlsx", sheet = "ConfMatClean") %>% 
  rename(code = ...1)

FN3.tbl <- read_csv("data/EUCROPMAP_legend.csv",
                    col_types = cols(code = col_character()))
FN1.tbl %>% 
  mutate(value = format(value, nsmall = 0, big.mark = ",", digits = 10)) %>% 
  kable(caption = "Table FN1", align = "lcr")%>%
  kable_styling(bootstrap_options = 
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12)
Table FN1
crop code value
common wheat 211 20,287,640
barley 213 5,168,348
maize 216 36,809,904
potatoes 221 10,542,589
sugar beet 222 7,981,893
other roots crops 223 914,813
dry pulses, vegetables and flowers 240 1,807,891
fodder crops (cereals and leguminous) 250 443,079
bare arable land 290 472,773
woodland and shrubland vegetation (incl. permanent crops) 300 72,256,453
permanent and temporary grasslands 500 1,641,199
Masked 1 82,561,683
FN2.tbl %>% kable(caption = "Table FN2") %>%
  kable_styling(bootstrap_options = 
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12) %>%
  scroll_box(width = "100%")
Table FN2
code 100 211 213 214 215 216 218 221 222 223 230 240 250 290 300 500 600 other classes TOTAL
211 7 57 9 2 1 1 1 0 0 1 0 2 3 5 0 44 0 2 135
213 0 5 5 1 1 0 0 0 0 0 1 1 0 2 0 6 0 0 22
216 3 1 2 0 1 102 0 20 10 5 2 4 2 1 3 22 1 3 182
221 0 0 0 0 0 0 0 59 6 0 0 1 0 0 0 0 0 0 66
222 1 0 0 0 0 0 0 14 22 1 0 0 0 0 0 0 0 0 38
223 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1
240 0 0 0 0 0 0 0 0 1 4 0 4 0 2 0 1 0 0 12
250 0 0 0 0 0 1 0 0 0 1 0 1 0 0 0 0 0 0 3
290 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 2
300 31 0 0 0 0 6 0 1 0 0 0 0 0 0 218 27 3 4 290
500 31 9 5 0 0 10 0 0 0 0 1 2 2 1 18 595 2 8 684
Masked 130 1 0 0 0 0 0 0 1 1 0 6 0 0 32 57 6 9 243
Total 203 73 21 3 3 120 1 94 40 14 4 21 7 13 271 752 12 26 1678
FN3.tbl %>% kable(caption = "Table FN3") %>% 
  kable_styling(bootstrap_options = 
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12) 
Table FN3
code label LUCAS
100 artificial land A11/A12/A13/A21/A22
211 common wheat B11
212 durum wheat B12
213 barley B13
214 rye B14
215 oats B15
216 maize B16
217 rice B17
218 triticale B18
219 other cereals B19
221 potatoes B21
222 sugar beet B22
223 other roots crops B23
230 other non permanent industrial crops B34/B35/B36/B37
231 sunflower B31
232 rape and turnip rape B32
233 soya B33
240 dry pulses, vegetables and flowers B41/B42/B43/B44/B45
250 fodder crops (cereals and leguminous) B51/B52/B53/B54
290 bare arable land F40 U111/112/113
300 woodland and shrubland vegetation (incl. permanent crops) C10/C21/C22/C23/C31/C32/C33/D10/D20/B71-77/B81-84
500 permanent and temporary grasslands E10/E20/E30/B55
600 Bare land and lichens/moss F10/F20/F20/F40 not agricultural land use

PCG and TC matrices

Leg <- FN3.tbl 

class_codes <- setdiff(names(FN2.tbl), c("code", "other classes", "TOTAL"))
ground_codes <- FN2.tbl %>% pull(code) %>% setdiff(c("Masked", "Total"))

TC <- FN1.tbl %>%
  #filter(crop != "Masked") %>% 
  select(code, value) %>%
  mutate_all(as.numeric) %>%
  column_to_rownames(var = "code") %>%
  as.matrix()

PCG <- FN2.tbl %>% 
  select(-one_of(c(setdiff(class_codes, ground_codes), "other classes"))) %>% 
  bind_cols(tibble(`other classes` = FN2.tbl %>% 
                     select_at(.vars = 
                                 c(setdiff(class_codes, ground_codes), 
                                   "other classes")) %>% 
                     rowSums())) %>% 
  filter(code != "Total") %>%
  #filter(code != "Total", code != "Masked") %>%
  select(-TOTAL) %>%
  #select(-TOTAL, -`other classes`) %>%
  column_to_rownames(var = "code") %>%
  mutate_all(as.numeric) %>%
  as.matrix()

TC %>% 
  as.data.frame() %>% 
  rownames_to_column(var = "code") %>% 
  mutate(value = format(value, nsmall = 0, big.mark = ",", digits = 10)) %>% 
  kable(caption = "Table TC", align = "lr") %>%
  kable_styling(bootstrap_options =
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12)
Table TC
code value
211 20,287,640
213 5,168,348
216 36,809,904
221 10,542,589
222 7,981,893
223 914,813
240 1,807,891
250 443,079
290 472,773
300 72,256,453
500 1,641,199
1 82,561,683
PCG %>% kable(caption = "Table PCG") %>% 
  kable_styling(bootstrap_options = 
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12)
Table PCG
211 213 216 221 222 223 240 250 290 300 500 other classes
211 57 9 1 0 0 1 2 3 5 0 44 13
213 5 5 0 0 0 0 1 0 2 0 6 3
216 1 2 102 20 10 5 4 2 1 3 22 10
221 0 0 0 59 6 0 1 0 0 0 0 0
222 0 0 0 14 22 1 0 0 0 0 0 1
223 0 0 0 0 0 1 0 0 0 0 0 0
240 0 0 0 0 1 4 4 0 2 0 1 0
250 0 0 1 0 0 1 1 0 0 0 0 0
290 0 0 0 0 0 0 0 0 2 0 0 0
300 0 0 6 1 0 0 0 0 0 218 27 38
500 9 5 10 0 0 0 2 2 1 18 595 42
Masked 1 0 0 0 1 1 6 0 0 32 57 145

Confest function

Confest <- function(PCG, TC, cloud, replications = 1000){
  # PCG matrice de confusion mxn  (effectifs) avec classif 
  # en ligne et terrain en colonne. Attention existe une 
  # classe nuages terrain
  # TrueC: proportions classes pour la totalitè de l'image
  # cloud should equal 1  if cloud ground class
  #PCG <- t(PCG)
  ncg = dim(PCG) # dimension
  nc = ncg[1]
  ng = ncg[2]
  
  if (cloud > 0){
    #  elimination of the last column with cloud ground class
    PCG <- PCG[, 1:(ng-1)]
    ng <- ng - 1
  }
  
  Ng <- colSums(PCG)
  N <- sum(Ng)
  PCG <- PCG/N
  PG <- colSums(PCG)
  PC <- rowSums(PCG)
  # Conditional probabilities of ground categories per given image class
  RC <- diag(PC)
  PGIC <- solve(RC) %*% PCG
  # Conditional probabilities of image categories per given ground class
  RG <- diag(PG)
  PCIG <- PCG %*% solve(RG) 
  # Attention nuages
  # Direct estimates of ground categories probalilities 
  # (sum can be greater than one)
  TGD <- t(TC) %*% PGIC
  # Indirect estimates of ground categories probablitities
  X <- PCIG
  y <- TC
  #mdl <- lm(y ~ X - 1)
  mdl <- glmnet(X, y, lambda = 0, lower.limits = 0, intercept = FALSE)
  beta <- mdl$beta #mdl$coefficients
  
  # C <- diag(ng)*(-1)
  # Aeq <- matrix(rep(1, ng), ncol = 1)
  # beq <- 1;
  # m = matrix(rep(0, ng), ncol = 1)
  # bb <- pracma::lsqlin(X, y, C, m)
  
  # Bootstrapping 100 times 
  edges <- rbind(matrix(rep(0, ng), nrow = 1), apply(PCIG, 2, cumsum))
  b <- NULL
  Pcig <- array(dim = c(nc, ng))
  for (i in 1:replications){
    # Resampling on each ground class sepatately  
    for (j in 1:ng) {
      numbers <- matrix(runif(Ng[j]), ncol = 1) 
      Eff <- pracma::histc(numbers, edges[,j])$cnt 
      Pcig[,j] <- Eff[-nc - 1]/Ng[j]
    }
    X <- Pcig
    #mdl <- lm(y ~ X - 1)
    #bi <- mdl$coefficients
    mdl <- glmnet(X, y, lambda = 0, lower.limits = 0, intercept = FALSE)
    beta <- mdl$beta #mdl$coefficients
    b <- cbind(b, beta)
  }
  
  # Image population classes percentages
  TC
  # Direct estimation of ground classes percentages
  direct <- TGD
  TOT <- colSums(direct)
  # Bootstrap inverse estimation without constraint
  mb <- rowMeans(b)
  #TOT <- colSums(mb)
  # Bootstrap inverse estimation with constraints: sum=1 and p>0
  # mbb <- mean(bb)
  # TOT <- colSums(mbb)
  # Standard deviations of inverse estimator
  stb <- apply(b, 1, function(x) sqrt(var(x)))
  #stbb <- sqrt(var(bb))
  # Coefficient of variation of inverse estimator
  cvb <- apply(b, 1, function(x) 100*sqrt(var(x))/mean(x))
  #cvbb <- 100*sqrt(var(bb))./mean(bb)
  # Correlation matrix for inverse with constraints
  #COR <- cor(bb)
  return(list(mb = mb, stb = stb, cvb = cvb, b = b))
}

Results

result <- Confest(PCG, TC, cloud = 0, replications = 1000)

format_values <- 
  function(x, y) 
    format(x, nsmall = y, big.mark = ",", digits = 10)

result[c("mb", "stb", "cvb")] %>% 
  as.data.frame() %>% 
  rownames_to_column(var = "Name") %>% 
  rename(Mean = mb, 'Deviation' = stb, CV = cvb) %>% 
  mutate(Mean = format_values(round(Mean), 0),
         Deviation = format_values(round(Deviation), 0),
         CV = format_values(round(CV,2), 2)) %>% 
  kable(caption = "Table Bootstrap Results (1000 rounds)", align = "lrrr") %>%
  kable_styling(bootstrap_options = 
                  c("striped", "hover", "condensed", "responsive"), 
                full_width = FALSE, font_size = 12)
Table Bootstrap Results (1000 rounds)
Name Mean Deviation CV
V1 6,398,297 6,065,856 94.80
V2 623,259 3,016,114 483.93
V3 15,229,800 8,700,489 57.13
V4 13,708,097 4,314,375 31.47
V5 10,085,957 4,835,054 47.94
V6 14,180,464 15,700,989 110.72
V7 21,328,754 33,263,070 155.95
V8 1,981,591 6,617,650 333.96
V9 4,650,887 7,964,841 171.25
V10 68,351,509 6,687,291 9.78
V11 0 0 NaN
V12 102,264,099 24,786,395 24.24
n_var <- nrow(result$b)

df <- result$b %>% t() %>% as.matrix() %>% as_tibble() %>% pivot_longer(everything(), names_to = "var", values_to = "value") %>% mutate(var = factor(as.integer(str_remove(var, "V")), levels = 1:n_var,labels = paste("Beta", 1:n_var))) 

p <-  df %>% ggplot(aes(x = value)) + geom_histogram(alpha=0.25, position="identity", bins = 30, aes(color = var, fill = var)) + facet_wrap(. ~ var,nrow = 4) + theme_bw() + guides(fill= FALSE, color = FALSE) + theme(strip.background = element_rect(fill = gray(.95)))

ggplotly(p,tooltip = c("y")) %>% 
  layout(showlegend = FALSE) %>% 
  layout(plot_bgcolor='rgba(254, 254, 254, .25)') %>% 
  layout(paper_bgcolor='#eeeeee') %>% 
  config(displayModeBar = FALSE)

Stay Tuned

Please visit the castlab.org page for latest updates and news. Comments, bug reports and pull requests are always welcome.