Packages
library(pracma)
library(tidyverse)
library(readxl)
library(glmnet)
library(knitr)
library(kableExtra)
library(plotly)Reading the data
FN1.tbl <- read_excel("data/NL_3035_countpixelEUcropmap.xlsx",
sheet = "Total", range = "I4:K15", col_names = FALSE,
col_types = c("text", "text", "numeric")) %>%
rename(crop = ...1, code = ...2, value = ...3)
FN2.tbl <- read_excel("data/ConfMatrix_NL.xlsx", sheet = "ConfMatClean") %>%
rename(code = ...1)
FN3.tbl <- read_csv("data/EUCROPMAP_legend.csv",
col_types = cols(code = col_character()))FN1.tbl %>%
mutate(value = format(value, nsmall = 0, big.mark = ",", digits = 10)) %>%
kable(caption = "Table FN1", align = "lcr")%>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12)| crop | code | value |
|---|---|---|
| common wheat | 211 | 20,287,640 |
| barley | 213 | 5,168,348 |
| maize | 216 | 36,809,904 |
| potatoes | 221 | 10,542,589 |
| sugar beet | 222 | 7,981,893 |
| other roots crops | 223 | 914,813 |
| dry pulses, vegetables and flowers | 240 | 1,807,891 |
| fodder crops (cereals and leguminous) | 250 | 443,079 |
| bare arable land | 290 | 472,773 |
| woodland and shrubland vegetation (incl. permanent crops) | 300 | 72,256,453 |
| permanent and temporary grasslands | 500 | 1,641,199 |
| Masked | 1 | 82,561,683 |
FN2.tbl %>% kable(caption = "Table FN2") %>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12) %>%
scroll_box(width = "100%")| code | 100 | 211 | 213 | 214 | 215 | 216 | 218 | 221 | 222 | 223 | 230 | 240 | 250 | 290 | 300 | 500 | 600 | other classes | TOTAL |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 211 | 7 | 57 | 9 | 2 | 1 | 1 | 1 | 0 | 0 | 1 | 0 | 2 | 3 | 5 | 0 | 44 | 0 | 2 | 135 |
| 213 | 0 | 5 | 5 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 2 | 0 | 6 | 0 | 0 | 22 |
| 216 | 3 | 1 | 2 | 0 | 1 | 102 | 0 | 20 | 10 | 5 | 2 | 4 | 2 | 1 | 3 | 22 | 1 | 3 | 182 |
| 221 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 59 | 6 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 66 |
| 222 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 14 | 22 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 38 |
| 223 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
| 240 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 4 | 0 | 4 | 0 | 2 | 0 | 1 | 0 | 0 | 12 |
| 250 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 3 |
| 290 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 | 0 | 2 |
| 300 | 31 | 0 | 0 | 0 | 0 | 6 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 218 | 27 | 3 | 4 | 290 |
| 500 | 31 | 9 | 5 | 0 | 0 | 10 | 0 | 0 | 0 | 0 | 1 | 2 | 2 | 1 | 18 | 595 | 2 | 8 | 684 |
| Masked | 130 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 0 | 6 | 0 | 0 | 32 | 57 | 6 | 9 | 243 |
| Total | 203 | 73 | 21 | 3 | 3 | 120 | 1 | 94 | 40 | 14 | 4 | 21 | 7 | 13 | 271 | 752 | 12 | 26 | 1678 |
FN3.tbl %>% kable(caption = "Table FN3") %>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12) | code | label | LUCAS |
|---|---|---|
| 100 | artificial land | A11/A12/A13/A21/A22 |
| 211 | common wheat | B11 |
| 212 | durum wheat | B12 |
| 213 | barley | B13 |
| 214 | rye | B14 |
| 215 | oats | B15 |
| 216 | maize | B16 |
| 217 | rice | B17 |
| 218 | triticale | B18 |
| 219 | other cereals | B19 |
| 221 | potatoes | B21 |
| 222 | sugar beet | B22 |
| 223 | other roots crops | B23 |
| 230 | other non permanent industrial crops | B34/B35/B36/B37 |
| 231 | sunflower | B31 |
| 232 | rape and turnip rape | B32 |
| 233 | soya | B33 |
| 240 | dry pulses, vegetables and flowers | B41/B42/B43/B44/B45 |
| 250 | fodder crops (cereals and leguminous) | B51/B52/B53/B54 |
| 290 | bare arable land | F40 U111/112/113 |
| 300 | woodland and shrubland vegetation (incl. permanent crops) | C10/C21/C22/C23/C31/C32/C33/D10/D20/B71-77/B81-84 |
| 500 | permanent and temporary grasslands | E10/E20/E30/B55 |
| 600 | Bare land and lichens/moss | F10/F20/F20/F40 not agricultural land use |
PCG and TC matrices
Leg <- FN3.tbl
class_codes <- setdiff(names(FN2.tbl), c("code", "other classes", "TOTAL"))
ground_codes <- FN2.tbl %>% pull(code) %>% setdiff(c("Masked", "Total"))
TC <- FN1.tbl %>%
#filter(crop != "Masked") %>%
select(code, value) %>%
mutate_all(as.numeric) %>%
column_to_rownames(var = "code") %>%
as.matrix()
PCG <- FN2.tbl %>%
select(-one_of(c(setdiff(class_codes, ground_codes), "other classes"))) %>%
bind_cols(tibble(`other classes` = FN2.tbl %>%
select_at(.vars =
c(setdiff(class_codes, ground_codes),
"other classes")) %>%
rowSums())) %>%
filter(code != "Total") %>%
#filter(code != "Total", code != "Masked") %>%
select(-TOTAL) %>%
#select(-TOTAL, -`other classes`) %>%
column_to_rownames(var = "code") %>%
mutate_all(as.numeric) %>%
as.matrix()
TC %>%
as.data.frame() %>%
rownames_to_column(var = "code") %>%
mutate(value = format(value, nsmall = 0, big.mark = ",", digits = 10)) %>%
kable(caption = "Table TC", align = "lr") %>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12)| code | value |
|---|---|
| 211 | 20,287,640 |
| 213 | 5,168,348 |
| 216 | 36,809,904 |
| 221 | 10,542,589 |
| 222 | 7,981,893 |
| 223 | 914,813 |
| 240 | 1,807,891 |
| 250 | 443,079 |
| 290 | 472,773 |
| 300 | 72,256,453 |
| 500 | 1,641,199 |
| 1 | 82,561,683 |
PCG %>% kable(caption = "Table PCG") %>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12)| 211 | 213 | 216 | 221 | 222 | 223 | 240 | 250 | 290 | 300 | 500 | other classes | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 211 | 57 | 9 | 1 | 0 | 0 | 1 | 2 | 3 | 5 | 0 | 44 | 13 |
| 213 | 5 | 5 | 0 | 0 | 0 | 0 | 1 | 0 | 2 | 0 | 6 | 3 |
| 216 | 1 | 2 | 102 | 20 | 10 | 5 | 4 | 2 | 1 | 3 | 22 | 10 |
| 221 | 0 | 0 | 0 | 59 | 6 | 0 | 1 | 0 | 0 | 0 | 0 | 0 |
| 222 | 0 | 0 | 0 | 14 | 22 | 1 | 0 | 0 | 0 | 0 | 0 | 1 |
| 223 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
| 240 | 0 | 0 | 0 | 0 | 1 | 4 | 4 | 0 | 2 | 0 | 1 | 0 |
| 250 | 0 | 0 | 1 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 |
| 290 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 2 | 0 | 0 | 0 |
| 300 | 0 | 0 | 6 | 1 | 0 | 0 | 0 | 0 | 0 | 218 | 27 | 38 |
| 500 | 9 | 5 | 10 | 0 | 0 | 0 | 2 | 2 | 1 | 18 | 595 | 42 |
| Masked | 1 | 0 | 0 | 0 | 1 | 1 | 6 | 0 | 0 | 32 | 57 | 145 |
Confest function
Confest <- function(PCG, TC, cloud, replications = 1000){
# PCG matrice de confusion mxn (effectifs) avec classif
# en ligne et terrain en colonne. Attention existe une
# classe nuages terrain
# TrueC: proportions classes pour la totalitè de l'image
# cloud should equal 1 if cloud ground class
#PCG <- t(PCG)
ncg = dim(PCG) # dimension
nc = ncg[1]
ng = ncg[2]
if (cloud > 0){
# elimination of the last column with cloud ground class
PCG <- PCG[, 1:(ng-1)]
ng <- ng - 1
}
Ng <- colSums(PCG)
N <- sum(Ng)
PCG <- PCG/N
PG <- colSums(PCG)
PC <- rowSums(PCG)
# Conditional probabilities of ground categories per given image class
RC <- diag(PC)
PGIC <- solve(RC) %*% PCG
# Conditional probabilities of image categories per given ground class
RG <- diag(PG)
PCIG <- PCG %*% solve(RG)
# Attention nuages
# Direct estimates of ground categories probalilities
# (sum can be greater than one)
TGD <- t(TC) %*% PGIC
# Indirect estimates of ground categories probablitities
X <- PCIG
y <- TC
#mdl <- lm(y ~ X - 1)
mdl <- glmnet(X, y, lambda = 0, lower.limits = 0, intercept = FALSE)
beta <- mdl$beta #mdl$coefficients
# C <- diag(ng)*(-1)
# Aeq <- matrix(rep(1, ng), ncol = 1)
# beq <- 1;
# m = matrix(rep(0, ng), ncol = 1)
# bb <- pracma::lsqlin(X, y, C, m)
# Bootstrapping 100 times
edges <- rbind(matrix(rep(0, ng), nrow = 1), apply(PCIG, 2, cumsum))
b <- NULL
Pcig <- array(dim = c(nc, ng))
for (i in 1:replications){
# Resampling on each ground class sepatately
for (j in 1:ng) {
numbers <- matrix(runif(Ng[j]), ncol = 1)
Eff <- pracma::histc(numbers, edges[,j])$cnt
Pcig[,j] <- Eff[-nc - 1]/Ng[j]
}
X <- Pcig
#mdl <- lm(y ~ X - 1)
#bi <- mdl$coefficients
mdl <- glmnet(X, y, lambda = 0, lower.limits = 0, intercept = FALSE)
beta <- mdl$beta #mdl$coefficients
b <- cbind(b, beta)
}
# Image population classes percentages
TC
# Direct estimation of ground classes percentages
direct <- TGD
TOT <- colSums(direct)
# Bootstrap inverse estimation without constraint
mb <- rowMeans(b)
#TOT <- colSums(mb)
# Bootstrap inverse estimation with constraints: sum=1 and p>0
# mbb <- mean(bb)
# TOT <- colSums(mbb)
# Standard deviations of inverse estimator
stb <- apply(b, 1, function(x) sqrt(var(x)))
#stbb <- sqrt(var(bb))
# Coefficient of variation of inverse estimator
cvb <- apply(b, 1, function(x) 100*sqrt(var(x))/mean(x))
#cvbb <- 100*sqrt(var(bb))./mean(bb)
# Correlation matrix for inverse with constraints
#COR <- cor(bb)
return(list(mb = mb, stb = stb, cvb = cvb, b = b))
}Results
result <- Confest(PCG, TC, cloud = 0, replications = 1000)
format_values <-
function(x, y)
format(x, nsmall = y, big.mark = ",", digits = 10)
result[c("mb", "stb", "cvb")] %>%
as.data.frame() %>%
rownames_to_column(var = "Name") %>%
rename(Mean = mb, 'Deviation' = stb, CV = cvb) %>%
mutate(Mean = format_values(round(Mean), 0),
Deviation = format_values(round(Deviation), 0),
CV = format_values(round(CV,2), 2)) %>%
kable(caption = "Table Bootstrap Results (1000 rounds)", align = "lrrr") %>%
kable_styling(bootstrap_options =
c("striped", "hover", "condensed", "responsive"),
full_width = FALSE, font_size = 12)| Name | Mean | Deviation | CV |
|---|---|---|---|
| V1 | 6,398,297 | 6,065,856 | 94.80 |
| V2 | 623,259 | 3,016,114 | 483.93 |
| V3 | 15,229,800 | 8,700,489 | 57.13 |
| V4 | 13,708,097 | 4,314,375 | 31.47 |
| V5 | 10,085,957 | 4,835,054 | 47.94 |
| V6 | 14,180,464 | 15,700,989 | 110.72 |
| V7 | 21,328,754 | 33,263,070 | 155.95 |
| V8 | 1,981,591 | 6,617,650 | 333.96 |
| V9 | 4,650,887 | 7,964,841 | 171.25 |
| V10 | 68,351,509 | 6,687,291 | 9.78 |
| V11 | 0 | 0 | NaN |
| V12 | 102,264,099 | 24,786,395 | 24.24 |
n_var <- nrow(result$b)
df <- result$b %>% t() %>% as.matrix() %>% as_tibble() %>% pivot_longer(everything(), names_to = "var", values_to = "value") %>% mutate(var = factor(as.integer(str_remove(var, "V")), levels = 1:n_var,labels = paste("Beta", 1:n_var)))
p <- df %>% ggplot(aes(x = value)) + geom_histogram(alpha=0.25, position="identity", bins = 30, aes(color = var, fill = var)) + facet_wrap(. ~ var,nrow = 4) + theme_bw() + guides(fill= FALSE, color = FALSE) + theme(strip.background = element_rect(fill = gray(.95)))
ggplotly(p,tooltip = c("y")) %>%
layout(showlegend = FALSE) %>%
layout(plot_bgcolor='rgba(254, 254, 254, .25)') %>%
layout(paper_bgcolor='#eeeeee') %>%
config(displayModeBar = FALSE)Stay Tuned
Please visit the castlab.org page for latest updates and news. Comments, bug reports and pull requests are always welcome.