Pre-Homogenisation TX Tutorial

Introduction

This tutorial walks you through the process of preparing temperature data for homogenization. We will process station metadata, merge multiple data files, and format the dataset for further analysis.

# Load required libraries
library(dplyr)
library(reshape2)
library(lubridate)

Step 1: Set Up Your Environment

Before running the script, make sure to set your working directory and load any required functions.

# Set working directory (modify this for your local setup)
setwd("~/D/diverse/")
source("R/sorin/function_convert_to_decimals.R")

Step 2: Load and Process Metadata

We read the metadata file and convert latitude/longitude from degrees, minutes, and seconds to decimal format.

# Read metadata
meta.tx <- read.table("tabs/ECA_non-blended_custom_RO/_sources_TX.txt", sep = ",", skip = 23, header = T)

# Convert coordinates and clean station names
meta.tx <- meta.tx |> 
        mutate(Latitudine = as.numeric(sapply(LAT, dms_to_decimal)),
               Longitudine = as.numeric(sapply(LON, dms_to_decimal)),
               SOUNAME = gsub(" ", "", SOUNAME, fixed = TRUE)) |> 
        select(Longitudine, Latitudine, HGHT, SOUID, SOUNAME)

# Save processed metadata
write.csv(meta.tx, "tabs1/metadata_TX.csv", row.names = F)

Step 3: Generate a Complete Time Series

We create a complete time series from 1896 to 2024 for all stations.

# Define date range
dates <- seq(as.Date("1896-01-01"), as.Date("2024-12-31"), by = "day")
station_codes <- meta.tx$SOUID

# Create a full time-station data frame
df <- expand.grid(DATE = dates, SOUID = station_codes) |> 
  arrange(DATE, SOUID)

Step 4: Merge Data Files

We read and merge temperature data from multiple files.

# Get list of temperature files
files <- list.files("tabs/ECA_non-blended_custom_RO", full.names = T)
files.txx <- grep("TX_", files, value = T)

data <- NULL 

# Read and clean temperature data
for(i in seq_along(files.txx)) {
    tt <- read.table(files.txx[i], sep = ",", skip = 21, header = T)
    tt <- tt |> mutate(TX = TX * 0.1, 
                        TX = ifelse(TX <= -999, NA, TX), 
                        DATE = as.Date(as.character(DATE), "%Y%m%d")) |> 
              select(SOUID, DATE, TX)
    data <- rbind(data, tt)
}

Step 5: Ensure Data Completeness

We merge the complete time series with the station data to ensure all time steps are present.

data.j <- df |> left_join(data)

Joining with `by = join_by(DATE, SOUID)`

summary(data.j)

      DATE                SOUID              TX        
 Min.   :1896-01-01   Min.   :100683   Min.   :-28.0   
 1st Qu.:1928-04-02   1st Qu.:100700   1st Qu.:  5.6   
 Median :1960-07-02   Median :107474   Median : 15.1   
 Mean   :1960-07-02   Mean   :111639   Mean   : 14.6   
 3rd Qu.:1992-10-01   3rd Qu.:107481   3rd Qu.: 24.0   
 Max.   :2024-12-31   Max.   :254860   Max.   : 42.7   
                                       NA's   :593369  

# Save merged dataset
write.csv(data.j, "tabs1/ECAD_TX_merged.csv", row.names = F)

Step 6: Filter Data for 2010-2022

We extract data only from 2010 to 2022 for further analysis.

data.j <- data.j |> filter(year(DATE) >= 2010 & year(DATE) <= 2022)

Step 7: Reshape Data for Homogenization

We transform the dataset into a wide format where each station is a column.

data_wide <- dcast(data.j, DATE ~ SOUID, value.var = "TX")
data_values <- data_wide[, -1]  # Remove date column

# Ensure station names match metadata
meta.tx$SOUID <- as.character(meta.tx$SOUID)
meta.tx <- meta.tx[match(names(data_values), meta.tx$SOUID),]
identical(meta.tx$SOUID, names(data_values))

[1] TRUE

cbind(meta.tx$SOUID, names(data_values))

      [,1]     [,2]    
 [1,] "100683" "100683"
 [2,] "100685" "100685"
 [3,] "100687" "100687"
 [4,] "100689" "100689"
 [5,] "100691" "100691"
 [6,] "100693" "100693"
 [7,] "100695" "100695"
 [8,] "100701" "100701"
 [9,] "107469" "107469"
[10,] "107470" "107470"
[11,] "107471" "107471"
[12,] "107472" "107472"
[13,] "107473" "107473"
[14,] "107474" "107474"
[15,] "107475" "107475"
[16,] "107476" "107476"
[17,] "107477" "107477"
[18,] "107478" "107478"
[19,] "107479" "107479"
[20,] "107480" "107480"
[21,] "107481" "107481"
[22,] "107482" "107482"
[23,] "107483" "107483"
[24,] "112323" "112323"
[25,] "112328" "112328"
[26,] "112333" "112333"
[27,] "116381" "116381"
[28,] "254860" "254860"

Step 8: Export Data for Homogenization

We export the processed metadata and dataset in the required formats.

write.table(meta.tx, "tx_2010-2022.est", row.names = FALSE, col.names = FALSE)
write(as.matrix(data_values), 'tx_2010-2022.dat')