ODIN pre-CONA deployment
Gustavo Olivares
## Loading required package: openair
## Loading required package: lazyeval
## Loading required package: dplyr
##
## Attaching package: 'dplyr'
##
## The following object is masked from 'package:stats':
##
## filter
##
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
##
## Loading required package: maps
## Loading required package: reshape2Introduction
The purpose of this analysis is to document the performance of the ODIN during their pre-CONA deployment in Christchurch.
ODIN data
The units ODIN_03-07 were deployed at Rangiora (ODIN_02 failed to record data)
## ODIN_03
odin_03 <- read.table("/home/gustavo/data/CONA/ODIN/deployment/odin_03.data",
header=T, quote="")
#force GMT as the time zone to avoid openair issues with daylight saving switches
#The actual time zone is 'NZST'
odin_03$date=as.POSIXct(paste(odin_03$Date,odin_03$Time),tz='GMT')
odin_03$Time<-NULL
odin_03$Date<-NULL
odin_03$Battery<-5*odin_03$Battery/1024
## ODIN_04
odin_04 <- read.table("/home/gustavo/data/CONA/ODIN/deployment/odin_04.data",
header=T, quote="")
#force GMT as the time zone to avoid openair issues with daylight saving switches
#The actual time zone is 'NZST'
odin_04$date=as.POSIXct(paste(odin_04$Date,odin_04$Time),tz='GMT')
odin_04$Time<-NULL
odin_04$Date<-NULL
odin_04$Battery<-5*odin_04$Battery/1024
## ODIN_05
odin_05 <- read.table("/home/gustavo/data/CONA/ODIN/deployment/odin_05.data",
header=T, quote="")
#force GMT as the time zone to avoid openair issues with daylight saving switches
#The actual time zone is 'NZST'
odin_05$date=as.POSIXct(paste(odin_05$Date,odin_05$Time),tz='GMT')
odin_05$Time<-NULL
odin_05$Date<-NULL
odin_05$Battery<-5*odin_05$Battery/1024
## ODIN_06
odin_06 <- read.table("/home/gustavo/data/CONA/ODIN/deployment/odin_06.data",
header=T, quote="")
#force GMT as the time zone to avoid openair issues with daylight saving switches
#The actual time zone is 'NZST'
odin_06$date=as.POSIXct(paste(odin_06$Date,odin_06$Time),tz='GMT')
odin_06$Time<-NULL
odin_06$Date<-NULL
odin_06$Battery<-5*odin_06$Battery/1024
## ODIN_07
odin_07 <- read.table("/home/gustavo/data/CONA/ODIN/deployment/odin_07.data",
header=T, quote="")
#force GMT as the time zone to avoid openair issues with daylight saving switches
#The actual time zone is 'NZST'
odin_07$date=as.POSIXct(paste(odin_07$Date,odin_07$Time),tz='GMT')
odin_07$Time<-NULL
odin_07$Date<-NULL
odin_07$Battery<-5*odin_07$Battery/1024ECan data
The data from the Rangiora site was obtained from Environment Canterbury’s data catalogue and then corrected for proper date handling.
download.file(url = "http://data.ecan.govt.nz/data/29/Air/Air%20quality%20data%20for%20a%20monitored%20site%20(Hourly)/CSV?SiteId=5&StartDate=11%2F08%2F2015&EndDate=16%2F08%2F2015",destfile = "ecan_data.csv",method = "curl")
system("sed -i 's/a.m./AM/g' ecan_data.csv")
system("sed -i 's/p.m./PM/g' ecan_data.csv")
ecan_data_raw <- read.csv("ecan_data.csv",stringsAsFactors=FALSE)
ecan_data_raw$date<-as.POSIXct(ecan_data_raw$DateTime,format = "%d/%m/%Y %I:%M:%S %p",tz='GMT')
ecan_data<-as.data.frame(ecan_data_raw[,c('date','PM10.FDMS','Temperature..2m')])
names(ecan_data)<-c('date','PM10.FDMS','Temperature..1m')Merging the data
ECan’s data was provided as 10 minute values while ODIN reports every 1 minute so before merging the data, the timebase must be homogenized
names(odin_03)<-c('Dust.03','Humidity.03','Temperature.03','Battery.03','date')
names(odin_04)<-c('Dust.04','Humidity.04','Temperature.04','Battery.04','date')
names(odin_05)<-c('Dust.05','Humidity.05','Temperature.05','Battery.05','date')
names(odin_06)<-c('Dust.06','Humidity.06','Temperature.06','Battery.06','date')
names(odin_07)<-c('Dust.07','Humidity.07','Temperature.07','Battery.07','date')
odin <- merge(odin_03,odin_04,by='date',all=TRUE)
odin <- merge(odin,odin_05,by='date',all=TRUE)
odin <- merge(odin,odin_06,by='date',all=TRUE)
odin <- merge(odin,odin_07,by='date',all=TRUE)
odin.10min<-timeAverage(selectByDate(odin,start = '2015-08-10',end = '2015-08-16'),avg.time='10 min')
all_merged.10min<-merge(odin.10min,ecan_data,by='date',all=TRUE)Time sync
lag_test=ccf(all_merged.10min$Temperature.07,
all_merged.10min$Temperature..1m,
na.action=na.pass,
lag.max=100,
type='covariance',
ylab='Correlation',
main='Temperature correlation as function of clock lag')
odin_lag=lag_test$lag[which.max(lag_test$acf)]ECan’s record is behind by -10 minutes with respect to ODIN data.
The correction was applied to the ODIN data as follows:
odin$date=odin$date-odin_lag*10*60
odin.10min<-timeAverage(selectByDate(odin,start = '2015-08-10',end = '2015-08-16'),avg.time='10 min')
all_merged.10min<-merge(odin.10min,ecan_data,by='date',all=TRUE)
lag_test=ccf(all_merged.10min$Temperature.03,
all_merged.10min$Temperature..1m,
na.action=na.pass,
lag.max=100,
type='covariance',
ylab='Correlation',
main='Temperature correlation as function of clock lag')
Remove drift from ODIN raw data
It has been documented that the dust sensors suffer from significant drift, therefore a linear fit of the baseline drift is estimated.
# Estimate the baseline from a simple linear regression
all_merged.10min$ODIN_drift.03<-predict(lm(all_merged.10min$Dust.03~seq(all_merged.10min$Dust.03)),newdata = all_merged.10min)
all_merged.10min$ODIN_drift.04<-predict(lm(all_merged.10min$Dust.04~seq(all_merged.10min$Dust.04)),newdata = all_merged.10min)
all_merged.10min$ODIN_drift.05<-predict(lm(all_merged.10min$Dust.05~seq(all_merged.10min$Dust.05)),newdata = all_merged.10min)
all_merged.10min$ODIN_drift.06<-predict(lm(all_merged.10min$Dust.06~seq(all_merged.10min$Dust.06)),newdata = all_merged.10min)
all_merged.10min$ODIN_drift.07<-predict(lm(all_merged.10min$Dust.07~seq(all_merged.10min$Dust.07)),newdata = all_merged.10min)
# Remove the baseline drift from the raw ODIN data
all_merged.10min$Dust.03.raw <- all_merged.10min$Dust.03
all_merged.10min$Dust.03.detrend<-all_merged.10min$Dust.03.raw - all_merged.10min$ODIN_drift.03
all_merged.10min$Dust.04.raw <- all_merged.10min$Dust.04
all_merged.10min$Dust.04.detrend<-all_merged.10min$Dust.04.raw - all_merged.10min$ODIN_drift.04
all_merged.10min$Dust.05.raw <- all_merged.10min$Dust.05
all_merged.10min$Dust.05.detrend<-all_merged.10min$Dust.05.raw - all_merged.10min$ODIN_drift.05
all_merged.10min$Dust.06.raw <- all_merged.10min$Dust.06
all_merged.10min$Dust.06.detrend<-all_merged.10min$Dust.06.raw - all_merged.10min$ODIN_drift.06
all_merged.10min$Dust.07.raw <- all_merged.10min$Dust.07
all_merged.10min$Dust.07.detrend<-all_merged.10min$Dust.07.raw - all_merged.10min$ODIN_drift.07
## Testing not correcting drift
all_merged.10min$Dust.03.detrend<-all_merged.10min$Dust.03.raw
all_merged.10min$Dust.04.detrend<-all_merged.10min$Dust.04.raw
all_merged.10min$Dust.05.detrend<-all_merged.10min$Dust.05.raw
all_merged.10min$Dust.06.detrend<-all_merged.10min$Dust.06.raw
all_merged.10min$Dust.07.detrend<-all_merged.10min$Dust.07.rawCalculate the temperature interference
First we divide the temperature range for each unit.
all_merged.10min$Temperature.03.bin<-cut(all_merged.10min$Temperature.03,breaks = c(0,5,10,15,20,25),labels = c('2.5','7.5','12.5','17.5','22.5'))
all_merged.10min$Temperature.04.bin<-cut(all_merged.10min$Temperature.04,breaks = c(0,5,10,15,20,25),labels = c('2.5','7.5','12.5','17.5','22.5'))
all_merged.10min$Temperature.05.bin<-cut(all_merged.10min$Temperature.05,breaks = c(0,5,10,15,20,25),labels = c('2.5','7.5','12.5','17.5','22.5'))
all_merged.10min$Temperature.06.bin<-cut(all_merged.10min$Temperature.06,breaks = c(0,5,10,15,20,25),labels = c('2.5','7.5','12.5','17.5','22.5'))
all_merged.10min$Temperature.07.bin<-cut(all_merged.10min$Temperature.07,breaks = c(0,5,10,15,20,25),labels = c('2.5','7.5','12.5','17.5','22.5'))
Temp <- c(2.5,7.5,12.5,17.5,22.5)
Dust.03<-tapply(all_merged.10min$Dust.03.detrend,all_merged.10min$Temperature.03.bin,quantile,0.25)
Dust.04<-tapply(all_merged.10min$Dust.04.detrend,all_merged.10min$Temperature.04.bin,quantile,0.25)
Dust.05<-tapply(all_merged.10min$Dust.05.detrend,all_merged.10min$Temperature.05.bin,quantile,0.25)
Dust.06<-tapply(all_merged.10min$Dust.06.detrend,all_merged.10min$Temperature.06.bin,quantile,0.25)
Dust.07<-tapply(all_merged.10min$Dust.07.detrend,all_merged.10min$Temperature.07.bin,quantile,0.25)
TC_Dust.03 <- data.frame(Dust.03.detrend = Dust.03,Temperature.03 = Temp)
TC_Dust.04 <- data.frame(Dust.04.detrend = Dust.04,Temperature.04 = Temp)
TC_Dust.05 <- data.frame(Dust.05.detrend = Dust.05,Temperature.05 = Temp)
TC_Dust.06 <- data.frame(Dust.06.detrend = Dust.06,Temperature.06 = Temp)
TC_Dust.07 <- data.frame(Dust.07.detrend = Dust.07,Temperature.07 = Temp)Now we calculate the linear regression for the minimum dust response in each temperature bin and subtract it from the detrended data
summary(odin.03_T<-lm(data = TC_Dust.03,Dust.03.detrend~Temperature.03))##
## Call:
## lm(formula = Dust.03.detrend ~ Temperature.03, data = TC_Dust.03)
##
## Residuals:
## 2.5 7.5 12.5 17.5 22.5
## 0.590 1.428 -2.185 -2.273 2.440
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 357.1412 2.2420 159.30 5.45e-07 ***
## Temperature.03 5.0075 0.1561 32.08 6.66e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 2.468 on 3 degrees of freedom
## Multiple R-squared: 0.9971, Adjusted R-squared: 0.9961
## F-statistic: 1029 on 1 and 3 DF, p-value: 6.659e-05summary(odin.04_T<-lm(data = TC_Dust.04,Dust.04.detrend~Temperature.04))##
## Call:
## lm(formula = Dust.04.detrend ~ Temperature.04, data = TC_Dust.04)
##
## Residuals:
## 2.5 7.5 12.5 17.5 22.5
## -1.000 1.025 0.375 0.175 -0.575
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 33.51250 0.83782 40.000 3.44e-05 ***
## Temperature.04 0.39500 0.05834 6.771 0.00658 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.9224 on 3 degrees of freedom
## Multiple R-squared: 0.9386, Adjusted R-squared: 0.9181
## F-statistic: 45.84 on 1 and 3 DF, p-value: 0.006583summary(odin.05_T<-lm(data = TC_Dust.05,Dust.05.detrend~Temperature.05))##
## Call:
## lm(formula = Dust.05.detrend ~ Temperature.05, data = TC_Dust.05)
##
## Residuals:
## 2.5 7.5 12.5 17.5 22.5
## -1.00 1.15 0.30 -0.05 -0.40
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 43.82500 0.84150 52.080 1.56e-05 ***
## Temperature.05 0.47000 0.05859 8.021 0.00405 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 0.9265 on 3 degrees of freedom
## Multiple R-squared: 0.9554, Adjusted R-squared: 0.9406
## F-statistic: 64.34 on 1 and 3 DF, p-value: 0.004045summary(odin.06_T<-lm(data = TC_Dust.06,Dust.06.detrend~Temperature.06))##
## Call:
## lm(formula = Dust.06.detrend ~ Temperature.06, data = TC_Dust.06)
##
## Residuals:
## 2.5 7.5 12.5 17.5
## -1.1 1.8 -0.3 -0.4
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 50.8000 1.5708 32.340 0.000955 ***
## Temperature.06 0.5200 0.1371 3.792 0.063025 .
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 1.533 on 2 degrees of freedom
## (1 observation deleted due to missingness)
## Multiple R-squared: 0.8779, Adjusted R-squared: 0.8169
## F-statistic: 14.38 on 1 and 2 DF, p-value: 0.06303summary(odin.07_T<-lm(data = TC_Dust.07,Dust.07.detrend~Temperature.07))##
## Call:
## lm(formula = Dust.07.detrend ~ Temperature.07, data = TC_Dust.07)
##
## Residuals:
## 2.5 7.5 12.5 17.5 22.5
## -1.6 1.5 0.6 0.7 -1.2
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 52.4000 1.3973 37.500 4.17e-05 ***
## Temperature.07 0.6800 0.0973 6.989 0.00601 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 1.538 on 3 degrees of freedom
## Multiple R-squared: 0.9421, Adjusted R-squared: 0.9228
## F-statistic: 48.85 on 1 and 3 DF, p-value: 0.006013all_merged.10min$Dust.03.corr <- all_merged.10min$Dust.03.detrend - predict(odin.03_T,newdata = all_merged.10min)
all_merged.10min$Dust.04.corr <- all_merged.10min$Dust.04.detrend - predict(odin.04_T,newdata = all_merged.10min)
all_merged.10min$Dust.05.corr <- all_merged.10min$Dust.05.detrend - predict(odin.05_T,newdata = all_merged.10min)
all_merged.10min$Dust.06.corr <- all_merged.10min$Dust.06.detrend - predict(odin.06_T,newdata = all_merged.10min)
all_merged.10min$Dust.07.corr <- all_merged.10min$Dust.07.detrend - predict(odin.07_T,newdata = all_merged.10min)Dust performance using ECan data for calibration
With ECan’s PM data available, a more accurate calibration can be applied to the corrected Dust signal from ODIN.
\(Dust_{calibrated}=A*Dust_{corrected}+B\)
Full dataset 1 hour PM\(_{2.5}\) fdms
all_merged.1hr<-timeAverage(all_merged.10min,avg.time='1 hour')
summary(odin3.lm.full.1hr.pm2.5<-
lm(data=all_merged.1hr,PM10.FDMS~
Dust.03.corr))##
## Call:
## lm(formula = PM10.FDMS ~ Dust.03.corr, data = all_merged.1hr)
##
## Residuals:
## Min 1Q Median 3Q Max
## -19.818 -9.906 -3.076 3.261 113.002
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 18.8764 2.2810 8.276 1.84e-12 ***
## Dust.03.corr 0.9548 0.3734 2.557 0.0124 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 18 on 83 degrees of freedom
## (36 observations deleted due to missingness)
## Multiple R-squared: 0.07302, Adjusted R-squared: 0.06185
## F-statistic: 6.538 on 1 and 83 DF, p-value: 0.01238summary(odin4.lm.full.1hr.pm2.5<-
lm(data=all_merged.1hr,PM10.FDMS~
Dust.04.corr))##
## Call:
## lm(formula = PM10.FDMS ~ Dust.04.corr, data = all_merged.1hr)
##
## Residuals:
## Min 1Q Median 3Q Max
## -18.463 -8.351 -3.138 3.020 114.744
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 21.437 2.714 7.898 2.38e-11 ***
## Dust.04.corr 2.701 1.930 1.399 0.166
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 19.15 on 72 degrees of freedom
## (47 observations deleted due to missingness)
## Multiple R-squared: 0.02646, Adjusted R-squared: 0.01294
## F-statistic: 1.957 on 1 and 72 DF, p-value: 0.1661summary(odin5.lm.full.1hr.pm2.5<-
lm(data=all_merged.1hr,PM10.FDMS~
Dust.05.corr))##
## Call:
## lm(formula = PM10.FDMS ~ Dust.05.corr, data = all_merged.1hr)
##
## Residuals:
## Min 1Q Median 3Q Max
## -15.783 -8.402 -3.568 1.547 113.056
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 20.557 2.652 7.751 4.46e-11 ***
## Dust.05.corr 3.196 1.561 2.047 0.0443 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 18.86 on 72 degrees of freedom
## (47 observations deleted due to missingness)
## Multiple R-squared: 0.05498, Adjusted R-squared: 0.04185
## F-statistic: 4.189 on 1 and 72 DF, p-value: 0.04435summary(odin6.lm.full.1hr.pm2.5<-
lm(data=all_merged.1hr,PM10.FDMS~
Dust.06.corr))##
## Call:
## lm(formula = PM10.FDMS ~ Dust.06.corr, data = all_merged.1hr)
##
## Residuals:
## Min 1Q Median 3Q Max
## -14.055 -7.643 -3.648 0.713 116.088
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 19.379 2.119 9.146 3.34e-14 ***
## Dust.06.corr 3.328 1.151 2.892 0.00488 **
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 17.82 on 83 degrees of freedom
## (36 observations deleted due to missingness)
## Multiple R-squared: 0.09157, Adjusted R-squared: 0.08062
## F-statistic: 8.366 on 1 and 83 DF, p-value: 0.00488summary(odin7.lm.full.1hr.pm2.5<-
lm(data=all_merged.1hr,PM10.FDMS~
Dust.07.corr))##
## Call:
## lm(formula = PM10.FDMS ~ Dust.07.corr, data = all_merged.1hr)
##
## Residuals:
## Min 1Q Median 3Q Max
## -17.470 -8.643 -2.718 2.858 113.226
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 18.730 2.328 8.046 5.26e-12 ***
## Dust.07.corr 2.562 1.023 2.504 0.0143 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 18.03 on 83 degrees of freedom
## (36 observations deleted due to missingness)
## Multiple R-squared: 0.07022, Adjusted R-squared: 0.05901
## F-statistic: 6.268 on 1 and 83 DF, p-value: 0.01425Calibrated Dust
all_merged.1hr$Dust.03.cal<-predict(odin3.lm.full.1hr.pm2.5,newdata = all_merged.1hr)
all_merged.1hr$Dust.04.cal<-predict(odin4.lm.full.1hr.pm2.5,newdata = all_merged.1hr)
all_merged.1hr$Dust.05.cal<-predict(odin5.lm.full.1hr.pm2.5,newdata = all_merged.1hr)
all_merged.1hr$Dust.06.cal<-predict(odin6.lm.full.1hr.pm2.5,newdata = all_merged.1hr)
all_merged.1hr$Dust.07.cal<-predict(odin7.lm.full.1hr.pm2.5,newdata = all_merged.1hr)
timePlot(all_merged.1hr,pollutant = c('PM10.FDMS',
'Dust.03.corr',
'Dust.04.corr',
'Dust.05.corr',
'Dust.06.corr',
'Dust.07.corr')
,group = TRUE)
timePlot(all_merged.1hr,pollutant = c('PM10.FDMS',
'Dust.03.cal',
'Dust.04.cal',
'Dust.05.cal',
'Dust.06.cal',
'Dust.07.cal')
,group = TRUE)
timeVariation(all_merged.1hr,pollutant = c('PM10.FDMS',
'Dust.03.cal',
'Dust.04.cal',
'Dust.05.cal',
'Dust.06.cal',
'Dust.07.cal'))
timeVariation(all_merged.1hr,pollutant = c('PM10.FDMS',
'Dust.03.corr',
'Dust.04.corr',
'Dust.05.corr',
'Dust.06.corr',
'Dust.07.corr'))
timeVariation(all_merged.1hr,pollutant = c('Dust.03.corr',
'Dust.04.corr',
'Dust.05.corr',
'Dust.06.corr',
'Dust.07.corr'))