#=========== Quality checking of HF108 - Air and soil temperature data of the Hemlock Removal Experiment in the Simes Tract in Harvard Forest ==========

#Date: 24 January 2012
#Authors: Elizabeth Nicoll & Monique Lustenhouwer
#Lead: Aaron Ellison
#Data updated until: DECEMBER 2010

#Explanation of method used: see hf108-06-filter-method.pdf
#This script consists of 2 parts:
	#PART 1: MAIN FILTER FOR THE WHOLE DATASET
	#PART 2: ADDITIONAL FILTERS FOR WINTER 2007 / 2008

#============================ PART 1: MAIN FILTER FOR THE WHOLE DATASET =======================================================
#Graphical analyis of min-max values; determine cut off value of outliers

#get data
simes<-read.csv("hf108-03-air-soil-temp.csv")
summary(simes)
par(mfrow = c(2,1)) #graphics parameters

## AIR TEMPERATURES ##

#air 1
plot(simes$meanair1, main="meanair1, raw data", ylab="meanair1")
plot(simes$maxair1-simes$minair1, main="maxair1 - minair1, raw data", ylab="maxair1 - minair1")
summary(simes$maxair1-simes$minair1)

meanair1<-ifelse(simes$maxair1-simes$minair1>10,NA,ifelse(simes$maxair1-simes$minair1<0,NA,simes$meanair1))
summary(meanair1)
maxair1<-ifelse(simes$maxair1-simes$minair1>10,NA,ifelse(simes$maxair1-simes$minair1<0,NA,simes$maxair1))
minair1<-ifelse(simes$maxair1-simes$minair1>10,NA,ifelse(simes$maxair1-simes$minair1<0,NA,simes$minair1))
 plot(meanair1, main="meanair1, filtered data", ylab="meanair1"  )
 plot(maxair1-minair1, main="maxair1 - minair1, filtered data", ylab="maxair1 - minair1")
summary(maxair1-minair1) 

#air 2
plot(simes$meanair2)
plot(simes$maxair2-simes$minair2)
summary(simes$maxair2-simes$minair2)

meanair2<-ifelse(simes$maxair2-simes$minair2>15,NA,ifelse(simes$maxair2-simes$minair2<0,NA,simes$meanair2))
summary(meanair2)
maxair2<-ifelse(simes$maxair2-simes$minair2>15,NA,ifelse(simes$maxair2-simes$minair2<0,NA,simes$maxair2))
minair2<-ifelse(simes$maxair2-simes$minair2>15,NA,ifelse(simes$maxair2-simes$minair2<0,NA,simes$minair2))
 plot(meanair2  )
 plot(maxair2-minair2)
summary(maxair2-minair2)

#air 3
plot(simes$meanair3, main="meanair3, raw data", ylab="meanair3")
plot(simes$maxair3-simes$minair3, main="maxair3 - minair3, raw data", ylab="maxair3 - minair3")
summary(simes$maxair3-simes$minair3)

meanair3<-ifelse(simes$maxair3-simes$minair3>8,NA,ifelse(simes$maxair3-simes$minair3<0,NA,simes$meanair3))
summary(meanair3)
maxair3<-ifelse(simes$maxair3-simes$minair3>8,NA,ifelse(simes$maxair3-simes$minair3<0,NA,simes$maxair3))
minair3<-ifelse(simes$maxair3-simes$minair3>8,NA,ifelse(simes$maxair3-simes$minair3<0,NA,simes$minair3))
 plot(meanair3, main="meanair3, filtered data", ylab="meanair3"  )
 plot(maxair3-minair3, main="maxair3 - minair3, filtered data", ylab="maxair3 - minair3")
summary(maxair3-minair3)

#air 4
plot(simes$meanair4)
plot(simes$maxair4-simes$minair4)
summary(simes$maxair4-simes$minair4)

meanair4<-ifelse(simes$maxair4-simes$minair4>18,NA,ifelse(simes$maxair4-simes$minair4<0,NA,simes$meanair4))
summary(meanair4)
maxair4<-ifelse(simes$maxair4-simes$minair4>18,NA,ifelse(simes$maxair4-simes$minair4<0,NA,simes$maxair4))
minair4<-ifelse(simes$maxair4-simes$minair4>18,NA,ifelse(simes$maxair4-simes$minair4<0,NA,simes$minair4))
 plot(meanair4  )
 plot(maxair4-minair4)
summary(maxair4-minair4)

#air 5
plot(simes$meanair5)
plot(simes$maxair5-simes$minair5)
summary(simes$maxair5-simes$minair5)

meanair5<-ifelse(simes$maxair5-simes$minair5>15,NA,ifelse(simes$maxair5-simes$minair5<0,NA,simes$meanair5))
summary(meanair5)
maxair5<-ifelse(simes$maxair5-simes$minair5>15,NA,ifelse(simes$maxair5-simes$minair5<0,NA,simes$maxair5))
minair5<-ifelse(simes$maxair5-simes$minair5>15,NA,ifelse(simes$maxair5-simes$minair5<0,NA,simes$minair5))
 plot(meanair5  )
 plot(maxair5-minair5)
summary(maxair5-minair5)

#air 6
plot(simes$meanair6)
plot(simes$maxair6-simes$minair6)
summary(simes$maxair6-simes$minair6)

meanair6<-ifelse(simes$maxair6-simes$minair6>8,NA,ifelse(simes$maxair6-simes$minair6<0,NA,simes$meanair6))
summary(meanair6)
maxair6<-ifelse(simes$maxair6-simes$minair6>8,NA,ifelse(simes$maxair6-simes$minair6<0,NA,simes$maxair6))
minair6<-ifelse(simes$maxair6-simes$minair6>8,NA,ifelse(simes$maxair6-simes$minair6<0,NA,simes$minair6))
 plot(meanair6  )
 plot(maxair6-minair6)
summary(maxair6-minair6)

#air 7
plot(simes$meanair7)
plot(simes$maxair7-simes$minair7)
summary(simes$maxair7-simes$minair7)

meanair7<-ifelse(simes$maxair7-simes$minair7>10,NA,ifelse(simes$maxair7-simes$minair7<0,NA,simes$meanair7))
summary(meanair7)
maxair7<-ifelse(simes$maxair7-simes$minair7>10,NA,ifelse(simes$maxair7-simes$minair7<0,NA,simes$maxair7))
minair7<-ifelse(simes$maxair7-simes$minair7>10,NA,ifelse(simes$maxair7-simes$minair7<0,NA,simes$minair7))
 plot(meanair7  )
 plot(maxair7-minair7)
summary(maxair7-minair7)

#air 8
plot(simes$meanair8)
plot(simes$maxair8-simes$minair8)
summary(simes$maxair8-simes$minair8)

meanair8<-ifelse(simes$maxair8-simes$minair8>10,NA,ifelse(simes$maxair8-simes$minair8<0,NA,simes$meanair8))
summary(meanair8)
maxair8<-ifelse(simes$maxair8-simes$minair8>10,NA,ifelse(simes$maxair8-simes$minair8<0,NA,simes$maxair8))
minair8<-ifelse(simes$maxair8-simes$minair8>10,NA,ifelse(simes$maxair8-simes$minair8<0,NA,simes$minair8))
 plot(meanair8  )
 plot(maxair8-minair8)
summary(maxair8-minair8)

## SOIL TEMPERATURES ##

#In general, soil temperatures will not fluxuate as quickly as air temperatures, so the max-min value will be lower - usually 0 to 3 for mineral soil and 0 to 5 for organic soil.
#Max and Min data does not appear for plot 1 until 2008, because min minerailsoil1 has NA in the datafile until 2008. max mineralsoil1 is ok.

#mineral soil 2
plot(simes$meanmineralsoil2 )
plot(simes$maxmineralsoil2 -simes$minmineralsoil2 )
summary(simes$maxmineralsoil2 -simes$minmineralsoil2 )

meanmineralsoil2 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$meanmineralsoil2 ))
summary(meanmineralsoil2 )
maxmineralsoil2 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$maxmineralsoil2 ))
minmineralsoil2 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$minmineralsoil2 ))
 plot(meanmineralsoil2   )
 plot(maxmineralsoil2 -minmineralsoil2 )

#mineral soil 1
#We used the filters of plot 2 for plot 1, because the minimum mineral soil temperatures were missing until 2008.

plot(simes$meanmineralsoil1 )
plot(simes$maxmineralsoil1 -simes$minmineralsoil1 )
summary(simes$maxmineralsoil1 -simes$minmineralsoil1 )

#meanmineralsoil11 <-ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 >5,NA,ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 <0,NA,simes$meanmineralsoil1 ))
#summary(meanmineralsoil1 )
#maxmineralsoil11 <-ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 >5,NA,ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 <0,NA,simes$maxmineralsoil1 ))
#minmineralsoil11 <-ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 >5,NA,ifelse(simes$maxmineralsoil1 -simes$minmineralsoil1 <0,NA,simes$minmineralsoil1 ))
# plot(meanmineralsoil11   )
# plot(maxmineralsoil11 -minmineralsoil11 )

meanmineralsoil1 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$meanmineralsoil1 ))
summary(meanmineralsoil1 )
maxmineralsoil1 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$maxmineralsoil1 ))
minmineralsoil1 <-ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 >2.5,NA,ifelse(simes$maxmineralsoil2 -simes$minmineralsoil2 <0,NA,simes$minmineralsoil1 ))
 plot(meanmineralsoil1   )
 plot(maxmineralsoil1 -minmineralsoil1 )

#mineral soil 3
plot(simes$meanmineralsoil3 )
plot(simes$maxmineralsoil3 -simes$minmineralsoil3 )
summary(simes$maxmineralsoil3 -simes$minmineralsoil3 )

meanmineralsoil3 <-ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 >5,NA,ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 <0,NA,simes$meanmineralsoil3 ))
summary(meanmineralsoil3 )
maxmineralsoil3 <-ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 >5,NA,ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 <0,NA,simes$maxmineralsoil3 ))
minmineralsoil3 <-ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 >5,NA,ifelse(simes$maxmineralsoil3 -simes$minmineralsoil3 <0,NA,simes$minmineralsoil3 ))
 plot(meanmineralsoil3   )
 plot(maxmineralsoil3 -minmineralsoil3 )

##There are lots of bad values for winter 2007/2008 plots 4, 5 and 6. See extra filters in part 2 of the script
#mineral soil 4
plot(simes$meanmineralsoil4 )
plot(simes$maxmineralsoil4 -simes$minmineralsoil4 )
summary(simes$maxmineralsoil4 -simes$minmineralsoil4 )

meanmineralsoil4 <-ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 >5,NA,ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 <0,NA,simes$meanmineralsoil4 ))
summary(meanmineralsoil4 )
maxmineralsoil4 <-ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 >5,NA,ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 <0,NA,simes$maxmineralsoil4 ))
minmineralsoil4 <-ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 >5,NA,ifelse(simes$maxmineralsoil4 -simes$minmineralsoil4 <0,NA,simes$minmineralsoil4 ))
 plot(meanmineralsoil4   )
 plot(maxmineralsoil4 -minmineralsoil4 )

#mineral soil 5
plot(simes$meanmineralsoil5 )
plot(simes$maxmineralsoil5 -simes$minmineralsoil5 )
summary(simes$maxmineralsoil5 -simes$minmineralsoil5 )

meanmineralsoil5 <-ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 >5,NA,ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 <0,NA,simes$meanmineralsoil5 ))
summary(meanmineralsoil5 )
maxmineralsoil5 <-ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 >5,NA,ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 <0,NA,simes$maxmineralsoil5 ))
minmineralsoil5 <-ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 >5,NA,ifelse(simes$maxmineralsoil5 -simes$minmineralsoil5 <0,NA,simes$minmineralsoil5 ))
 plot(meanmineralsoil5   )
 plot(maxmineralsoil5 -minmineralsoil5 )

#mineral soil 6
plot(simes$meanmineralsoil6 )
plot(simes$maxmineralsoil6 -simes$minmineralsoil6 )
summary(simes$maxmineralsoil6 -simes$minmineralsoil6 )

meanmineralsoil6 <-ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 >6,NA,ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 <0,NA,simes$meanmineralsoil6 ))
summary(meanmineralsoil6 )
maxmineralsoil6 <-ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 >6,NA,ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 <0,NA,simes$maxmineralsoil6 ))
minmineralsoil6 <-ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 >6,NA,ifelse(simes$maxmineralsoil6 -simes$minmineralsoil6 <0,NA,simes$minmineralsoil6 ))
 plot(meanmineralsoil6   )
 plot(maxmineralsoil6 -minmineralsoil6 )

#mineral soil 7
plot(simes$meanmineralsoil7 )
plot(simes$maxmineralsoil7 -simes$minmineralsoil7 )
summary(simes$maxmineralsoil7 -simes$minmineralsoil7 )

meanmineralsoil7 <-ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 >3,NA,ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 <0,NA,simes$meanmineralsoil7 ))
summary(meanmineralsoil7 )
maxmineralsoil7 <-ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 >3,NA,ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 <0,NA,simes$maxmineralsoil7 ))
minmineralsoil7 <-ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 >3,NA,ifelse(simes$maxmineralsoil7 -simes$minmineralsoil7 <0,NA,simes$minmineralsoil7 ))
 plot(meanmineralsoil7   )
 plot(maxmineralsoil7 -minmineralsoil7 )

#mineral soil 8
plot(simes$meanmineralsoil8 )
plot(simes$maxmineralsoil8 -simes$minmineralsoil8 )
summary(simes$maxmineralsoil8 -simes$minmineralsoil8 )

meanmineralsoil8 <-ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 >3,NA,ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 <0,NA,simes$meanmineralsoil8 ))
summary(meanmineralsoil8 )
maxmineralsoil8 <-ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 >3,NA,ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 <0,NA,simes$maxmineralsoil8 ))
minmineralsoil8 <-ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 >3,NA,ifelse(simes$maxmineralsoil8 -simes$minmineralsoil8 <0,NA,simes$minmineralsoil8 ))
 plot(meanmineralsoil8   )
 plot(maxmineralsoil8 -minmineralsoil8 )

## ORGANIC SOIL ##

#organic soil 1
plot(simes$meanorgsoil1  )
plot(simes$maxorgsoil1  -simes$minorgsoil1  )
summary(simes$maxorgsoil1  -simes$minorgsoil1  )

meanorgsoil1  <-ifelse(simes$maxorgsoil1  -simes$minorgsoil1  >8,NA,ifelse(simes$maxorgsoil1  -simes$minorgsoil1  <0,NA,simes$meanorgsoil1  ))
summary(meanorgsoil1  )
maxorgsoil1  <-ifelse(simes$maxorgsoil1  -simes$minorgsoil1  >8,NA,ifelse(simes$maxorgsoil1  -simes$minorgsoil1  <0,NA,simes$maxorgsoil1  ))
minorgsoil1  <-ifelse(simes$maxorgsoil1  -simes$minorgsoil1  >8,NA,ifelse(simes$maxorgsoil1  -simes$minorgsoil1  <0,NA,simes$minorgsoil1  ))
 plot(meanorgsoil1    )
 plot(maxorgsoil1  -minorgsoil1  )

#organic soil 2
plot(simes$meanorgsoil2  )
plot(simes$maxorgsoil2  -simes$minorgsoil2  )
summary(simes$maxorgsoil2  -simes$minorgsoil2  )

meanorgsoil2  <-ifelse(simes$maxorgsoil2  -simes$minorgsoil2  >8,NA,ifelse(simes$maxorgsoil2  -simes$minorgsoil2  <0,NA,simes$meanorgsoil2  ))
summary(meanorgsoil2  )
maxorgsoil2  <-ifelse(simes$maxorgsoil2  -simes$minorgsoil2  >8,NA,ifelse(simes$maxorgsoil2  -simes$minorgsoil2  <0,NA,simes$maxorgsoil2  ))
minorgsoil2  <-ifelse(simes$maxorgsoil2  -simes$minorgsoil2  >8,NA,ifelse(simes$maxorgsoil2  -simes$minorgsoil2  <0,NA,simes$minorgsoil2  ))
 plot(meanorgsoil2    )
 plot(maxorgsoil2  -minorgsoil2  )

#organic soil 3
plot(simes$meanorgsoil3  )
plot(simes$maxorgsoil3  -simes$minorgsoil3  )
summary(simes$maxorgsoil3  -simes$minorgsoil3  )

meanorgsoil3  <-ifelse(simes$maxorgsoil3  -simes$minorgsoil3  >8,NA,ifelse(simes$maxorgsoil3  -simes$minorgsoil3  <0,NA,simes$meanorgsoil3  ))
summary(meanorgsoil3  )
maxorgsoil3  <-ifelse(simes$maxorgsoil3  -simes$minorgsoil3  >8,NA,ifelse(simes$maxorgsoil3  -simes$minorgsoil3  <0,NA,simes$maxorgsoil3  ))
minorgsoil3  <-ifelse(simes$maxorgsoil3  -simes$minorgsoil3  >8,NA,ifelse(simes$maxorgsoil3  -simes$minorgsoil3  <0,NA,simes$minorgsoil3  ))
 plot(meanorgsoil3    )
 plot(maxorgsoil3  -minorgsoil3  )

#organic soil 4
plot(simes$meanorgsoil4  )
plot(simes$maxorgsoil4  -simes$minorgsoil4  )
summary(simes$maxorgsoil4  -simes$minorgsoil4  )

meanorgsoil4  <-ifelse(simes$maxorgsoil4  -simes$minorgsoil4  >8,NA,ifelse(simes$maxorgsoil4  -simes$minorgsoil4  <0,NA,simes$meanorgsoil4  ))
summary(meanorgsoil4  )
maxorgsoil4  <-ifelse(simes$maxorgsoil4  -simes$minorgsoil4  >8,NA,ifelse(simes$maxorgsoil4  -simes$minorgsoil4  <0,NA,simes$maxorgsoil4  ))
minorgsoil4  <-ifelse(simes$maxorgsoil4  -simes$minorgsoil4  >8,NA,ifelse(simes$maxorgsoil4  -simes$minorgsoil4  <0,NA,simes$minorgsoil4  ))
 plot(meanorgsoil4    )
 plot(maxorgsoil4  -minorgsoil4  )

#organic soil 5
plot(simes$meanorgsoil5  )
plot(simes$maxorgsoil5  -simes$minorgsoil5  )
summary(simes$maxorgsoil5  -simes$minorgsoil5  )

meanorgsoil5  <-ifelse(simes$maxorgsoil5  -simes$minorgsoil5  >8,NA,ifelse(simes$maxorgsoil5  -simes$minorgsoil5  <0,NA,simes$meanorgsoil5  ))
summary(meanorgsoil5  )
maxorgsoil5  <-ifelse(simes$maxorgsoil5  -simes$minorgsoil5  >8,NA,ifelse(simes$maxorgsoil5  -simes$minorgsoil5  <0,NA,simes$maxorgsoil5  ))
minorgsoil5  <-ifelse(simes$maxorgsoil5  -simes$minorgsoil5  >8,NA,ifelse(simes$maxorgsoil5  -simes$minorgsoil5  <0,NA,simes$minorgsoil5  ))
 plot(meanorgsoil5    )
 plot(maxorgsoil5  -minorgsoil5  )

#organic soil 6
plot(simes$meanorgsoil6  )
plot(simes$maxorgsoil6  -simes$minorgsoil6  )
summary(simes$maxorgsoil6  -simes$minorgsoil6  )

meanorgsoil6  <-ifelse(simes$maxorgsoil6  -simes$minorgsoil6  >8,NA,ifelse(simes$maxorgsoil6  -simes$minorgsoil6  <0,NA,simes$meanorgsoil6  ))
summary(meanorgsoil6  )
maxorgsoil6  <-ifelse(simes$maxorgsoil6  -simes$minorgsoil6  >8,NA,ifelse(simes$maxorgsoil6  -simes$minorgsoil6  <0,NA,simes$maxorgsoil6  ))
minorgsoil6  <-ifelse(simes$maxorgsoil6  -simes$minorgsoil6  >8,NA,ifelse(simes$maxorgsoil6  -simes$minorgsoil6  <0,NA,simes$minorgsoil6  ))
 plot(meanorgsoil6    )
 plot(maxorgsoil6  -minorgsoil6  )

#organic soil 7
plot(simes$meanorgsoil7  )
plot(simes$maxorgsoil7  -simes$minorgsoil7  )
summary(simes$maxorgsoil7  -simes$minorgsoil7  )

meanorgsoil7  <-ifelse(simes$maxorgsoil7  -simes$minorgsoil7  >8,NA,ifelse(simes$maxorgsoil7  -simes$minorgsoil7  <0,NA,simes$meanorgsoil7  ))
summary(meanorgsoil7  )
maxorgsoil7  <-ifelse(simes$maxorgsoil7  -simes$minorgsoil7  >8,NA,ifelse(simes$maxorgsoil7  -simes$minorgsoil7  <0,NA,simes$maxorgsoil7  ))
minorgsoil7  <-ifelse(simes$maxorgsoil7  -simes$minorgsoil7  >8,NA,ifelse(simes$maxorgsoil7  -simes$minorgsoil7  <0,NA,simes$minorgsoil7  ))
 plot(meanorgsoil7    )
 plot(maxorgsoil7  -minorgsoil7  )

#organic soil 8
plot(simes$meanorgsoil8  )
plot(simes$maxorgsoil8  -simes$minorgsoil8  )
summary(simes$maxorgsoil8  -simes$minorgsoil8  )

meanorgsoil8  <-ifelse(simes$maxorgsoil8  -simes$minorgsoil8  >8,NA,ifelse(simes$maxorgsoil8  -simes$minorgsoil8  <0,NA,simes$meanorgsoil8  ))
summary(meanorgsoil8  )
maxorgsoil8  <-ifelse(simes$maxorgsoil8  -simes$minorgsoil8  >8,NA,ifelse(simes$maxorgsoil8  -simes$minorgsoil8  <0,NA,simes$maxorgsoil8  ))
minorgsoil8  <-ifelse(simes$maxorgsoil8  -simes$minorgsoil8  >8,NA,ifelse(simes$maxorgsoil8  -simes$minorgsoil8  <0,NA,simes$minorgsoil8  ))
 plot(meanorgsoil8    )
 plot(maxorgsoil8  -minorgsoil8  )

## Create datafile with the filtered data ##
#NB: no filter was set up for the ref measurements.

year<-simes$year
month<-simes$month
week<-simes$week
ym<-simes$ym
yw<- simes$yw
julianday<-simes$julianday
hour<-simes$hour
daytime<-simes$daytime 
meanref<-simes$meanref 
meanref2<-simes$meanref2
meanref7<-simes$meanref7
meanref8<-simes$meanref8 
minref<-simes$minref 
minref2<-simes$minref2
minref7<-simes$minref7
minref8<-simes$minref8 
maxref<-simes$maxref 
maxref2<-simes$maxref2
maxref7<-simes$maxref7
maxref8<-simes$maxref8 

simes.filt <- cbind(year ,  month ,week   ,ym   ,      
yw  , julianday,hour ,daytime  ,      
meanair1 , meanair2 ,meanair3, meanair4 ,     
meanair5    ,   meanair6 ,  meanair7  ,    meanair8  ,   
 meanmineralsoil1 ,meanmineralsoil2, meanmineralsoil3 ,meanmineralsoil4,
 meanmineralsoil5, meanmineralsoil6, meanmineralsoil7, meanmineralsoil8,
 meanorgsoil1  ,meanorgsoil2   ,  meanorgsoil3    , meanorgsoil4    ,
 meanorgsoil5   , meanorgsoil6 ,    meanorgsoil7,     meanorgsoil8  ,  
 meanref  , meanref2 , meanref7, meanref8  , 
 minair1         , minair2        ,  minair3 ,         minair4         ,
 minair5         , minair6         , minair7,          minair8         ,
 minmineralsoil1  ,minmineralsoil2  ,minmineralsoil3 , minmineralsoil4 ,
 minmineralsoil5 , minmineralsoil6,  minmineralsoil7,  minmineralsoil8 ,
 minorgsoil1 ,     minorgsoil2     , minorgsoil3  ,    minorgsoil4     ,
 minorgsoil5  ,    minorgsoil6,      minorgsoil7 ,     minorgsoil8     ,
minref        , minref2     , minref7   ,  minref8  ,
 maxair1        ,  maxair2      ,    maxair3   ,       maxair4         ,
 maxair5         , maxair6       ,   maxair7  ,        maxair8         ,
 maxmineralsoil1,  maxmineralsoil2 , maxmineralsoil3 , maxmineralsoil4 ,
 maxmineralsoil5 , maxmineralsoil6 , maxmineralsoil7,  maxmineralsoil8 ,
 maxorgsoil1    ,  maxorgsoil2  ,    maxorgsoil3 ,     maxorgsoil4     ,
 maxorgsoil5     , maxorgsoil6   ,   maxorgsoil7,      maxorgsoil8     ,
maxref          , maxref2        ,  maxref7,   maxref8)    


str(simes.filt)
head(simes.filt)
#write.table(simes.filt, "simes_microclim_filtered.csv", append=FALSE, col.names=TRUE, row.names=FALSE, sep=",")
 
 
#============================ PART 2: ADDITIONAL FILTERS FOR WINTER 2007 / 2008 =======================================================

# get data
#simes.filt<-read.csv("simes_microclim_filtered.csv")		#if you did not run part 1 of the script
simes.filt<-as.data.frame(simes.filt)                   #if you ran part 1 of the script

winter2008<-subset(simes.filt, ym > 200707 & yw < 200824) 
winter2008.unfiltered<-subset(simes, ym > 200707 & yw < 200824)  #for graphs comparing with the filtered data
summary(winter2008)

#===== MINERAL SOIL 4 ================
plot(winter2008$meanmineralsoil4)
plot(winter2008$maxmineralsoil4-winter2008$minmineralsoil4)

meanmineralsoil4.2008m <-ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 >.6,NA,ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 <0,NA,winter2008$meanmineralsoil4 ))
summary(meanmineralsoil4.2008m )
maxmineralsoil4.2008m <-ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 >.6,NA,ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 <0,NA,winter2008$maxmineralsoil4 ))
minmineralsoil4.2008m <-ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 >.6,NA,ifelse(winter2008$maxmineralsoil4 -winter2008$minmineralsoil4 <0,NA,winter2008$minmineralsoil4 ))
 plot(meanmineralsoil4.2008m   )
 plot(maxmineralsoil4.2008m - minmineralsoil4.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanmineralsoil4.2008m)
plot(meanmineralsoil4.2008m-winter2008$meanmineralsoil2)

##remove the values that are higher than in the valley
meanmineralsoil4.2008v1 <-ifelse(meanmineralsoil4.2008m - winter2008$meanmineralsoil2 >4,NA,meanmineralsoil4.2008m )
maxmineralsoil4.2008v1 <-ifelse(meanmineralsoil4.2008m -winter2008$meanmineralsoil2 >4,NA,maxmineralsoil4.2008m )
minmineralsoil4.2008v1 <-ifelse(meanmineralsoil4.2008m -winter2008$meanmineralsoil2 >4,NA,minmineralsoil4.2008m )
 par(mfrow=c(2,1))
 plot(meanmineralsoil4.2008v1, main="mean values")
 plot(meanmineralsoil4.2008v1 - winter2008$meanmineralsoil2, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanmineralsoil4.2008v2 <-ifelse(winter2008$meanmineralsoil2 - meanmineralsoil4.2008v1 >100,NA,meanmineralsoil4.2008v1 )
maxmineralsoil4.2008v2 <-ifelse(winter2008$meanmineralsoil2 - meanmineralsoil4.2008v1 >100,NA,maxmineralsoil4.2008v1 )
minmineralsoil4.2008v2 <-ifelse(winter2008$meanmineralsoil2 - meanmineralsoil4.2008v1 >100,NA,minmineralsoil4.2008v1 )
 plot(meanmineralsoil4.2008v2, main="mean values")
 plot(meanmineralsoil4.2008v2 - winter2008$meanmineralsoil2, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanmineralsoil4.2008v<-ifelse(is.na(winter2008$meanmineralsoil2), meanmineralsoil4.2008m,meanmineralsoil4.2008v2)
maxmineralsoil4.2008v<-ifelse(is.na(winter2008$meanmineralsoil2), maxmineralsoil4.2008m,maxmineralsoil4.2008v2)
minmineralsoil4.2008v<-ifelse(is.na(winter2008$meanmineralsoil2), minmineralsoil4.2008m,minmineralsoil4.2008v2)
 plot(meanmineralsoil4.2008m, main="means after max min correction")
 plot(meanmineralsoil4.2008v, main="means after valley correction")
plot(meanmineralsoil4.2008m-winter2008$meanmineralsoil2, main="difference with valley after only max-min")
plot(meanmineralsoil4.2008v - winter2008$meanmineralsoil2, main="new difference with valley")

#CORRECTION OF THE FLAT PART 
#Filter out the remaining outliers in the flat part

 #add the time to the valley correction, to be able to select the flat part
plot(meanmineralsoil4.2008v)
mineralsoil4.2008vt<-as.data.frame(cbind(meanmineralsoil4.2008v, maxmineralsoil4.2008v, minmineralsoil4.2008v, winter2008$yw))
names(mineralsoil4.2008vt)<-c("meanmineralsoil4", "maxmineralsoil4", "minmineralsoil4", "yw")
 
 #specify the period of the flat part
mineralsoil4.flat<-subset(mineralsoil4.2008vt, yw > 200748 & yw < 200813 )
 plot(mineralsoil4.flat$meanmineralsoil4)

 #filter the bad values from the flat part
meanmineralsoil4.2008ff<-ifelse(mineralsoil4.flat$meanmineralsoil4 > 3.3, NA, mineralsoil4.flat$meanmineralsoil4)
maxmineralsoil4.2008ff<-ifelse(mineralsoil4.flat$meanmineralsoil4 > 3.3, NA, mineralsoil4.flat$maxmineralsoil4)
minmineralsoil4.2008ff<-ifelse(mineralsoil4.flat$meanmineralsoil4 > 3.3, NA, mineralsoil4.flat$minmineralsoil4)
 plot(meanmineralsoil4.2008ff)

 #insert the corrected flat part into the valley correction
meanmineralsoil4.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, meanmineralsoil4.2008ff, meanmineralsoil4.2008v)
maxmineralsoil4.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, maxmineralsoil4.2008ff, maxmineralsoil4.2008v)
minmineralsoil4.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, minmineralsoil4.2008ff, minmineralsoil4.2008v)
 plot(meanmineralsoil4.2008v, main="after valley filter")
 plot(meanmineralsoil4.2008final, main="after flat part filter")
plot(winter2008$meanmineralsoil4, main="start")
plot(meanmineralsoil4.2008final, main="final")

#plots for document explaining the filtering
plot(winter2008.unfiltered$meanmineralsoil4, main="raw data", ylab="meanmineralsoil4")
plot(winter2008$meanmineralsoil4, main="after main filter", ylab="meanmineralsoil4")
plot(meanmineralsoil4.2008m, main="after stronger max - min filter", ylab="meanmineralsoil4"   )
 plot(meanmineralsoil4.2008v, main="after valley filter", ylab="meanmineralsoil4")
 plot(meanmineralsoil4.2008final, main="after flat part filter - FINAL ", ylab="meanmineralsoil4")


#========= MINERAL SOIL 5 ==========================
plot(winter2008$meanmineralsoil5)
plot(winter2008$maxmineralsoil5-winter2008$minmineralsoil5)

meanmineralsoil5.2008m <-ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 >.6,NA,ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 <0,NA,winter2008$meanmineralsoil5 ))
summary(meanmineralsoil5.2008m )
maxmineralsoil5.2008m <-ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 >.6,NA,ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 <0,NA,winter2008$maxmineralsoil5 ))
minmineralsoil5.2008m <-ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 >.6,NA,ifelse(winter2008$maxmineralsoil5 -winter2008$minmineralsoil5 <0,NA,winter2008$minmineralsoil5 ))
 plot(meanmineralsoil5.2008m   )
 plot(maxmineralsoil5.2008m - minmineralsoil5.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanmineralsoil5.2008m)
plot(meanmineralsoil5.2008m-winter2008$meanmineralsoil1)

##remove the values that are higher than in the valley
meanmineralsoil5.2008v1 <-ifelse(meanmineralsoil5.2008m - winter2008$meanmineralsoil1 >5,NA,meanmineralsoil5.2008m )
maxmineralsoil5.2008v1 <-ifelse(meanmineralsoil5.2008m -winter2008$meanmineralsoil1 >5,NA,maxmineralsoil5.2008m )
minmineralsoil5.2008v1 <-ifelse(meanmineralsoil5.2008m -winter2008$meanmineralsoil1 >5,NA,minmineralsoil5.2008m )
 par(mfrow=c(2,1))
 plot(meanmineralsoil5.2008v1, main="mean values")
 plot(meanmineralsoil5.2008v1 - winter2008$meanmineralsoil1, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanmineralsoil5.2008v2 <-ifelse(winter2008$meanmineralsoil1 - meanmineralsoil5.2008v1 >3.5,NA,meanmineralsoil5.2008v1 )
maxmineralsoil5.2008v2 <-ifelse(winter2008$meanmineralsoil1 - meanmineralsoil5.2008v1 >3.5,NA,maxmineralsoil5.2008v1 )
minmineralsoil5.2008v2 <-ifelse(winter2008$meanmineralsoil1 - meanmineralsoil5.2008v1 >3.5,NA,minmineralsoil5.2008v1 )
 plot(meanmineralsoil5.2008v2, main="mean values")
 plot(meanmineralsoil5.2008v2 - winter2008$meanmineralsoil1, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanmineralsoil5.2008v<-ifelse(is.na(winter2008$meanmineralsoil1), meanmineralsoil5.2008m,meanmineralsoil5.2008v2)
maxmineralsoil5.2008v<-ifelse(is.na(winter2008$meanmineralsoil1), maxmineralsoil5.2008m,maxmineralsoil5.2008v2)
minmineralsoil5.2008v<-ifelse(is.na(winter2008$meanmineralsoil1), minmineralsoil5.2008m,minmineralsoil5.2008v2)
 plot(meanmineralsoil5.2008m, main="means after max min correction")
 plot(meanmineralsoil5.2008v, main="means after valley correction")
plot(meanmineralsoil5.2008m-winter2008$meanmineralsoil1, main="difference with valley after only max-min")
plot(meanmineralsoil5.2008v - winter2008$meanmineralsoil1, main="new difference with valley")

#CORRECTION OF THE FLAT PART
#Filter out the remaining outliers in the flat part

 #add the time to the valley correction, to be able to select the flat part
plot(meanmineralsoil5.2008v)
mineralsoil5.2008vt<-as.data.frame(cbind(meanmineralsoil5.2008v, maxmineralsoil5.2008v, minmineralsoil5.2008v, winter2008$yw))
names(mineralsoil5.2008vt)<-c("meanmineralsoil5", "maxmineralsoil5", "minmineralsoil5", "yw")
 
 #specify the period of the flat part
mineralsoil5.flat<-subset(mineralsoil5.2008vt, yw > 200748 & yw < 200813 )
 plot(mineralsoil5.flat$meanmineralsoil5)

 #filter the bad values from the flat part
meanmineralsoil5.2008ff<-ifelse(mineralsoil5.flat$meanmineralsoil5 > 3.4, NA, mineralsoil5.flat$meanmineralsoil5)
maxmineralsoil5.2008ff<-ifelse(mineralsoil5.flat$meanmineralsoil5 > 3.4, NA, mineralsoil5.flat$maxmineralsoil5)
minmineralsoil5.2008ff<-ifelse(mineralsoil5.flat$meanmineralsoil5 > 3.4, NA, mineralsoil5.flat$minmineralsoil5)
 plot(meanmineralsoil5.2008ff)

  #insert the corrected flat part into the valley correction
meanmineralsoil5.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, meanmineralsoil5.2008ff, meanmineralsoil5.2008v)
maxmineralsoil5.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, maxmineralsoil5.2008ff, maxmineralsoil5.2008v)
minmineralsoil5.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, minmineralsoil5.2008ff, minmineralsoil5.2008v)
 plot(meanmineralsoil5.2008v, main="after valley filter")
 plot(meanmineralsoil5.2008final, main="after flat part filter")
plot(winter2008$meanmineralsoil5, main="start")
plot(meanmineralsoil5.2008final, main="final")


#======= MINERAL SOIL 6 ========================

plot(winter2008$meanmineralsoil6)
plot(winter2008$maxmineralsoil6-winter2008$minmineralsoil6)

meanmineralsoil6.2008m <-ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 >.6,NA,ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 <0,NA,winter2008$meanmineralsoil6 ))
summary(meanmineralsoil6.2008 )
maxmineralsoil6.2008m <-ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 >.6,NA,ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 <0,NA,winter2008$maxmineralsoil6 ))
minmineralsoil6.2008m <-ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 >.6,NA,ifelse(winter2008$maxmineralsoil6 -winter2008$minmineralsoil6 <0,NA,winter2008$minmineralsoil6 ))
 plot(meanmineralsoil6.2008m   )
 plot(maxmineralsoil6.2008m - minmineralsoil6.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanmineralsoil6.2008m)
plot(meanmineralsoil6.2008m-winter2008$meanmineralsoil3)

##remove the values that are higher than in the valley
meanmineralsoil6.2008v1 <-ifelse(meanmineralsoil6.2008m - winter2008$meanmineralsoil3 >100,NA,meanmineralsoil6.2008m )
maxmineralsoil6.2008v1 <-ifelse(meanmineralsoil6.2008m -winter2008$meanmineralsoil3 >100,NA,maxmineralsoil6.2008m )
minmineralsoil6.2008v1 <-ifelse(meanmineralsoil6.2008m -winter2008$meanmineralsoil3 >100,NA,minmineralsoil6.2008m )
 par(mfrow=c(2,1))
 plot(meanmineralsoil6.2008v1, main="mean values")
 plot(meanmineralsoil6.2008v1 - winter2008$meanmineralsoil3, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanmineralsoil6.2008v2 <-ifelse(winter2008$meanmineralsoil3 - meanmineralsoil6.2008v1 >6,NA,meanmineralsoil6.2008v1 )
maxmineralsoil6.2008v2 <-ifelse(winter2008$meanmineralsoil3 - meanmineralsoil6.2008v1 >6,NA,maxmineralsoil6.2008v1 )
minmineralsoil6.2008v2 <-ifelse(winter2008$meanmineralsoil3 - meanmineralsoil6.2008v1 >6,NA,minmineralsoil6.2008v1 )
 plot(meanmineralsoil6.2008v2, main="mean values")
 plot(meanmineralsoil6.2008v2 - winter2008$meanmineralsoil3, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanmineralsoil6.2008v<-ifelse(is.na(winter2008$meanmineralsoil3), meanmineralsoil6.2008m,meanmineralsoil6.2008v2)
maxmineralsoil6.2008v<-ifelse(is.na(winter2008$meanmineralsoil3), maxmineralsoil6.2008m,maxmineralsoil6.2008v2)
minmineralsoil6.2008v<-ifelse(is.na(winter2008$meanmineralsoil3), minmineralsoil6.2008m,minmineralsoil6.2008v2)
 plot(meanmineralsoil6.2008m, main="means after max min correction")
 plot(meanmineralsoil6.2008v, main="means after valley correction")
plot(meanmineralsoil6.2008m-winter2008$meanmineralsoil3, main="difference with valley after only max-min")
plot(meanmineralsoil6.2008v - winter2008$meanmineralsoil3, main="new difference with valley")

#CORRECTION OF THE FLAT PART
#Filter out the remaining outliers in the flat part

 #add the time to the valley correction, to be able to select the flat part
plot(meanmineralsoil6.2008v)
mineralsoil6.2008vt<-as.data.frame(cbind(meanmineralsoil6.2008v, maxmineralsoil6.2008v, minmineralsoil6.2008v, winter2008$yw))
names(mineralsoil6.2008vt)<-c("meanmineralsoil6", "maxmineralsoil6", "minmineralsoil6", "yw")
 
 #specify the period of the flat part
mineralsoil6.flat<-subset(mineralsoil6.2008vt, yw > 200748 & yw < 200813 )
 plot(mineralsoil6.flat$meanmineralsoil6)

 #filter the bad values from the flat part
meanmineralsoil6.2008ff<-ifelse(mineralsoil6.flat$meanmineralsoil6 > 3.5, NA, mineralsoil6.flat$meanmineralsoil6)
maxmineralsoil6.2008ff<-ifelse(mineralsoil6.flat$meanmineralsoil6 > 3.5, NA, mineralsoil6.flat$maxmineralsoil6)
minmineralsoil6.2008ff<-ifelse(mineralsoil6.flat$meanmineralsoil6 > 3.5, NA, mineralsoil6.flat$minmineralsoil6)
 plot(meanmineralsoil6.2008ff)
 
  #insert the corrected flat part into the valley correction
meanmineralsoil6.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, meanmineralsoil6.2008ff, meanmineralsoil6.2008v)
maxmineralsoil6.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, maxmineralsoil6.2008ff, maxmineralsoil6.2008v)
minmineralsoil6.2008final<-ifelse(winter2008$yw > 200748 & winter2008$yw < 200813, minmineralsoil6.2008ff, minmineralsoil6.2008v)
 plot(meanmineralsoil6.2008v, main="after valley filter")
 plot(meanmineralsoil6.2008final, main="after flat part filter") #WHY are there suddenly more datapoints?!?! Or is this just the weird spot?!
plot(winter2008$meanmineralsoil6, main="start")
plot(meanmineralsoil6.2008final, main="final")


#====== MINERAL SOIL 1 ===========================
plot(winter2008$meanmineralsoil1)
plot(winter2008$maxmineralsoil1-winter2008$minmineralsoil1)

#====== MINERAL SOIL 2 ===================================
plot(winter2008$meanmineralsoil2)
plot(winter2008$maxmineralsoil2-winter2008$minmineralsoil2)

#====== MINERAL SOIL 3 ===========================
plot(winter2008$meanmineralsoil3)
plot(winter2008$maxmineralsoil3-winter2008$minmineralsoil3)

# Mineral soil 1, 2 and 3 don't need extra filtering, judging from the graphs.


#====== AIR 4 ==================================

plot(winter2008$meanair4)
plot(winter2008$maxair4-winter2008$minair4)

meanair4.2008m <-ifelse(winter2008$maxair4 -winter2008$minair4 >5,NA,ifelse(winter2008$maxair4 -winter2008$minair4 <0,NA,winter2008$meanair4 ))
summary(meanair4.2008m )
maxair4.2008m <-ifelse(winter2008$maxair4 -winter2008$minair4 >5,NA,ifelse(winter2008$maxair4 -winter2008$minair4 <0,NA,winter2008$maxair4 ))
minair4.2008m <-ifelse(winter2008$maxair4 -winter2008$minair4 >5,NA,ifelse(winter2008$maxair4 -winter2008$minair4 <0,NA,winter2008$minair4 ))
 plot(meanair4.2008m   )
 plot(maxair4.2008m - minair4.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanair4.2008m)
plot(meanair4.2008m-winter2008$meanair2)

##remove the values that are higher than in the valley
meanair4.2008v1 <-ifelse(meanair4.2008m - winter2008$meanair2 >5,NA,meanair4.2008m )
maxair4.2008v1 <-ifelse(meanair4.2008m -winter2008$meanair2 >5,NA,maxair4.2008m )
minair4.2008v1 <-ifelse(meanair4.2008m -winter2008$meanair2 >5,NA,minair4.2008m )
 par(mfrow=c(2,1))
 plot(meanair4.2008v1, main="mean values")
 plot(meanair4.2008v1 - winter2008$meanair2, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanair4.2008v2 <-ifelse(winter2008$meanair2 - meanair4.2008v1 >3,NA,meanair4.2008v1 )
maxair4.2008v2 <-ifelse(winter2008$meanair2 - meanair4.2008v1 >3,NA,maxair4.2008v1 )
minair4.2008v2 <-ifelse(winter2008$meanair2 - meanair4.2008v1 >3,NA,minair4.2008v1 )
 plot(meanair4.2008v2, main="mean values")
 plot(meanair4.2008v2 - winter2008$meanair2, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanair4.2008v<-ifelse(is.na(winter2008$meanair2), meanair4.2008m,meanair4.2008v2)
maxair4.2008v<-ifelse(is.na(winter2008$meanair2), maxair4.2008m,maxair4.2008v2)
minair4.2008v<-ifelse(is.na(winter2008$meanair2), minair4.2008m,minair4.2008v2)
 plot(meanair4.2008m, main="means after max min correction")
 plot(meanair4.2008v, main="means after valley correction")
plot(meanair4.2008m-winter2008$meanair2, main="difference with valley after only max-min")
plot(meanair4.2008v - winter2008$meanair2, main="new difference with valley")
 plot(winter2008$meanair4, main="start")
 plot(meanair4.2008v, main="final")


#====== AIR 5 ==================================
plot(winter2008$meanair5)
plot(winter2008$maxair5-winter2008$minair5)

meanair5.2008m <-ifelse(winter2008$maxair5 -winter2008$minair5 >4,NA,ifelse(winter2008$maxair5 -winter2008$minair5 <0,NA,winter2008$meanair5 ))
summary(meanair5.2008m )
maxair5.2008m <-ifelse(winter2008$maxair5 -winter2008$minair5 >4,NA,ifelse(winter2008$maxair5 -winter2008$minair5 <0,NA,winter2008$maxair5 ))
minair5.2008m <-ifelse(winter2008$maxair5 -winter2008$minair5 >4,NA,ifelse(winter2008$maxair5 -winter2008$minair5 <0,NA,winter2008$minair5 ))
 plot(meanair5.2008m   )
 plot(maxair5.2008m - minair5.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanair5.2008m)
plot(meanair5.2008m-winter2008$meanair1)

##remove the values that are higher than in the valley
meanair5.2008v1 <-ifelse(meanair5.2008m - winter2008$meanair1 >4,NA,meanair5.2008m )
maxair5.2008v1 <-ifelse(meanair5.2008m -winter2008$meanair1 >4,NA,maxair5.2008m )
minair5.2008v1 <-ifelse(meanair5.2008m -winter2008$meanair1 >4,NA,minair5.2008m )
 par(mfrow=c(2,1))
 plot(meanair5.2008v1, main="mean values")
 plot(meanair5.2008v1 - winter2008$meanair1, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanair5.2008v2 <-ifelse(winter2008$meanair1 - meanair5.2008v1 >100,NA,meanair5.2008v1 )
maxair5.2008v2 <-ifelse(winter2008$meanair1 - meanair5.2008v1 >100,NA,maxair5.2008v1 )
minair5.2008v2 <-ifelse(winter2008$meanair1 - meanair5.2008v1 >100,NA,minair5.2008v1 )
 plot(meanair5.2008v2, main="mean values")
 plot(meanair5.2008v2 - winter2008$meanair1, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanair5.2008v<-ifelse(is.na(winter2008$meanair1), meanair5.2008m,meanair5.2008v2)
maxair5.2008v<-ifelse(is.na(winter2008$meanair1), maxair5.2008m,maxair5.2008v2)
minair5.2008v<-ifelse(is.na(winter2008$meanair1), minair5.2008m,minair5.2008v2)
 plot(meanair5.2008m, main="means after max min correction")
 plot(meanair5.2008v, main="means after valley correction")
plot(meanair5.2008m-winter2008$meanair1, main="difference with valley after only max-min")
plot(meanair5.2008v - winter2008$meanair1, main="new difference with valley")
 plot(winter2008$meanair5, main="start")
 plot(meanair5.2008v, main="final")


#====== AIR 6 ==================================

plot(winter2008$meanair6)
plot(winter2008$maxair6-winter2008$minair6)

meanair6.2008m <-ifelse(winter2008$maxair6 -winter2008$minair6 >2,NA,ifelse(winter2008$maxair6 -winter2008$minair6 <0,NA,winter2008$meanair6 ))
summary(meanair6.2008m )
maxair6.2008m <-ifelse(winter2008$maxair6 -winter2008$minair6 >2,NA,ifelse(winter2008$maxair6 -winter2008$minair6 <0,NA,winter2008$maxair6 ))
minair6.2008m <-ifelse(winter2008$maxair6 -winter2008$minair6 >2,NA,ifelse(winter2008$maxair6 -winter2008$minair6 <0,NA,winter2008$minair6 ))
 plot(meanair6.2008m   )
 plot(maxair6.2008m - minair6.2008m )

#VALLEY CORRECTIONS
#Compare to the valley plot
plot(meanair6.2008m)
plot(meanair6.2008m-winter2008$meanair3)

##remove the values that are higher than in the valley
meanair6.2008v1 <-ifelse(meanair6.2008m - winter2008$meanair3 >4,NA,meanair6.2008m )
maxair6.2008v1 <-ifelse(meanair6.2008m -winter2008$meanair3 >4,NA,maxair6.2008m )
minair6.2008v1 <-ifelse(meanair6.2008m -winter2008$meanair3 >4,NA,minair6.2008m )
 par(mfrow=c(2,1))
 plot(meanair6.2008v1, main="mean values")
 plot(meanair6.2008v1 - winter2008$meanair3, main="new difference with valley")

##remove the values that are lower than in the valley (if necessary - if you don't want to remove anything, set this very high)
meanair6.2008v2 <-ifelse(winter2008$meanair3 - meanair6.2008v1 >10,NA,meanair6.2008v1 )
maxair6.2008v2 <-ifelse(winter2008$meanair3 - meanair6.2008v1 >10,NA,maxair6.2008v1 )
minair6.2008v2 <-ifelse(winter2008$meanair3 - meanair6.2008v1 >10,NA,minair6.2008v1 )
 plot(meanair6.2008v2, main="mean values")
 plot(meanair6.2008v2 - winter2008$meanair3, main="new difference with valley")

##if the valley plot has NA, take the ridge plot after max-min filter; otherwise take the valley correction:
meanair6.2008v<-ifelse(is.na(winter2008$meanair3), meanair6.2008m,meanair6.2008v2)
maxair6.2008v<-ifelse(is.na(winter2008$meanair3), maxair6.2008m,maxair6.2008v2)
minair6.2008v<-ifelse(is.na(winter2008$meanair3), minair6.2008m,minair6.2008v2)
 plot(meanair6.2008m, main="means after max min correction")
 plot(meanair6.2008v, main="means after valley correction")
plot(meanair6.2008m-winter2008$meanair3, main="difference with valley after only max-min")
plot(meanair6.2008v - winter2008$meanair3, main="new difference with valley")
 plot(winter2008$meanair6, main="start")
 plot(meanair6.2008v, main="final")

#======= AIR 1 ============================
plot(winter2008$meanair1)
plot(winter2008$maxair1-winter2008$minair1)

#======= AIR 2 ============================
plot(winter2008$meanair3)
plot(winter2008$maxair3-winter2008$minair3)

#======= AIR 3 ============================
plot(winter2008$meanair3)
plot(winter2008$maxair3-winter2008$minair3)

#The air data for plots 1, 2 and 3 don't need additional filtering, judging from the graphs. It seems to be mainly a datalogger problem in plots 4, 5 and 6.


#======== WRITE A NEW TABLE FOR WINTER 2007 / 2008 ===========================

year<-winter2008$year
month<-winter2008$month
week<-winter2008$week
ym<-winter2008$ym
yw<- winter2008$yw
julianday<-winter2008$julianday
hour<-winter2008$hour
daytime<-winter2008$daytime 
meanref<-winter2008$meanref 
meanref2<-winter2008$meanref2
meanref7<-winter2008$meanref7
meanref8<-winter2008$meanref8 
minref<-winter2008$minref 
minref2<-winter2008$minref2
minref7<-winter2008$minref7
minref8<-winter2008$minref8 
maxref<-winter2008$maxref 
maxref2<-winter2008$maxref2
maxref7<-winter2008$maxref7
maxref8<-winter2008$maxref8 


meanair1<-winter2008$meanair1
meanair2<-winter2008$meanair2
meanair3<-winter2008$meanair3
meanair7<-winter2008$meanair7
meanair8<-winter2008$meanair8
meanmineralsoil1<-winter2008$meanmineralsoil1
meanmineralsoil2<-winter2008$meanmineralsoil2
meanmineralsoil3<-winter2008$meanmineralsoil3
meanmineralsoil7<-winter2008$meanmineralsoil7
meanmineralsoil8<-winter2008$meanmineralsoil8
meanorgsoil1<-winter2008$meanorgsoil1  
meanorgsoil2<-winter2008$meanorgsoil2   
meanorgsoil3<-winter2008$meanorgsoil3    
meanorgsoil4<-winter2008$meanorgsoil4
meanorgsoil5<-winter2008$meanorgsoil5   
meanorgsoil6<-winter2008$meanorgsoil6 
meanorgsoil7<-winter2008$meanorgsoil7
meanorgsoil8<-winter2008$meanorgsoil8

maxair1<-winter2008$maxair1
maxair2<-winter2008$maxair2
maxair3<-winter2008$maxair3
maxair7<-winter2008$maxair7
maxair8<-winter2008$maxair8
maxmineralsoil1<-winter2008$maxmineralsoil1
maxmineralsoil2<-winter2008$maxmineralsoil2
maxmineralsoil3<-winter2008$maxmineralsoil3
maxmineralsoil7<-winter2008$maxmineralsoil7
maxmineralsoil8<-winter2008$maxmineralsoil8
maxorgsoil1<-winter2008$maxorgsoil1  
maxorgsoil2<-winter2008$maxorgsoil2   
maxorgsoil3<-winter2008$maxorgsoil3    
maxorgsoil4<-winter2008$maxorgsoil4
maxorgsoil5<-winter2008$maxorgsoil5   
maxorgsoil6<-winter2008$maxorgsoil6 
maxorgsoil7<-winter2008$maxorgsoil7
maxorgsoil8<-winter2008$maxorgsoil8

minair1<-winter2008$minair1
minair2<-winter2008$minair2
minair3<-winter2008$minair3
minair7<-winter2008$minair7
minair8<-winter2008$minair8
minmineralsoil1<-winter2008$minmineralsoil1
minmineralsoil2<-winter2008$minmineralsoil2
minmineralsoil3<-winter2008$minmineralsoil3
minmineralsoil7<-winter2008$minmineralsoil7
minmineralsoil8<-winter2008$minmineralsoil8
minorgsoil1<-winter2008$minorgsoil1  
minorgsoil2<-winter2008$minorgsoil2   
minorgsoil3<-winter2008$minorgsoil3    
minorgsoil4<-winter2008$minorgsoil4
minorgsoil5<-winter2008$minorgsoil5   
minorgsoil6<-winter2008$minorgsoil6 
minorgsoil7<-winter2008$minorgsoil7
minorgsoil8<-winter2008$minorgsoil8

meanmineralsoil4<-meanmineralsoil4.2008final
meanmineralsoil5<-meanmineralsoil5.2008final
meanmineralsoil6<-meanmineralsoil6.2008final
maxmineralsoil4<-maxmineralsoil4.2008final
maxmineralsoil5<-maxmineralsoil5.2008final
maxmineralsoil6<-maxmineralsoil6.2008final
minmineralsoil4<-minmineralsoil4.2008final
minmineralsoil5<-minmineralsoil5.2008final
minmineralsoil6<-minmineralsoil6.2008final

meanair4<-meanair4.2008v 
meanair5<-meanair5.2008v
meanair6<-meanair6.2008v
maxair4<-maxair4.2008v
maxair5<-maxair5.2008v
maxair6<-maxair6.2008v
minair4<-minair4.2008v 
minair5<-minair5.2008v
minair6<-minair6.2008v


temp.table <- cbind(year ,  month ,week   ,ym   ,      
yw  , julianday,hour ,daytime  ,      
meanair1 , meanair2 ,meanair3, meanair4 ,     
meanair5    ,   meanair6 ,  meanair7  ,    meanair8  ,   
 meanmineralsoil1 ,meanmineralsoil2, meanmineralsoil3 ,meanmineralsoil4,
 meanmineralsoil5, meanmineralsoil6, meanmineralsoil7, meanmineralsoil8,
 meanorgsoil1  ,meanorgsoil2   ,  meanorgsoil3    , meanorgsoil4    ,
 meanorgsoil5   , meanorgsoil6 ,    meanorgsoil7,     meanorgsoil8  ,  
 meanref  , meanref2 , meanref7, meanref8  , 
 minair1         , minair2        ,  minair3 ,         minair4         ,
 minair5         , minair6         , minair7,          minair8         ,
 minmineralsoil1  ,minmineralsoil2  ,minmineralsoil3 , minmineralsoil4 ,
 minmineralsoil5 , minmineralsoil6,  minmineralsoil7,  minmineralsoil8 ,
 minorgsoil1 ,     minorgsoil2     , minorgsoil3  ,    minorgsoil4     ,
 minorgsoil5  ,    minorgsoil6,      minorgsoil7 ,     minorgsoil8     ,
minref        , minref2     , minref7   ,  minref8  ,
 maxair1        ,  maxair2      ,    maxair3   ,       maxair4         ,
 maxair5         , maxair6       ,   maxair7  ,        maxair8         ,
 maxmineralsoil1,  maxmineralsoil2 , maxmineralsoil3 , maxmineralsoil4 ,
 maxmineralsoil5 , maxmineralsoil6 , maxmineralsoil7,  maxmineralsoil8 ,
 maxorgsoil1    ,  maxorgsoil2  ,    maxorgsoil3 ,     maxorgsoil4     ,
 maxorgsoil5     , maxorgsoil6   ,   maxorgsoil7,      maxorgsoil8     ,
maxref          , maxref2        ,  maxref7,   maxref8)    


str(temp.table)

head(temp.table)
summary(temp.table)

write.table(temp.table, "simes_microclim_filtered2008take2.csv", append=FALSE, col.names=TRUE, row.names=FALSE, sep=",")

## FINAL STEP ##
#This data was manually inserted into simes_microclim_filtered.csv, and saved as hf108-04-air-soil-temp-filt.csv