big rewrite to work on tar.gz files instead of unzipped data, this removes any potential to mix up files (and might be faster)
big rewrite to work on tar.gz files instead of unzipped data, this removes any potential to mix up files (and might be faster)

khufkens/phenor
#' Preprocessing of PEP725 data, merges separate files into tidy
#' data, with each observation a line, each column a different
#' parameter value.
#'
#' @param path a path to the PEP725 data (as tar.gz files, not unzipped)
#' @return concatted data of all data in the path as a tidy data frame
#' including all normal parameters, and the species name and country code
#' as derived from the file name
#' @keywords phenology, model, preprocessing
#' @export
#' @examples
#'
#' \dontrun{
#' tidy_pep_data = merge_pep725()
#'}

merge_pep725 = function(path = "~"){

  # create tmp directory
  tmpdir = tempdir()

  # list all files
  archive_files = list.files(path, "*.tar.gz", full.names = TRUE)

  # return data from a do call, binding the different
  # data sets by row
  do.call("rbind",lapply(archive_files, function(file){

    # check the contents of the tar.gz file
    # and only select true data files (discard README and descriptor)
    pep_files = untar(file, list=TRUE)
    pep_files = pep_files[!grepl("^.*PEP725_BBCH.csv$|PEP725_README.txt", pep_files)]

    # extract only the true data files and station info files
    # drop the BBCH and README data (but don't delete it - delist)
    data_file = pep_files[!grepl("stations",pep_files)]
    station_file = pep_files[grepl("stations",pep_files)]

    # unzip only the selected files into the output path
    # use path.expand to deal with the fact that untar
    # does not work with relative paths
    untar(file,
          files = pep_files,
          exdir = path.expand(tmpdir))

    # read in observation data from a particular gziped file
    observation_data = utils::read.csv2(sprintf("%s/%s",tmpdir,data_file),
                                        sep = ";",
                                        stringsAsFactors = FALSE)
    observation_data$country = substr(data_file,8,9)
    observation_data$species = sub("_"," ",substr(data_file,11,nchar(data_file)-4))

    station_locations = utils::read.csv2(sprintf("%s/%s",tmpdir,station_file),
                                         sep = ";",
                                         stringsAsFactors = FALSE)
    station_locations$LON = as.numeric(station_locations$LON)
    station_locations$LAT = as.numeric(station_locations$LAT)

    # discard any columns > 6 (errors in NAME field)
    if(ncol(station_locations) > 6){
      station_locations = station_locations[,-c(7:ncol(station_locations))]
    }

    # do a left merge to combine the observational data and the
    # station location meta-data returning basically the original
    # database structure (as a tidy file)
    pep_data = merge(observation_data, station_locations, by = "PEP_ID")
    names(pep_data) = tolower(names(pep_data))

    # cleanup extracted data for good measure
    # and return the combined data frame
    file.remove(paste(tmpdir,pep_files,sep = "/"))
    return(pep_data)
  }))
}

#' Preprocessing of PEP725 data, merges separate files into tidy
#' data, with each observation a line, each column a different
#' parameter value.
#'
#' Mind that the PEP725 database is badly managed. For example the Irish
#' station file includes site description with semi-colons in the description
#' of a semi-colon delimited file format. This will crash the routine. I will
#' not provide a bug fix for these known issues with the underlying
#' PEP725 database due to mismanagement on their part.
#'
#' @param path a path to the PEP725 data (species files only)
#' @return concatted data of all data in the path as a tidy data frame
#' including all normal parameters, and the species name and country code
#' as derived from the file name
#' @keywords phenology, model, preprocessing
#' @export
#' @examples
#'
#' \dontrun{
#' tidy_pep_data = merge_pep725()
#'}

merge_pep725 = function(path = "~"){

  # list all files
  pep_files = list.files(path, "*.csv", full.names = TRUE)

  # list file type locations
  data_file_locations = !grepl("^.*PEP725_BBCH.csv$",pep_files)
  station_file_locations = grepl("^.*stations\\.csv$",pep_files)

  # extract only the true data files and station info files
  # drop the BBCH and README data (but don't delete it - delist)
  data_files = pep_files[data_file_locations & !station_file_locations]
  station_files = pep_files[station_file_locations]

  print(station_files)

  # merge all the csv files for both the true observations and the
  # station meta-data (name, lat, lon etc.)
  observation_data = do.call(rbind,
                 lapply(data_files, function(file){
                   tmp = utils::read.csv2(file, sep = ";",
                                          stringsAsFactors = FALSE)
                   filename = basename(file)
                   tmp$country = substr(filename,8,9)
                   tmp$species = sub("_"," ",substr(filename,11,nchar(filename)-4))
                   return(tmp)
                   })
                 )

  station_locations = do.call(rbind,
                     lapply(station_files, function(file){
                       print(file)
                      tmp = utils::read.csv2(file,
                                        sep = ";",
                                        stringsAsFactors = FALSE)
                      tmp$LON = as.numeric(tmp$LON)
                      tmp$LAT = as.numeric(tmp$LAT)
                      return(tmp)
                     })
                )

  # do a left merge to combine the observational data and the
  # station location meta-data returning basically the original
  # database structure (as a tidy file)
  pep_data = merge(observation_data, station_locations, by = "PEP_ID")
  names(pep_data) = tolower(names(pep_data))

  # return this dataframe
  return(pep_data)
}
