/**
    * Copyright 2010, CSIRO Australia.
    *
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    *
    *         http://www.apache.org/licenses/LICENSE-2.0
    *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package au.csiro.netcdf.wron;
  
  import java.io.BufferedReader;
  import java.io.ByteArrayInputStream;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileReader;
  import java.io.IOException;
  import java.io.PrintWriter;
  import java.io.RandomAccessFile;
  import java.io.StringWriter;
  import java.io.UnsupportedEncodingException;
  import java.text.DateFormat;
  import java.text.NumberFormat;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Calendar;
  import java.util.Collections;
  import java.util.Comparator;
  import java.util.Date;
  import java.util.HashMap;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import java.util.Scanner;
  import java.util.Set;
  import java.util.TreeSet;
  import java.util.regex.Pattern;
  
  import org.apache.commons.cli.BasicParser;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.MissingOptionException;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.OptionBuilder;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.log4j.Logger;
  
  import ucar.ma2.DataType;
  import ucar.nc2.Attribute;
  import ucar.nc2.units.DateFormatter;
  import au.csiro.netcdf.NcDefineAttributes;
  import au.csiro.netcdf.NcDefineDimension;
  import au.csiro.netcdf.NcDefineVariable;
  import au.csiro.netcdf.NcWriteVariable;
  import au.csiro.netcdf.cli.CommandLineOptionsComparator;
  import au.csiro.netcdf.util.CSVTokenizer;
  import au.csiro.netcdf.util.NetCDFUtils;
  
  /**
   * The class is a converter control class to convert Murray Darling Basin 
   * Sustainable Yields CSV files into netCDF files. This class will process 
   * the Scenario A historical dataset files.  
   * 
   * Copyright 2010, CSIRO Australia 
   * All rights reserved.
   * 
   * @author James Dempsey on 17/04/2010
   * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $
   */
  public class MdbsyScenarioAConverter
  {
      // netCDF file naming components
      private static String COLLECTION = "Collection-MDBSY_Climate";
      private static String SCENARIO = "";
      private static String MODEL = "";
      private static String CASE = "";
  
      /**
       * The file containing the location lookup values.
       */
      private static String LOCATION_LOOKUP_FILE = "";
  
      /**
       * The file containing the metadata attributes (if any).
       */
      private static String METADATA_FILE = "";
  
      /**
       * The directory containing the Scenario C csv data files.
      */
     private static String INPUT_CSV_DIRECTORY = "";
 
     /**
      * The netCDF file to be created.
      */
     private static String OUTPUT_NETCDF_DIRECTORY = "";
 
     /**
      * NetCDF file extension.
      */
     private static final String NETCDF_FILE_EXTENSION = ".nc";
 
     /**
      * Encoding used to fill variables.
      */
     private static final String ENCODING = "UTF-8";
 
     // Variables
     private static final String[] variableNames = new String[] { "SILO_rainfall", "SILO_minTemp", "SILO_maxTemp",
             "SILO_incomingSolarRadiation", "SILO_actualVapourPressure", "APET", "PPET", "relativeHumidity", "Evappt" };
     private static final String[] variableUnits = new String[] { "mm", "degrees_celcius", "degrees_celcius", "MJ",
             "hpa", "mm", "mm", "TBA", "mm" };
     private static final String[] variableLongNames = new String[] { "SILO_rainfall", "SILO_minTemp", "SILO_maxTemp",
             "SILO_incomingSolarRadiation", "SILO_actualVapourPressure", "APET", "PPET", "relativeHumidity", "P-TET" };
     private static final String[] variableFillValues = new String[] { "-9999.0f", "-9999.0f", "-9999.0f", "-9999.0f",
             "-9999.0f", "-9999.0f", "-9999.0f", "-9999.0f", "-9999.0f" };
     private static final String[] variableMissingValues = variableFillValues;
     private static final String[] variableMinValues = new String[] { "0.0f", "-20.0f", "-20.0f", "0.0f",
         "0.0f", "0.0f", "0.0f", "0.0f", "0.0f" };
     private static final String[] variableMaxValues = new String[] { "500.0f", "100.0f", "100.0f", "100.0f",
         "100.0f", "100.0f", "100.0f", "100.0f", "500.0f" };
     private static final DataType[] variableDataTypes = new DataType[] { DataType.FLOAT, DataType.FLOAT,
             DataType.FLOAT, DataType.FLOAT, DataType.FLOAT, DataType.FLOAT, DataType.FLOAT, DataType.FLOAT,
             DataType.FLOAT };
     static final int numVariables = variableNames.length;
     private static final String GRID_MAPPING = "crs";
 
     // time constants
     private static final String TIME = "time";
     private static final String TIME_RANGE = "0-43097";
     private static final int TIME_SIZE = 43098; // 365
     private static final String TIME_LONG_NAME = "reference time";
     private static final String TIME_STANDARD_NAME = "time";
     private static final String TIME_UNITS = "days since 1889-01-01 0:0:0";
     private static final String TIME_CALENDAR = "Gregorian";
     private static final String TIME_AXIS = "T";
     private static final String TIME_BOUNDS = "time_bnds";
 
     // longitude constants
     private static final String LONG = "longitude";
     private static final int LONG_COLUMN_INDEX = 1;
     private static final String LONG_RANGE = "0-296";
     private static final int LONG_SIZE = 297;
     private static final String LONG_UNITS = "degrees_east";
     private static final String LONG_STANDARD_NAME = "longitude";
     private static final String LONG_AXIS = "X";
     private static final String LONG_LONG_NAME = "longitude";
 
     // latitude constants
     private static final String LAT = "latitude";
     private static final int LAT_COLUMN_INDEX = 2;
     private static final String LAT_RANGE = "0-278";
     private static final int LAT_SIZE = 279;
     private static final String LAT_UNITS = "degrees_north";
     private static final String LAT_STANDARD_NAME = "latitude";
     private static final String LAT_AXIS = "Y";
     private static final String LAT_LONG_NAME = "latitude";
 
     // num values constants
     private static final String NV = "nv";
     private static final String NV_RANGE = "0-1";
     private static final int NV_SIZE = 2;
 
     // elevation constants
     private static String ELEVATION = "elev";
     private static int ELEVATION_COLUMN_INDEX = 3;
     private static final String ELEVATION_UNITS = "m";
     private static final String ELEVATION_STANDARD_NAME = "altitude";
     private static final String ELEVATION_LONG_NAME = "Elevation above sea level";
     private static final float ELEVATION_MISSING_VALUE = -999.99f;
 
     // catchment id constants
     private static String CATCHMENT_ID = "catchmentId";
     private static int CATCHMENT_ID_COLUMN_INDEX = 4;
     private static final String CATCHMENT_ID_LONG_NAME = "MDB Catchment Id";
     private static final float CATCHMENT_ID_MISSING_VALUE = 0f;
 
     // reporting region id constants
     private static String REPORTING_REGION = "repRegionId";
     private static int REPORTING_REGION_ID_COLUMN_INDEX = 5;
     private static String REPORTING_REGION_LONG_NAME = "MDB Reporting Region Id";
     private static final float REPORTING_REGION_MISSING_VALUE = 0f;
 
     // crs constants
     private static String CRS_GRID_MAPPING_NAME = "latitude_longitude";  
     private static float CRS_LONGITUDE_OF_PRIME_MERIDIAN = 0.0f;
     private static float CRS_SEMI_MAJOR_AXIS = 6378137.0f;
     private static float CRS_INVERSE_FLATTENING = 298.257222101f;
     
     // cell id
     private static final int CELL_ID_COLUMN_INDEX = 0;
 
     // Other cell data
 
     // date conversion constants
     private static final String EPOC_DATE_STRING = "1889-01-01";
     private static final DateFormatter isoDateTimeFormat = new DateFormatter();
     private static final int DATE_COLUMN_INDEX = 0;
 
     // number of time values to read from all the csv files and load into memory.
     private static int ROW_CHUNK_SIZE = 1000;
 
     /** The number of columns expected in the CSV data files */
     private static final int NUM_CSV_COLUMNS = 1 + numVariables;
 
     /**
      * Constant that defines the logger to be used.
      */
     private static final Logger LOG = Logger.getLogger(MdbsyScenarioAConverter.class.getName());
 
     /** Conversion utilities  */
     private ConversionUtils convUtils = new ConversionUtils();
 
     /**
      * @param args
      */
     @SuppressWarnings("static-access")
     public static void main(String[] args) throws Exception
     {
         Options options = new Options();
         try
         {
             Option inputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(
                     " 1: the directory containing the csv files for the scenario.").isRequired(true).withLongOpt(
                     "inputDirectory").create("i");
 
             Option lookupFile = OptionBuilder.withArgName("file").hasArg()
                     .withDescription(" 2: the cellId lookup file.").isRequired(true).withLongOpt("lookupFile").create(
                             "f");
 
             Option outputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(
                     " 3: the output directory.").isRequired(true).withLongOpt("outputDirectory").create("o");
 
             Option scenario = OptionBuilder.withArgName("text").hasArg().withDescription(
                     " 4: the modelling scenario used to name the netCDF file.").isRequired(true).withLongOpt("scenario")
                     .create("s");
 
             Option metadataFile = OptionBuilder.withArgName("file").hasArg().withDescription(
                     " 5: a file of global metadata attributes to be added to the netCDF file.").isRequired(true)
                     .withLongOpt("metadataFile").create("m");
 
             Option numRows = OptionBuilder.withArgName("int").hasArg().withDescription(
                     " 6: the number of rows to read at a time.").isRequired(true).withLongOpt("rows").create("r");
 
             Option byDecade = OptionBuilder.withDescription(
                     " 7: OPTIONAL, set if we want to convert into netCDF files by decade.").isRequired(false)
                     .withLongOpt("decade").create("d");
 
             Option byLatitude = OptionBuilder.withDescription(
                     " 8: OPTIONAL, set if we want to convert into netCDF files by degrees of latitude.").isRequired(
                     false).withLongOpt("latitude").create("l");
 
             Option startLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(
                     " 9: OPTIONAL, The latitude at which to start processing. For byDecade, this option will "
                             + "assume already existing netCDF files.").isRequired(false).create("startLat");
 
             Option endLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(
                     "10: OPTIONAL, The latitude at which to end processing (inclusive).").isRequired(false).create(
                     "endLat");
 
             options.addOption(inputDirectory);
             options.addOption(lookupFile);
             options.addOption(outputDirectory);
             options.addOption(scenario);
             options.addOption(metadataFile);
             options.addOption(numRows);
             options.addOption(byDecade);
             options.addOption(byLatitude);
             options.addOption(startLat);
             options.addOption(endLat);
 
             // parse the command line arguments
             CommandLine parsedCommandLine = new BasicParser().parse(options, args);
 
             INPUT_CSV_DIRECTORY = (parsedCommandLine.hasOption("inputDirectory")) ? parsedCommandLine
                     .getOptionValue("inputDirectory") : "";
             LOCATION_LOOKUP_FILE = (parsedCommandLine.hasOption("lookupFile")) ? parsedCommandLine
                     .getOptionValue("lookupFile") : "";
             OUTPUT_NETCDF_DIRECTORY = (parsedCommandLine.hasOption("outputDirectory")) ? parsedCommandLine
                     .getOptionValue("outputDirectory") : "";
             SCENARIO = (parsedCommandLine.hasOption("scenario")) ? parsedCommandLine.getOptionValue("scenario") : "";
             METADATA_FILE = (parsedCommandLine.hasOption("metadataFile")) ? parsedCommandLine
                     .getOptionValue("metadataFile") : "";
             MODEL = "";
             CASE = "";
             ROW_CHUNK_SIZE = (parsedCommandLine.hasOption("rows")) ? Integer.valueOf(parsedCommandLine
                     .getOptionValue("rows")) : ROW_CHUNK_SIZE;
             String startingLatitude = (parsedCommandLine.hasOption("startLat")) ? parsedCommandLine
                     .getOptionValue("startLat") : "";
             String endingLatitude = (parsedCommandLine.hasOption("endLat")) ? parsedCommandLine
                     .getOptionValue("endLat") : "";
 
             boolean doSplitByDecade = parsedCommandLine.hasOption("decade");
             boolean doSplitByLatitude = parsedCommandLine.hasOption("latitude");
 
             MdbsyScenarioAConverter converter = new MdbsyScenarioAConverter();
 
             if (doSplitByDecade)
             {
                 long start = System.currentTimeMillis();
                 converter.splitByDecade(startingLatitude, endingLatitude);
                 long end = System.currentTimeMillis() - start;
                 LOG.warn("Split by decade in: " + end + " ms.");
             }
 
             if (doSplitByLatitude)
             {
                 long start = System.currentTimeMillis();
                 converter.splitByLatitude(startingLatitude, endingLatitude);
                 long end = System.currentTimeMillis() - start;
                 LOG.warn("Split by latitude in: " + end + " ms.");
             }
         }
         catch (MissingOptionException moe)
         {
             LOG.error(moe.getMessage());
 
             StringWriter sw = new StringWriter();
             HelpFormatter formatter = new HelpFormatter();
             formatter.setOptionComparator(new CommandLineOptionsComparator());
             formatter.printHelp(new PrintWriter(sw), 80, "-", "header", options, 0, 1, "footer");
             System.out.println(sw.toString());
         }
     }
 
     /**
      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by degrees of latitude.
      * 
      * @throws Exception
      */
     public void splitByLatitude(String startingLatitude, String endingLatitude) throws Exception
     {
         // group each row in lookup file according to latitude
         Map<String, List<CellData>> triplesGroupedByLatitude = this.getTriplesGroupedByLatitude(this.readLookupFile());
 
         // sort by latitude
         Set<String> latitudeKeySet = triplesGroupedByLatitude.keySet();
         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(latitudeKeySet, startingLatitude, endingLatitude);
         List<String> latitudeKeyList = new ArrayList<String>(limitedLatitudes);
         Collections.sort(latitudeKeyList, new Comparator<String>()
         {
             @Override
             public int compare(String o1, String o2)
             {
                 return Integer.valueOf(o2).compareTo(Integer.valueOf(o1));
             }
         });
 
         // for each latitude create a netCDF file.
         for (String latitudeKey : latitudeKeyList)
         {
             LOG.warn("Started work on latitude: " + latitudeKey + ".");
             String latitudeDegree = latitudeKey;
             List<CellData> triples = triplesGroupedByLatitude.get(latitudeDegree);
 
             // get the values to fill the coordinate variables.
             Set<String> sortedLatitudes = this.getSortedLatitudes(triples);
             int blockLatSize = sortedLatitudes.size();
             String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);
 
             Set<String> sortedLongitudes = this.getSortedLongitudes(triples);
             int blockLongSize = sortedLongitudes.size();
             String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);
 
             // create file names
             String filenames[] = new String[numVariables];
             List<String> dates = this.getDatesAsDaysSinceEpoc(this.getDates());
             for (int i = 0; i < filenames.length; i++)
             {
                 filenames[i] = this.generateLatitudeFilename(latitudeDegree, variableNames[i]);
                 this.createVariableFile(filenames[i], Arrays.asList(LAT, LONG, TIME), i, TIME_SIZE, blockLatSize,
                         blockLongSize);
                 this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,
                         blockLongRange, TIME_RANGE);
                 this.fillSpatialVariables(filenames[i], triples);
                 this.fillDataForLatitude(filenames[i], i, sortedLongitudes, sortedLatitudes, dates, Arrays.asList(LAT,
                         LONG, TIME));
             }
 
             // for each cellId csv file we read, we will have available the values for both rainFall and PVET, therefore
             // it makes sense to fill these variables at the same time.
             this.fillDataByLatitudeVariables(filenames, triples, Arrays.asList(LAT, LONG, TIME));
         }
     }
 
     /**
      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by decade.
      * 
      * @return
      * 
      * @throws Exception
      */
     public void splitByDecade(String startingLatitude, String endingLatitude) throws Exception
     {
         // get all rows in lookup file
         List<CellData> triples = this.getAllTriples(this.readLookupFile());
 
         // get the values to fill the coordinate variables.
         Set<String> sortedLatitudes = this.getSortedLatitudes(triples);
         String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);
 
         Set<String> sortedLongitudes = this.getSortedLongitudes(triples);
         String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);
         String startingLongitude = new ArrayList<String>(sortedLongitudes).get(0);
 
         // groups dates by decade
         Map<Integer, List<String>> datesByDecade = this.getDatesByDecade(this.getDates());
 
         // sort by decade
         Set<Integer> decadeKeySet = datesByDecade.keySet();
         List<Integer> decadeKeyList = new ArrayList<Integer>(decadeKeySet);
         Collections.sort(decadeKeyList);
         Map<Integer, String[]> allFilenames = new HashMap<Integer, String[]>();
         
         // for each decade create a netCDF file.
         int start = 0;
         int end = start;
         int filenameStart = 0;
         for (Integer decadeKey : decadeKeyList)
         {
             LOG.warn("Started work on decade: " + decadeKey + "0s.");
             System.gc();
             LOG.warn("  Mem: " + getMemDisplay());
             List<String> dates = datesByDecade.get(decadeKey);
             end = end + dates.size();
 
             int blockTimeSize = dates.size();
             String blockTimeRange = "0-" + String.valueOf(dates.size() - 1);
 
 //            if (decadeKey > 189)
 //            {
 //                continue;
 //            }
             // create file names
             String decadeStr = String.valueOf(decadeKey + "0");
 
             String filenames[] = new String[numVariables];
             dates = this.getDatesAsDaysSinceEpoc(dates);
             for (int i = 0; i < filenames.length; i++)
             {
                 filenames[i] = this.generateDecadeFilename(decadeStr, variableNames[i]);
                 if ("".equals(startingLatitude))
                 {
                     this.createVariableFile(filenames[i], Arrays.asList(TIME, LAT, LONG), i, blockTimeSize,
                             LAT_SIZE, LONG_SIZE);
                     this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,
                             blockLongRange, blockTimeRange);
                     this.fillSpatialVariables(filenames[i], triples);
                 }
             }
             
             allFilenames.put(decadeKey, filenames);
             filenameStart += numVariables;
         }
 
         LOG.warn(" Latitude limits : " + startingLatitude + " to " + endingLatitude + ".");
         
         int numBlocks = LAT_SIZE / ROW_CHUNK_SIZE + (LAT_SIZE % ROW_CHUNK_SIZE > 0 ? 1 : 0);
         Calendar baseCal = convUtils.getCalendar(new Date());
         baseCal.clear();
         baseCal.set(1889, 0, 1);
         
         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(sortedLatitudes, startingLatitude, endingLatitude);
         for (int blockNum = 0; blockNum < numBlocks; blockNum++)
         {
             LOG.warn(" Processing block : " + blockNum + ".");
             LOG.warn("  Mem: " + getMemDisplay());
             // Grab the next set of latitudes to be read
             List<String> lats = convUtils.getLatitudeBlock(limitedLatitudes, blockNum, ROW_CHUNK_SIZE);
             if (lats.isEmpty())
             {
                 LOG.warn(" Skipping block as it is outside the latitude range.");
             }
             else
             {
                 LOG.warn(" Processing latitudes : " + lats + ".");
                 
                 // Build a set of files covering those latitudes
                 List<CellData> targetCells = convUtils.buildCsvFilenamesForLatitudes(lats, triples, INPUT_CSV_DIRECTORY, ".csv");
                 
                 // Read all data from the file set
                 LOG.warn(" Reading data from CSVs...");
                 Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(targetCells, lats,
                         numVariables, TIME_SIZE, LONG_SIZE, variableFillValues, startingLongitude);
                 
                 // Write out the data in latitude blocks
                 LOG.warn(" Writing data to netCDF...");
                 LOG.warn("  Mem: " + getMemDisplay());
                 convUtils.writeLatDataByDecade(latData, allFilenames, variableNames, baseCal.getTime());
     
                 LOG.warn(" Writing finished.");
             }
         }
     }
 
     public String getMemDisplay() 
     {
         Runtime rt = Runtime.getRuntime();
         NumberFormat numFmt = NumberFormat.getNumberInstance();
         numFmt.setMaximumFractionDigits(0);
         StringBuffer sb = new StringBuffer();
         sb.append(numFmt.format(rt.totalMemory() / 1024.0));
         sb.append(" Kb total, ");
         sb.append(numFmt.format(rt.freeMemory() / 1024.0));
         sb.append(" Kb free, ");
         sb.append(numFmt.format(rt.maxMemory() / 1024.0));
         sb.append(" Kb max.");
         return sb.toString();
     }
     
     /**
      * Defines a netCDF file for the rainFall variable, but does not fill the variables
      * 
      * @param outputFileName
      * @throws Exception
      */
     private void createVariableFile(String outputFileName, List<String> dimensionOrdering, int variableIndex, int blockTimeSize, int blockLatSize, int blockLongSize)
             throws Exception
     {
         this.defineDimensions(outputFileName, dimensionOrdering, blockTimeSize, blockLatSize, blockLongSize);
         this.defineCoordinateVariables(outputFileName);
         this.defineSpatialVariables(outputFileName);
 
         // Add attributes
         DateFormat utcDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
         Attribute converter = new Attribute("History", utcDateTime.format(new Date()) + " Converted to netCDF by "
                 + "$Id: MdbsyScenarioAConverter.java 84 2010-08-25 05:56:46Z robertbridle $");
         List<Attribute> globalAttrs = new ArrayList<Attribute>();
         globalAttrs.add(converter);
         if (METADATA_FILE.length() > 0)
         {
             globalAttrs.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(METADATA_FILE)));
         }
         NcDefineAttributes ncDefineAttr = new NcDefineAttributes();
         ncDefineAttr.execute(outputFileName, globalAttrs, false /* isLargeFileSupport */);
 
         String dimensions = "";
         for (String dimension : dimensionOrdering)
         {
             dimensions += dimension + " ";
         }
         this.defineVariable(outputFileName, dimensions, variableIndex);
     }
 
     /**
      * Define the dimensions for a WRON netCDF file, e.g. latitude, longitude, time.
      * 
      * @param outputFileName
      * @throws IllegalArgumentException
      * @throws ParseException
      * @throws IOException
      */
     private void defineDimensions(String outputFileName, List<String> dimensionOrdering, int blockTimeSize, int blockLatSize, int blockLongSize)
             throws IllegalArgumentException, ParseException, IOException
     {
         NcDefineDimension command = new NcDefineDimension();
 
         for (String dimName : dimensionOrdering)
         {
             if (dimName.equals(TIME))
             {
                 command.execute(outputFileName, TIME, blockTimeSize, false /* isUnlimited */, false/* fillValue */);
             }
             else if (dimName.equals(LAT))
             {
                 command.execute(outputFileName, LAT, blockLatSize, false /* isUnlimited */, false/* fillValue */);
             }
             else if (dimName.equals(LONG))
             {
                 command.execute(outputFileName, LONG, blockLongSize, false /* isUnlimited */, false/* fillValue */);
             }
         }
         
         // Add NV as the final dimension
         command.execute(outputFileName, NV, NV_SIZE, false /* isUnlimited */, false/* fillValue */);
     }
 
     /**
      * Defines the coordinate variables for a WRON netCDF file, e.g. latitude, longitude, time.
      * 
      * @param outputFileName
      * @throws IllegalArgumentException
      * @throws ParseException
      * @throws IOException
      */
     private void defineCoordinateVariables(String outputFileName) throws IllegalArgumentException, ParseException,
             IOException
     {
         NcDefineVariable command = new NcDefineVariable();
 
         command.execute(new File(outputFileName), LAT, DataType.FLOAT, Arrays.asList(new Attribute("units", LAT_UNITS),
                 new Attribute("long_name", LAT_LONG_NAME), new Attribute("standard_name", LAT_STANDARD_NAME), new Attribute("axis", LAT_AXIS)), LAT,
                 false /* isLargeFileSupport */, false/* fillValue */);
         command.execute(new File(outputFileName), LONG, DataType.FLOAT, Arrays.asList(
                 new Attribute("long_name", LONG_LONG_NAME), new Attribute("units", LONG_UNITS), new Attribute("standard_name", LONG_STANDARD_NAME), new Attribute(
                         "axis", LONG_AXIS)), LONG, false /* isLargeFileSupport */, false/* fillValue */);
         command.execute(new File(outputFileName), TIME, DataType.INT, Arrays.asList(new Attribute("units", TIME_UNITS),
                 new Attribute("long_name", TIME_LONG_NAME), new Attribute("standard_name", TIME_STANDARD_NAME),
                 new Attribute("axis", TIME_AXIS), new Attribute("calendar", TIME_CALENDAR), new Attribute("bounds",
                         TIME_BOUNDS)), TIME, false /* isLargeFileSupport */, false/* fillValue */);
         command.execute(new File(outputFileName), TIME_BOUNDS, DataType.INT, new ArrayList<Attribute>(), TIME + " "
                 + NV, false /* isLargeFileSupport */, false/* fillValue */);
         command.execute(new File(outputFileName), GRID_MAPPING, DataType.INT, Arrays.asList(new Attribute(
                 "grid_mapping_name", CRS_GRID_MAPPING_NAME), new Attribute("longitude_of_prime_meridian",
                 CRS_LONGITUDE_OF_PRIME_MERIDIAN), new Attribute("semi_major_axis", CRS_SEMI_MAJOR_AXIS),
                 new Attribute("inverse_flattening", CRS_INVERSE_FLATTENING)), "",
                 false /* isLargeFileSupport */, true/* fillValue */);
     }
 
     /**
      * Defines the spatial metadata variables for a WRON netCDF file, e.g. elevation, catchment id etc.
      * 
      * @param outputFileName
      *            The file to written to
      * @throws IOException
      *             If the definitions cannot be written
      */
     private void defineSpatialVariables(String outputFileName) throws IOException
     {
         NcDefineVariable command = new NcDefineVariable();
 
         String spatialDim = LAT + " " + LONG;
         command.execute(new File(outputFileName), ELEVATION, DataType.FLOAT, Arrays.asList(new Attribute("units",
                 ELEVATION_UNITS), new Attribute("standard_name", ELEVATION_STANDARD_NAME), new Attribute("long_name",
                 ELEVATION_LONG_NAME), new Attribute("_FillValue", ELEVATION_MISSING_VALUE), new Attribute(
                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);
         command
                 .execute(new File(outputFileName), CATCHMENT_ID, DataType.FLOAT, Arrays.asList(new Attribute(
                         "long_name", CATCHMENT_ID_LONG_NAME), new Attribute("_FillValue", CATCHMENT_ID_MISSING_VALUE), new Attribute(
                         "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);
         command
                 .execute(new File(outputFileName), REPORTING_REGION, DataType.FLOAT, Arrays.asList(new Attribute(
                         "long_name", REPORTING_REGION_LONG_NAME), new Attribute("_FillValue", REPORTING_REGION_MISSING_VALUE), new Attribute(
                                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */,
                         false/* fillValue */);
     }
 
     /**
      * Defines the rainFall variable
      * 
      * @param outputFileName
      * @throws IllegalArgumentException
      * @throws IOException
      * @throws ParseException
      */
     private void defineVariable(String outputFileName, String dimensions, int variableIndex)
             throws IllegalArgumentException, IOException, ParseException
     {
         NcDefineVariable command = new NcDefineVariable();
         command
                 .execute(new File(outputFileName), variableNames[variableIndex], variableDataTypes[variableIndex],
                         Arrays.asList(new Attribute("units", variableUnits[variableIndex]), new Attribute("long_name",
                                 variableLongNames[variableIndex]), new Attribute("missing_value", Float
                                 .valueOf(variableMissingValues[variableIndex])), new Attribute("_FillValue", Float
                                 .valueOf(variableFillValues[variableIndex])), new Attribute("valid_min", Float
                                 .valueOf(variableMinValues[variableIndex])), new Attribute("valid_max", Float
                                 .valueOf(variableMaxValues[variableIndex])), new Attribute("grid_mapping", 
                                 GRID_MAPPING)), dimensions,
                         false /* isLargeFileSupport */, false/* fillValue */);
     }
 
     /**
      * Fills the coordinate variables, e.g. latitude, longitude, time.
      * 
      * @param outputFileName
      * @param sortedLongitudes
      * @param sortedLatitudes
      * @param dates
      * @param timeRange 
      * @param timeSize 
      * @param blockLongRange 
      * @param blockLongSize 
      * @param blockLatRange 
      * @param blockLatSize 
      * @throws UnsupportedEncodingException
      * @throws IOException
      * @throws java.text.ParseException
      */
     private void fillCoordinateVariables(String outputFileName, Set<String> sortedLongitudes,
             Set<String> sortedLatitudes, List<String> dates, String blockLatRange, String blockLongRange, String timeRange) throws UnsupportedEncodingException, IOException,
             java.text.ParseException
     {
         NcWriteVariable command = new NcWriteVariable();
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill longitude variable
         // create String of latitudes to populate the variable
         StringBuffer longitudeStringBuffer = new StringBuffer();
         for (Iterator<String> iter = sortedLongitudes.iterator(); iter.hasNext();)
         {
             longitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));
         }
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), LONG, blockLongRange, new ByteArrayInputStream(longitudeStringBuffer
                 .toString().getBytes(ENCODING)), false);
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill latitude variable
         // create String of latitudes to populate the variable
         StringBuffer latitudeStringBuffer = new StringBuffer();
         for (Iterator<String> iter = sortedLatitudes.iterator(); iter.hasNext();)
         {
             latitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));
         }
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), LAT, blockLatRange, new ByteArrayInputStream(latitudeStringBuffer
                 .toString().getBytes(ENCODING)), false);
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill time and time_bnds variables
         StringBuffer timeBoundsStringBuffer = new StringBuffer();
         StringBuffer dateStringBuffer = new StringBuffer();
         boolean first = true;
         for (String date : dates)
         {
             dateStringBuffer.append(date).append(System.getProperty("line.separator"));
             if (!first)
             {
                 timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));
             }
             timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));
             first = false;
         }
         int dayAfterLastDay = Integer.parseInt(dates.get(dates.size()-1)) +1;
         String finalString = String.valueOf(dayAfterLastDay);
         timeBoundsStringBuffer.append(finalString).append(System.getProperty("line.separator"));
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), TIME, timeRange, new ByteArrayInputStream(dateStringBuffer
                 .toString().getBytes(ENCODING)), false);
                 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), TIME_BOUNDS, timeRange+","+NV_RANGE, new ByteArrayInputStream(
                 timeBoundsStringBuffer.toString().getBytes(ENCODING)), false);
         
     }
 
     /**
      * Populates the spatial variables with data
      * 
      * @param outputFileName
      *            The netCDF file to be updated
      * @param sortedCellData
      *            The list of cell data in the same order as the spatial coordinates
      * @throws IOException
      *             If the data cannot be read
      */
     private void fillSpatialVariables(String outputFileName, List<CellData> sortedCellData) throws IOException
     {
         NcWriteVariable command = new NcWriteVariable();
 
         Set<String> latitudes = getSortedLatitudes(sortedCellData);
         Set<String> longitudes = getSortedLongitudes(sortedCellData);
         String spatialRange = "0-" + (latitudes.size() - 1) + ",0-" + (longitudes.size() - 1);
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill elevation variable
         // create String of elevations to populate the variable
         StringBuffer dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes,
                 CellDataVariable.ELEVATION, "-999.99");
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), ELEVATION, spatialRange, new ByteArrayInputStream(dataBuffer
                 .toString().getBytes(ENCODING)), false);
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill catchment id variable
         // create String of catchment ids to populate the variable
         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.CATCHMENTID, "0");
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), CATCHMENT_ID, spatialRange, new ByteArrayInputStream(dataBuffer
                 .toString().getBytes(ENCODING)), false);
 
         // ///////////////////////////////////////////////////////////////////////////////////////////
         // fill reporting region id variable
         // create String of reporting region ids to populate the variable
         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.REPORTINGREGIONID,
                 "0");
 
         // call command to actually perform fill the file
         command.execute(new File(outputFileName), REPORTING_REGION, spatialRange, new ByteArrayInputStream(dataBuffer
                 .toString().getBytes(ENCODING)), false);
     }
 
     /**
      * @param sortedCellData
      * @param latitudes
      * @param longitudes
      * @param varNum
      * @param fillValue
      * @return
      */
     private StringBuffer fillSpatialDataBuffer(List<CellData> sortedCellData, Set<String> latitudes,
             Set<String> longitudes, CellDataVariable varNum, String fillValue)
     {
         StringBuffer dataBuffer = new StringBuffer();
         int cellNum = 0;
         for (String latVal : latitudes)
         {
             for (String longVal : longitudes)
             {
                 if (cellNum < sortedCellData.size())
                 {
                     CellData cellData = sortedCellData.get(cellNum);
                     if (cellData.latitude.equals(latVal) && cellData.longitude.equals(longVal))
                     {
                         switch (varNum)
                         {
                         case ELEVATION:
                             dataBuffer.append(cellData.elevation);
                             break;
                         case CATCHMENTID:
                             dataBuffer.append(cellData.catchmentId);
                             break;
                         case REPORTINGREGIONID:
                             dataBuffer.append(cellData.reportingRegionId);
                             break;
                         }
                         cellNum++;
                     }
                     else
                     {
                         dataBuffer.append(fillValue);
                     }
                 }
                 else
                 {
                     dataBuffer.append(fillValue);
                 }
 
                 dataBuffer.append(System.getProperty("line.separator"));
             }
 
         }
         return dataBuffer;
     }
 
     /**
      * Populate the data variable with fill values. Used to ensure cells within 
      * the rectangular bounds but for which we have no data are set to the fill value. 
      *   
      * @param varIndex The variable being filled
      * @param sortedLongitudes The longitudes to fill.
      * @param sortedLatitudes The latitudes to fill
      * @param dates The dates to fill
      * @param dimensionOrdering The order of the dimensions in the files
      * @throws IOException If the data cannot be written
      */
     private void fillDataForLatitude(String filename, int varIndex, Set<String> sortedLongitudes, Set<String> sortedLatitudes,
             List<String> dates, List<String> dimensionOrdering) throws IOException
     {
         StringBuffer fillBlock = new StringBuffer();
         String fillLine = variableFillValues[varIndex] + "\n";
         for (int i = 0; i < dates.size(); i++)
         {
             fillBlock.append(fillLine);
         }
 
         NcWriteVariable writeVarCmd = new NcWriteVariable();
         for (String latitude : sortedLatitudes)
         {
             for (String longitude : sortedLongitudes)
             {
                 String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, dates.size());
                 writeVarCmd.execute(new File(filename), variableNames[varIndex], fillRange,
                         new ByteArrayInputStream(fillBlock.toString().getBytes(ENCODING)), false);
             }
         }
     }
 
     /**
      * Fills the variables with data for a particular latitude.
      * 
      * @param variableFileNames The filenames for this latitude for each of the variables
      * @param triples The details of the cells that occur in this latitude 
      * @param dimensionOrdering The order of the dimensions in the files
      * @throws IOException If the data cannot be written
      */
     private void fillDataByLatitudeVariables(String[] variableFileNames, List<CellData> triples, List<String> dimensionOrdering)
             throws IOException
     {
         NcWriteVariable command = new NcWriteVariable();
 
         int counter = 0;
         for (CellData triple : triples)
         {
             if (++counter % 1000 == 0)
             {
                 LOG.warn("   Num files processed: " + counter);
             }
 
             String csvFileName = triple.getCellId();
             String latitude = triple.getLatitude();
             String longitude = triple.getLongitude();
 
             String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, TIME_SIZE);
 
             try
             {
                 StringBuffer[] valuesStringBuffer = new StringBuffer[numVariables];
                 for (int i = 0; i < valuesStringBuffer.length; i++)
                 {
                     valuesStringBuffer[i] = new StringBuffer();
                 }
 
                 Scanner s = null;
                 try
                 {
                     s = new Scanner(new BufferedReader(new FileReader(INPUT_CSV_DIRECTORY + csvFileName + ".csv")))
                             .useDelimiter(Pattern.compile("[\n,]"));
 
                     // remove header line
                     for (int i = 0; i < NUM_CSV_COLUMNS; i++)
                     {
                         s.next();
                     }
 
                     while (s.hasNext())
                     {
                         s.next(); // date column
 
                         for (int i = 0; i < numVariables; i++)
                         {
                             valuesStringBuffer[i].append(s.next());
                             // Last variable will already have newline at its end.
                             if (i < numVariables - 1)
                             {
                                 valuesStringBuffer[i].append(System.getProperty("line.separator"));
                             }
                         }
                     }
                 }
                 finally
                 {
                     if (s != null)
                     {
                         s.close();
                     }
                 }
 
                 try
                 {
                     for (int i = 0; i < numVariables; i++)
                     {
                         command.execute(new File(variableFileNames[i]), variableNames[i], fillRange,
                                 new ByteArrayInputStream(valuesStringBuffer[i].toString().getBytes(ENCODING)), false);
                     }
                 }
                 catch (IllegalArgumentException iae)
                 {
                     LOG.error(iae);
                     LOG.error("fillRange (date, lat, long): " + fillRange);
                 }
 
             }
             catch (FileNotFoundException fnfe)
             {
                 LOG.info(fnfe.getMessage());
                 //System.out.println(fnfe.getMessage());
             }
         }
     }
 
     /**
      * @param dimensionOrdering
      * @param latitude
      * @param longitude
      * @param dates
      * @return
      */
     private String constructFillRange(List<String> dimensionOrdering, String latitude, String longitude, int dateSize)
     {
         String time_range = "0-" + String.valueOf(dateSize - 1);
         String latitude_range = "lookup(" + latitude + ")-lookup(" + latitude + ")";
         String longitude_range = "lookup(" + longitude + ")-lookup(" + longitude + ")";
 
         String fillRange = new String();
         for (Iterator<String> iter = dimensionOrdering.iterator(); iter.hasNext();)
         {
             String dimension = iter.next();
             if (dimension.equals(TIME))
             {
                 fillRange += time_range;
             }
             else if (dimension.equals(LAT))
             {
                 fillRange += latitude_range;
             }
             else if (dimension.equals(LONG))
             {
                 fillRange += longitude_range;
             }
             if (iter.hasNext())
                 fillRange += ",";
         }
         return fillRange;
     }
 
     /**
      * Groups triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file) by degrees of latitude.
      * 
      * @param locationLookupValues
      * @return
      */
     private Map<String, List<CellData>> getTriplesGroupedByLatitude(String[][] locationLookupValues)
     {
         Map<String, List<CellData>> triplesGroupedByLatitude = new HashMap<String, List<CellData>>();
         for (int i = 0; i < locationLookupValues.length; i++)
         {
             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];
             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];
             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];
             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];
             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];
             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];
 
             String latitudeDegree = String.valueOf(Float.valueOf(latitude).intValue());
 
             List<CellData> triplesInLatitude = triplesGroupedByLatitude.get(latitudeDegree);
             if (triplesInLatitude == null)
             {
                 triplesInLatitude = new ArrayList<CellData>();
                 triplesGroupedByLatitude.put(latitudeDegree, triplesInLatitude);
             }
             final CellData cellData = new CellData(cellId, longitude, latitude);
             cellData.elevation = elevation;
             cellData.catchmentId = catchmentId;
             cellData.reportingRegionId = reportingRegionId;
             triplesInLatitude.add(cellData);
         }
 
         // check that we have actually grouped all rows by latitude
         int count = 0;
         for (Iterator<String> iter = triplesGroupedByLatitude.keySet().iterator(); iter.hasNext();)
         {
             count += triplesGroupedByLatitude.get(iter.next()).size();
         }
         if (count != locationLookupValues.length)
         {
             LOG.error("latitudes where not grouped by latitude correctly");
             System.exit(-1);
         }
 
         return triplesGroupedByLatitude;
     }
 
     /**
      * Groups dates, e.g. 1895-01-01 by decade.
      * 
      * @param dates
      * @return
      */
     public Map<Integer, List<String>> getDatesByDecade(List<String> dates)
     {
         Map<Integer, List<String>> datesGroupedByDecade = new HashMap<Integer, List<String>>();
 
         for (String date : dates)
         {
             int year = Integer.valueOf(date.substring(0, 4));
             int decade = year / 10;
 
             List<String> datesInDecade = datesGroupedByDecade.get(decade);
             if (datesInDecade == null)
             {
                 datesInDecade = new ArrayList<String>();
                 datesGroupedByDecade.put(decade, datesInDecade);
             }
             datesInDecade.add(date);
         }
 
         // check that we have actually grouped all rows by latitude
         int count = 0;
         for (Iterator<Integer> iter = datesGroupedByDecade.keySet().iterator(); iter.hasNext();)
         {
             count += datesGroupedByDecade.get(iter.next()).size();
         }
         if (count != dates.size())
         {
             LOG.error("dates where not grouped decade correctly");
             System.exit(-1);
         }
 
         return datesGroupedByDecade;
     }
 
     /**
      * Loads all triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file).
      * 
      * @param locationLookupValues
      * @return
      */
     private List<CellData> getAllTriples(String[][] locationLookupValues)
     {
         List<CellData> allTriples = new ArrayList<CellData>();
         for (int i = 0; i < locationLookupValues.length; i++)
         {
             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];
             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];
             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];
             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];
             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];
             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];
 
             CellData cellData = new CellData(cellId, longitude, latitude);
             cellData.elevation = elevation;
             cellData.catchmentId = catchmentId;
             cellData.reportingRegionId = reportingRegionId;
             allTriples.add(cellData);
         }
 
         // check that we have actually grouped all rows by latitude
         if (allTriples.size() != locationLookupValues.length)
         {
             LOG.error("lookup values where not read correctly");
             System.exit(-1);
         }
 
         return allTriples;
     }
 
     /**
      * Parses the WRON lookup file into an row-by-column string matrix.
      * 
      * @throws IOException
      */
     private String[][] readLookupFile() throws IOException
     {
         RandomAccessFile raf = new RandomAccessFile(new File(LOCATION_LOOKUP_FILE), "r");
         raf.readLine();// abandon the first line since it is a header line.
 
         List<String[]> lineList = new ArrayList<String[]>();
         String line = raf.readLine();
         while (line != null)
         {
             String[] cols = new CSVTokenizer(line).getAllColumns();
             lineList.add(cols);
             line = raf.readLine();
         }
 
         raf.close();
 
         String[][] lineCols = lineList.toArray(new String[][]{});
         return lineCols;
     }
 
     /**
      * Transpose a matrix, e.g. [a][b] -> [b][a]
      * 
      * @param gridValues
      * @return
      */
     private String[][] transpose(String[][] gridValues)
     {
         String[][] transposed = new String[gridValues[0].length][gridValues.length];
         for (int rows = 0; rows < gridValues.length; rows++)
         {
             for (int cols = 0; cols < gridValues[0].length; cols++)
             {
                 transposed[cols][rows] = gridValues[rows][cols];
             }
         }
         return transposed;
     }
 
     /**
      * Retrieve an ordered set containing all the latitude values from a list of triples.
      * 
      * @param triples
      * @return
      */
     private Set<String> getSortedLatitudes(List<CellData> triples)
     {
         Set<String> latitudes = new TreeSet<String>();
         for (CellData triple : triples)
         {
             latitudes.add(triple.getLatitude());
 //            if (latitudes.size() >= LAT_SIZE)
 //            {
 //                break;
 //            }
         }
         return latitudes;
     }
 
     /**
      * Retrieve an ordered set containing all the longitude values from a list of triples.
      * 
      * @param triples
      * @return
      */
     private Set<String> getSortedLongitudes(List<CellData> triples)
     {
         Set<String> longitudes = new TreeSet<String>();
         for (CellData triple : triples)
         {
             longitudes.add(triple.getLongitude());
 //            if (longitudes.size() >= LONG_SIZE)
 //            {
 //                break;
 //            }
         }
         return longitudes;
     }
 
     /**
      * @return
      * @throws IOException
      */
     private List<String> getDates() throws IOException
     {
         List<String> dates = new ArrayList<String>();
 
         RandomAccessFile raf = null;
         try
         {
             // open any csv file from scenario A directory and read in the date dimension
             File dir = new File(INPUT_CSV_DIRECTORY);
             String[] children = dir.list();
             if (children != null && children.length > 0)
             {
                 raf = new RandomAccessFile(new File(INPUT_CSV_DIRECTORY + children[0]), "r");
                 raf.readLine();// abandon the first line since it is a header line.
 
                 List<String[]> lineList = new ArrayList<String[]>();
                 String line = raf.readLine();
                 while (line != null)
                 {
                     String[] cols = new CSVTokenizer(line).getAllColumns();
                     lineList.add(cols);
                     line = raf.readLine();
                 }
                 String[][] lineCols = lineList.toArray(new String[][]{});
                 dates = Arrays.asList(transpose(lineCols)[DATE_COLUMN_INDEX]); // get the date column
             }
         }
         finally
         {
             if (raf != null)
                 raf.close();
         }
 
         return dates;
     }
 
     /**
      * Convert the date values into a newline delimited string of days since a given epoc.
      * 
      * @param dates
      * @return
      * @throws IOException
      * @throws java.text.ParseException
      */
     private List<String> getDatesAsDaysSinceEpoc(List<String> dates) throws IOException, java.text.ParseException
     {
         return daysSinceSinceEpocAsString(dates, isoDateTimeFormat.dateOnlyFormat(EPOC_DATE_STRING));
     }
 
     /**
      * Convert a string array representing dates to a list of strings representing the number of days since since a
      * given epoc.
      * 
      * @param dates
      * @param epoc
      * @return
      * @throws java.text.ParseException
      */
     private List<String> daysSinceSinceEpocAsString(List<String> dates, Date epoc) throws java.text.ParseException
     {
         List<String> convertedDates = new ArrayList<String>();
         for (String date : dates)
         {
             Date currentDate = isoDateTimeFormat.dateOnlyFormat(date);
             convertedDates.add(String.valueOf(daysBetween(epoc, currentDate)));
         }
         return convertedDates;
     }
 
     /**
      * @param d1
      *            the initial date.
      * @param d2
      *            the later date.
      * @return (d2 - d1) in days
      */
     public int daysBetween(Date d1, Date d2)
     {
         return (int) ((d2.getTime() - d1.getTime()) / (1000 * 60 * 60 * 24));
     }
 
     private String generateLatitudeFilename(String latitudeDegree, String variable)
     {
         String filename = OUTPUT_NETCDF_DIRECTORY + COLLECTION + "." + SCENARIO + ".";
         if (MODEL.length() > 0)
         {
             filename += MODEL + ".";
         }
         if (CASE.length() > 0)
         {
             filename += CASE + ".";
         }
         filename += "Latitude-";
         if (latitudeDegree.startsWith("-"))
         {
             filename += latitudeDegree.substring(1) + "S."; 
         }
         else
         {
             filename += latitudeDegree + "N.";
         }
         filename += "Var-" + variable + NETCDF_FILE_EXTENSION;
         return filename;
     }
 
     private String generateDecadeFilename(String decadeStr, String variable)
     {
         String filename = OUTPUT_NETCDF_DIRECTORY + COLLECTION + "." + SCENARIO + ".";
         if (MODEL.length() > 0)
         {
             filename += MODEL + ".";
         }
         if (CASE.length() > 0)
         {
             filename += CASE + ".";
         }
         filename += "Decade-" + decadeStr +  ".Var-" + variable + NETCDF_FILE_EXTENSION;
         return filename;
     }
 
     /**
      * Displays the contents of the in-memory data structure/
      * 
      * @param lookup
      */
     @SuppressWarnings("unused")
     private void printLookupInMemory(Map<Integer, Map<String, LongitudeRange>> lookup)
     {
         for (Iterator<Integer> dateIter = lookup.keySet().iterator(); dateIter.hasNext();)
         {
             Integer dateKey = dateIter.next();
             System.out.println(dateKey);
 
             for (Iterator<String> latIter = lookup.get(dateKey).keySet().iterator(); latIter.hasNext();)
             {
                 String latitude = latIter.next();
                 System.out.println("\t" + latitude);
 
                 LongitudeRange longitudeRange = lookup.get(dateKey).get(latitude);
                 System.out.println("\t\t" + longitudeRange.getStartLongitudeRange() + "-"
                         + longitudeRange.getEndLongitudeRange());
             }
         }
     }
 
     public enum CellDataVariable
     {
         CELLID, LONGITUDE, LATITUDE, ELEVATION, CATCHMENTID, REPORTINGREGIONID
     }
 
 }