/**

  * Copyright 2010, CSIRO Australia.

  *

  * Licensed under the Apache License, Version 2.0 (the "License");

  * you may not use this file except in compliance with the License.

  * You may obtain a copy of the License at

  *

  *         http://www.apache.org/licenses/LICENSE-2.0

  *

  * Unless required by applicable law or agreed to in writing, software

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  * See the License for the specific language governing permissions and

  * limitations under the License.

  */

 package au.csiro.netcdf.wron;

 import java.io.BufferedReader;

 import java.io.ByteArrayInputStream;

 import java.io.File;

 import java.io.FileInputStream;

 import java.io.FileNotFoundException;

 import java.io.FileReader;

 import java.io.IOException;

 import java.io.PrintWriter;

 import java.io.RandomAccessFile;

 import java.io.StringWriter;

 import java.io.UnsupportedEncodingException;

 import java.text.DateFormat;

 import java.text.NumberFormat;

 import java.text.SimpleDateFormat;

 import java.util.ArrayList;

 import java.util.Arrays;

 import java.util.Calendar;

 import java.util.Collections;

 import java.util.Comparator;

 import java.util.Date;

 import java.util.HashMap;

 import java.util.Iterator;

 import java.util.List;

 import java.util.Map;

 import java.util.Scanner;

 import java.util.Set;

 import java.util.TreeSet;

 import java.util.regex.Pattern;

 import org.apache.commons.cli.BasicParser;

 import org.apache.commons.cli.CommandLine;

 import org.apache.commons.cli.HelpFormatter;

 import org.apache.commons.cli.MissingOptionException;

 import org.apache.commons.cli.Option;

 import org.apache.commons.cli.OptionBuilder;

 import org.apache.commons.cli.Options;

 import org.apache.commons.cli.ParseException;

 import org.apache.log4j.Logger;

 import ucar.ma2.DataType;

 import ucar.nc2.Attribute;

 import ucar.nc2.units.DateFormatter;

 import au.csiro.netcdf.NcDefineAttributes;

 import au.csiro.netcdf.NcDefineDimension;

 import au.csiro.netcdf.NcDefineVariable;

 import au.csiro.netcdf.NcWriteVariable;

 import au.csiro.netcdf.cli.CommandLineOptionsComparator;

 import au.csiro.netcdf.util.CSVTokenizer;

 import au.csiro.netcdf.util.NetCDFUtils;

 /**

  * The class is a converter control class to convert Murray Darling Basin 

  * Sustainable Yields CSV files into netCDF files. This class will process 

  * the Scenario C historical dataset files.  

  * 

  * Copyright 2010, CSIRO Australia 

  * All rights reserved.

  * 

  * @author Robert Bridle on 17/04/2010

  * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $

  */

 public class MdbsyScenarioCConverter

 {

     // netCDF file naming components

     private static String COLLECTION = "Collection-MDBSY_Climate";

     private static String SCENARIO = "";

     private static String[] MODEL_NAMES;

     private static String CASE = "";

     /**

      * The file containing the location lookup values.

      */

     private static String LOCATION_LOOKUP_FILE = "";

     /**

      * The file containing the metadata attributes (if any).

      */

     private static String METADATA_FILE = "";

     /**

      * The directory containing the Scenario C csv data files.

      */

     private static String INPUT_CSV_DIRECTORY = "";

     /**

      * The netCDF file to be created.

      */

     private static String OUTPUT_NETCDF_DIRECTORY = "";

     /**

      * NetCDF file extension.

      */

     private static final String NETCDF_FILE_EXTENSION = ".nc";

     /**

      * Encoding used to fill variables.

      */

     private static final String ENCODING = "UTF-8";

     /**

      * The 15 climate models used in the MDBSY project.

      */

     private static Map<String, String> CLIMATE_MODELS = new HashMap<String, String>();

     static

     {

         CLIMATE_MODELS.put("cccma_t47", "CCCMA T47");

         CLIMATE_MODELS.put("cccma_t63", "CCCMA T63");

         CLIMATE_MODELS.put("cnrm", "CNRM");

         CLIMATE_MODELS.put("csiro", "CSIRO-MK3.0");

         CLIMATE_MODELS.put("gfdl", "GFDL 2.0");

         CLIMATE_MODELS.put("giss_aom", "GISS-AOM");

         CLIMATE_MODELS.put("iap", "IAP");

         CLIMATE_MODELS.put("inmcm", "INMCM");

         CLIMATE_MODELS.put("ipsl", "IPSL");

         CLIMATE_MODELS.put("miroc", "MIROC-M");

         CLIMATE_MODELS.put("miub", "MIUB");

         CLIMATE_MODELS.put("mpi", "MPI-ECHAM5");

         CLIMATE_MODELS.put("mri", "MRI");

         CLIMATE_MODELS.put("ncar_ccsm", "NCAR-CCSM");

         CLIMATE_MODELS.put("ncar_pcm", "NCAR-PCM1");

     }

     /**

      * Index of model names in {@link MdbsyScenarioCConverter#MODEL_NAMES}.

      */

     private static final int FILE_NAME_MODEL_NAME = 0;

     private static final int IPCC_MODEL_NAME = 1;

     // Variables

     private static final String[] variableNames = new String[] { "rainfall", "APET" };

     private static final String[] variableUnits = new String[] { "mm", "mm" };

     private static final String[] variableLongNames = new String[] { "rainfall", "APET" };

     private static final String[] variableFillValues = new String[] { "-9999.0f", "-9999.0f" };

     private static final String[] variableMissingValues = variableFillValues;

     private static final DataType[] variableDataTypes = new DataType[] { DataType.FLOAT, DataType.FLOAT };

     static final int numVariables = variableNames.length;

     private static final String GRID_MAPPING = "crs";

     // time constants

     private static final String TIME = "time";

     private static String TIME_RANGE = "0-40906";

     private static final int TIME_SIZE = 40907;

     private static final String TIME_LONG_NAME = "reference time";

     private static final String TIME_STANDARD_NAME = "time";

     private static final String TIME_UNITS = "days since 1895-01-01 0:0:0";

     private static final String TIME_CALENDAR = "Gregorian";

     private static final String TIME_AXIS = "T";

     private static final String TIME_BOUNDS = "time_bnds";

     // longitude constants

     private static final String LONG = "longitude";

     private static final int LONG_COLUMN_INDEX = 1;

     private static final String LONG_RANGE = "0-296";

     private static final int LONG_SIZE = 297;

     private static final String LONG_UNITS = "degrees_east";

     private static final String LONG_STANDARD_NAME = "longitude";

     private static final String LONG_AXIS = "X";

     private static final String LONG_LONG_NAME = "longitude";

     // latitude constants

     private static final String LAT = "latitude";

     private static final int LAT_COLUMN_INDEX = 2;

     private static final String LAT_RANGE = "0-278";

     private static final int LAT_SIZE = 279;

     private static final String LAT_UNITS = "degrees_north";

     private static final String LAT_STANDARD_NAME = "latitude";

     private static final String LAT_AXIS = "Y";

     private static final String LAT_LONG_NAME = "latitude";

     // num values constants

     private static final String NV = "nv";

     private static final String NV_RANGE = "0-1";

     private static final int NV_SIZE = 2;

     // elevation constants

     private static String ELEVATION = "elev";

     private static int ELEVATION_COLUMN_INDEX = 3;

     private static final String ELEVATION_UNITS = "m";

     private static final String ELEVATION_STANDARD_NAME = "altitude";

     private static final String ELEVATION_LONG_NAME = "Elevation above sea level";

     private static final float ELEVATION_MISSING_VALUE = -999.99f;

     // catchment id constants

     private static String CATCHMENT_ID = "catchmentId";

     private static int CATCHMENT_ID_COLUMN_INDEX = 4;

     private static final String CATCHMENT_ID_LONG_NAME = "MDB Catchment Id";

     private static final float CATCHMENT_ID_MISSING_VALUE = 0f;

     // reporting region id constants

     private static String REPORTING_REGION = "repRegionId";

     private static int REPORTING_REGION_ID_COLUMN_INDEX = 5;

     private static String REPORTING_REGION_LONG_NAME = "MDB Reporting Region Id";

     private static final float REPORTING_REGION_MISSING_VALUE = 0f;

     // crs constants

     private static String CRS_GRID_MAPPING_NAME = "latitude_longitude";  

     private static float CRS_LONGITUDE_OF_PRIME_MERIDIAN = 0.0f;

     private static float CRS_SEMI_MAJOR_AXIS = 6378137.0f;

     private static float CRS_INVERSE_FLATTENING = 298.257222101f;

     // cell id

     private static final int CELL_ID_COLUMN_INDEX = 0;

     // Other cell data

     // date conversion constants

     private static final String EPOC_DATE_STRING = "1895-01-01";

     private static final DateFormatter isoDateTimeFormat = new DateFormatter();

     private static final int DATE_COLUMN_INDEX = 0;

     // number of time values to read from all the csv files and load into memory.

     private static int ROW_CHUNK_SIZE = 1000;

     /** The number of columns expected in the CSV data files */

     private static final int NUM_CSV_COLUMNS = 1 + numVariables;

     /**

      * Constant that defines the logger to be used.

      */

     private static final Logger LOG = Logger.getLogger(MdbsyScenarioCConverter.class.getName());

     /** Conversion utilities  */

     private ConversionUtils convUtils = new ConversionUtils();

     /**

      * @param args

      */

     @SuppressWarnings("static-access")

     public static void main(String[] args) throws Exception

     {

         Options options = new Options();

         try

         {

             Option inputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(

                     " 1: the directory containing the csv files for the scenario.").isRequired(true).withLongOpt(

                     "inputDirectory").create("i");

             Option lookupFile = OptionBuilder.withArgName("file").hasArg()

                     .withDescription(" 2: the cellId lookup file.").isRequired(true).withLongOpt("lookupFile").create(

                             "f");

             Option outputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(

                     " 3: the output directory.").isRequired(true).withLongOpt("outputDirectory").create("o");

             Option scenario = OptionBuilder.withArgName("text").hasArg().withDescription(

                     " 4: the modelling scenario used to name the netCDF file.").isRequired(true).withLongOpt("scenario")

                     .create("s");

             Option model = OptionBuilder.withArgName("text").hasArg().withDescription(

                     " 5: the global climate change model used to name the netCDF file.").isRequired(true).withLongOpt(

                     "model").create("e");

             Option modelCase = OptionBuilder.withArgName("text").hasArg().withDescription(

                     " 6: the model case or sensitivity used to name the netCDF file.").isRequired(true).withLongOpt(

                     "case").create("c");

             Option metadataFile = OptionBuilder.withArgName("file").hasArg().withDescription(

                     " 7: a file of global metadata attributes to be added to the netCDF file.").isRequired(true)

                     .withLongOpt("metadataFile").create("m");

             Option numRows = OptionBuilder.withArgName("int").hasArg().withDescription(

                     " 8: the number of rows to read at a time.").isRequired(true).withLongOpt("rows").create("r");

             Option byDecade = OptionBuilder.withDescription(

                     " 9: OPTIONAL, set if we want to convert into netCDF files by decade.").isRequired(false)

                     .withLongOpt("decade").create("d");

             Option byLatitude = OptionBuilder.withDescription(

                     " 10: OPTIONAL, set if we want to convert into netCDF files by degrees of latitude.").isRequired(

                     false).withLongOpt("latitude").create("l");

             Option startLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(

                     " 11: OPTIONAL, The latitude at which to start processing. For byDecade, this option will "

                             + "assume already existing netCDF files.").isRequired(false).create("startLat");

             Option endLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(

                     " 12: OPTIONAL, The latitude at which to end processing (inclusive).").isRequired(false).create(

                     "endLat");

             options.addOption(inputDirectory);

             options.addOption(lookupFile);

             options.addOption(outputDirectory);

             options.addOption(scenario);

             options.addOption(model);

             options.addOption(modelCase);

             options.addOption(metadataFile);

             options.addOption(numRows);

             options.addOption(byDecade);

             options.addOption(byLatitude);

             options.addOption(startLat);

             options.addOption(endLat);

             // parse the command line arguments

             CommandLine parsedCommandLine = new BasicParser().parse(options, args);

             INPUT_CSV_DIRECTORY = (parsedCommandLine.hasOption("inputDirectory")) ? parsedCommandLine

                     .getOptionValue("inputDirectory") : "";

             LOCATION_LOOKUP_FILE = (parsedCommandLine.hasOption("lookupFile")) ? parsedCommandLine

                     .getOptionValue("lookupFile") : "";

             OUTPUT_NETCDF_DIRECTORY = (parsedCommandLine.hasOption("outputDirectory")) ? parsedCommandLine

                     .getOptionValue("outputDirectory") : "";

             SCENARIO = (parsedCommandLine.hasOption("scenario")) ? parsedCommandLine.getOptionValue("scenario") : "";

             METADATA_FILE = (parsedCommandLine.hasOption("metadataFile")) ? parsedCommandLine

                     .getOptionValue("metadataFile") : "";

             MODEL_NAMES = getModelMapping((parsedCommandLine.hasOption("model")) ? parsedCommandLine.getOptionValue("model") : "");

             CASE = (parsedCommandLine.hasOption("case")) ? parsedCommandLine.getOptionValue("case") : "";

             ROW_CHUNK_SIZE = (parsedCommandLine.hasOption("rows")) ? Integer.valueOf(parsedCommandLine

                     .getOptionValue("rows")) : ROW_CHUNK_SIZE;

             String startingLatitude = (parsedCommandLine.hasOption("startLat")) ? parsedCommandLine

                     .getOptionValue("startLat") : "";

             String endingLatitude = (parsedCommandLine.hasOption("endLat")) ? parsedCommandLine

                     .getOptionValue("endLat") : "";

             boolean doSplitByDecade = parsedCommandLine.hasOption("decade");

             boolean doSplitByLatitude = parsedCommandLine.hasOption("latitude");

             MdbsyScenarioCConverter converter = new MdbsyScenarioCConverter();

             if (doSplitByDecade)

             {

                 long start = System.currentTimeMillis();

                 converter.splitByDecade(startingLatitude, endingLatitude);

                 long end = System.currentTimeMillis() - start;

                 LOG.warn("Split by decade in: " + end + " ms.");

             }

             if (doSplitByLatitude)

             {

                 long start = System.currentTimeMillis();

                 converter.splitByLatitude(startingLatitude, endingLatitude);

                 long end = System.currentTimeMillis() - start;

                 LOG.warn("Split by latitude in: " + end + " ms.");

             }

         }

         catch (MissingOptionException moe)

         {

             LOG.error(moe.getMessage());

             StringWriter sw = new StringWriter();

             HelpFormatter formatter = new HelpFormatter();

             formatter.setOptionComparator(new CommandLineOptionsComparator());

             formatter.printHelp(new PrintWriter(sw), 80, "-", "header", options, 0, 1, "footer");

             System.out.println(sw.toString());

         }

     }

     /**

      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by degrees of latitude.

      * 

      * @throws Exception

      */

     public void splitByLatitude(String startingLatitude, String endingLatitude) throws Exception

     {

         // group each row in lookup file according to latitude

         Map<String, List<CellData>> triplesGroupedByLatitude = this.getTriplesGroupedByLatitude(this.readLookupFile());

         // sort by latitude

         Set<String> latitudeKeySet = triplesGroupedByLatitude.keySet();

         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(latitudeKeySet, startingLatitude, endingLatitude);

         List<String> latitudeKeyList = new ArrayList<String>(limitedLatitudes);

         Collections.sort(latitudeKeyList, new Comparator<String>()

         {

             @Override

             public int compare(String o1, String o2)

             {

                 return Integer.valueOf(o2).compareTo(Integer.valueOf(o1));

             }

         });

         // for each latitude create a netCDF file.

         for (String latitudeKey : latitudeKeyList)

         {

             LOG.warn("Started work on latitude: " + latitudeKey + ".");

             String latitudeDegree = latitudeKey;

             List<CellData> triples = triplesGroupedByLatitude.get(latitudeDegree);

             // get the values to fill the coordinate variables.

             Set<String> sortedLatitudes = this.getSortedLatitudes(triples);

             int blockLatSize = sortedLatitudes.size();

             String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);

             Set<String> sortedLongitudes = this.getSortedLongitudes(triples);

             int blockLongSize = sortedLongitudes.size();

             String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);

             // create file names

             String filenames[] = new String[numVariables];

             List<String> dates = this.getDatesAsDaysSinceEpoc(this.getDates());

             for (int i = 0; i < filenames.length; i++)

             {

                 filenames[i] = this.generateLatitudeFilename(latitudeDegree, variableNames[i]);

                 this.createVariableFile(filenames[i], Arrays.asList(LAT, LONG, TIME), i, TIME_SIZE, blockLatSize, blockLongSize);

                 this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,

                         blockLongRange, TIME_RANGE);

                 this.fillSpatialVariables(filenames[i], triples);

                 this.fillDataForLatitude(filenames[i], i, sortedLongitudes, sortedLatitudes, dates, Arrays.asList(LAT,

                         LONG, TIME));

             }

             // for each cellId csv file we read, we will have available the values for both rainFall and PVET, therefore

             // it makes sense to fill these variables at the same time.

             this.fillDataByLatitudeVariables(filenames, triples, Arrays.asList(LAT, LONG, TIME));

         }

     }

     /**

      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by decade.

      * 

      * @return

      * 

      * @throws Exception

      */

     public void splitByDecade(String startingLatitude, String endingLatitude) throws Exception

     {

         // get all rows in lookup file

         List<CellData> triples = this.getAllTriples(this.readLookupFile());

         // get the values to fill the coordinate variables.

         Set<String> sortedLatitudes = this.getSortedLatitudes(triples);

         String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);

         Set<String> sortedLongitudes = this.getSortedLongitudes(triples);

         String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);

         String startingLongitude = new ArrayList<String>(sortedLongitudes).get(0);

         // groups dates by decade

         Map<Integer, List<String>> datesByDecade = this.getDatesByDecade(this.getDates());

         // sort by decade

         Set<Integer> decadeKeySet = datesByDecade.keySet();

         List<Integer> decadeKeyList = new ArrayList<Integer>(decadeKeySet);

         Collections.sort(decadeKeyList);

         Map<Integer, String[]> allFilenames = new HashMap<Integer, String[]>();

         // for each decade create a netCDF file.

         int start = 0;

         int end = start;

         int filenameStart = 0;

         for (Integer decadeKey : decadeKeyList)

         {

             LOG.warn("Started work on decade: " + decadeKey + "0s.");

             System.gc();

             LOG.warn("  Mem: " + getMemDisplay());

             List<String> dates = datesByDecade.get(decadeKey);

             end = end + dates.size();

             int blockTimeSize = dates.size();

             String blockTimeRange = "0-" + String.valueOf(dates.size() - 1);

             // create file names

             String decadeStr = String.valueOf(decadeKey + "0");

             String filenames[] = new String[numVariables];

             dates = this.getDatesAsDaysSinceEpoc(dates);

             for (int i = 0; i < filenames.length; i++)

             {

                 filenames[i] = this.generateDecadeFilename(decadeStr, variableNames[i]);

                 if ("".equals(startingLatitude))

                 {

                     this.createVariableFile(filenames[i], Arrays.asList(TIME, LAT, LONG), i, blockTimeSize, LAT_SIZE, LONG_SIZE);

                     this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,

                             blockLongRange, blockTimeRange);

                     this.fillSpatialVariables(filenames[i], triples);

                 }

             }

             allFilenames.put(decadeKey, filenames);

             filenameStart += numVariables;

         }

         LOG.warn(" Latitude limits : " + startingLatitude + " to " + endingLatitude + ".");

         int numBlocks = LAT_SIZE / ROW_CHUNK_SIZE + (LAT_SIZE % ROW_CHUNK_SIZE > 0 ? 1 : 0);

         Calendar baseCal = convUtils.getCalendar(new Date());

         baseCal.clear();

         baseCal.set(1895, 0, 1);

         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(sortedLatitudes, startingLatitude, endingLatitude);

         for (int blockNum = 0; blockNum < numBlocks; blockNum++)

         {

             LOG.warn(" Processing block : " + blockNum + ".");

             LOG.warn("  Mem: " + getMemDisplay());

             // Grab the next set of latitudes to be read

             List<String> lats = convUtils.getLatitudeBlock(limitedLatitudes, blockNum, ROW_CHUNK_SIZE);

             if (lats.isEmpty())

             {

                 LOG.warn(" Skipping block as it is outside the latitude range.");

             }

             else

             {

                 LOG.warn(" Processing latitudes : " + lats + ".");

                 // Build a set of files covering those latitudes

                 List<CellData> targetCells = convUtils.buildCsvFilenamesForLatitudes(lats, triples, INPUT_CSV_DIRECTORY, ".csv");

                 // Read all data from the file set

                 LOG.warn(" Reading data from CSVs...");

                 Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(targetCells, lats,

                         numVariables, TIME_SIZE, LONG_SIZE, variableFillValues, startingLongitude);

                 // Write out the data in latitude blocks

                 LOG.warn(" Writing data to netCDF...");

                 LOG.warn("  Mem: " + getMemDisplay());

                 convUtils.writeLatDataByDecade(latData, allFilenames, variableNames, baseCal.getTime());

                 LOG.warn(" Writing finished.");

             }

         }

     }

     public String getMemDisplay() 

     {

         Runtime rt = Runtime.getRuntime();

         NumberFormat numFmt = NumberFormat.getNumberInstance();

         numFmt.setMaximumFractionDigits(0);

         StringBuffer sb = new StringBuffer();

         sb.append(numFmt.format(rt.totalMemory() / 1024.0));

         sb.append(" Kb total, ");

         sb.append(numFmt.format(rt.freeMemory() / 1024.0));

         sb.append(" Kb free, ");

         sb.append(numFmt.format(rt.maxMemory() / 1024.0));

         sb.append(" Kb max.");

         return sb.toString();

     }

     /**

      * Defines a netCDF file for the rainFall variable, but does not fill the variables

      * 

      * @param outputFileName

      * @throws Exception

      */

     private void createVariableFile(String outputFileName, List<String> dimensionOrdering, int variableIndex, int blockTimeSize, int blockLatSize, int blockLongSize)

             throws Exception

     {

         this.defineDimensions(outputFileName, dimensionOrdering, blockTimeSize, blockLatSize, blockLongSize);

         this.defineCoordinateVariables(outputFileName);

         this.defineSpatialVariables(outputFileName);

         // Add attributes

         DateFormat utcDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");

         Attribute converter = new Attribute("History", utcDateTime.format(new Date()) + " Converted to netCDF by "

                 + "$Id: MdbsyScenarioCConverter.java 84 2010-08-25 05:56:46Z robertbridle $"); 

         Attribute ipccClimateModel = new Attribute("IPCC Climate Model Name", MODEL_NAMES[IPCC_MODEL_NAME]);

         List<Attribute> globalAttrs = new ArrayList<Attribute>();

         globalAttrs.add(converter);

         globalAttrs.add(ipccClimateModel);

         if (METADATA_FILE.length() > 0)

         {

             globalAttrs.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(METADATA_FILE)));

         }

         NcDefineAttributes ncDefineAttr = new NcDefineAttributes();

         ncDefineAttr.execute(outputFileName, globalAttrs, false /* isLargeFileSupport */);

         String dimensions = "";

         for (String dimension : dimensionOrdering)

         {

             dimensions += dimension + " ";

         }

         this.defineVariable(outputFileName, dimensions, variableIndex);

     }

     /**

      * Define the dimensions for a WRON netCDF file, e.g. latitude, longitude, time.

      * 

      * @param outputFileName

      * @throws IllegalArgumentException

      * @throws ParseException

      * @throws IOException

      */

     private void defineDimensions(String outputFileName, List<String> dimensionOrdering, int blockTimeSize, int blockLatSize, int blockLongSize)

             throws IllegalArgumentException, ParseException, IOException

     {

         NcDefineDimension command = new NcDefineDimension();

         for (String dimName : dimensionOrdering)

         {

             if (dimName.equals(TIME))

             {

                 command.execute(outputFileName, TIME, blockTimeSize, false /* isUnlimited */, false/* fillValue */);

             }

             else if (dimName.equals(LAT))

             {

                 command.execute(outputFileName, LAT, blockLatSize, false /* isUnlimited */, false/* fillValue */);

             }

             else if (dimName.equals(LONG))

             {

                 command.execute(outputFileName, LONG, blockLongSize, false /* isUnlimited */, false/* fillValue */);

             }

         }

         // Add NV as the final dimension

         command.execute(outputFileName, NV, NV_SIZE, false /* isUnlimited */, false/* fillValue */);

     }

     /**

      * Defines the coordinate variables for a WRON netCDF file, e.g. latitude, longitude, time.

      * 

      * @param outputFileName

      * @throws IllegalArgumentException

      * @throws ParseException

      * @throws IOException

      */

     private void defineCoordinateVariables(String outputFileName) throws IllegalArgumentException, ParseException,

             IOException

     {

         NcDefineVariable command = new NcDefineVariable();

         command.execute(new File(outputFileName), LAT, DataType.FLOAT, Arrays.asList(new Attribute("units", LAT_UNITS),

                 new Attribute("long_name", LAT_LONG_NAME), new Attribute("standard_name", LAT_STANDARD_NAME), new Attribute("axis", LAT_AXIS)), LAT,

                 false /* isLargeFileSupport */, false/* fillValue */);

         command.execute(new File(outputFileName), LONG, DataType.FLOAT, Arrays.asList(

                 new Attribute("long_name", LONG_LONG_NAME), new Attribute("units", LONG_UNITS), new Attribute("standard_name", LONG_STANDARD_NAME), new Attribute(

                         "axis", LONG_AXIS)), LONG, false /* isLargeFileSupport */, false/* fillValue */);

         command.execute(new File(outputFileName), TIME, DataType.INT, Arrays.asList(new Attribute("units", TIME_UNITS),

                 new Attribute("long_name", TIME_LONG_NAME), new Attribute("standard_name", TIME_STANDARD_NAME),

                 new Attribute("axis", TIME_AXIS), new Attribute("calendar", TIME_CALENDAR), new Attribute("bounds",

                         TIME_BOUNDS)), TIME, false /* isLargeFileSupport */, false/* fillValue */);

         command.execute(new File(outputFileName), TIME_BOUNDS, DataType.INT, new ArrayList<Attribute>(), TIME + " "

                 + NV, false /* isLargeFileSupport */, false/* fillValue */);

         command.execute(new File(outputFileName), GRID_MAPPING, DataType.INT, Arrays.asList(new Attribute(

                 "grid_mapping_name", CRS_GRID_MAPPING_NAME), new Attribute("longitude_of_prime_meridian",

                 CRS_LONGITUDE_OF_PRIME_MERIDIAN), new Attribute("semi_major_axis", CRS_SEMI_MAJOR_AXIS),

                 new Attribute("inverse_flattening", CRS_INVERSE_FLATTENING)), "",

                 false /* isLargeFileSupport */, true/* fillValue */);

     }

     /**

      * Defines the spatial metadata variables for a WRON netCDF file, e.g. elevation, catchment id etc.

      * 

      * @param outputFileName

      *            The file to written to

      * @throws IOException

      *             If the definitions cannot be written

      */

     private void defineSpatialVariables(String outputFileName) throws IOException

     {

         NcDefineVariable command = new NcDefineVariable();

         String spatialDim = LAT + " " + LONG;

         command.execute(new File(outputFileName), ELEVATION, DataType.FLOAT, Arrays.asList(new Attribute("units",

                 ELEVATION_UNITS), new Attribute("standard_name", ELEVATION_STANDARD_NAME), new Attribute("long_name",

                 ELEVATION_LONG_NAME), new Attribute("_FillValue", ELEVATION_MISSING_VALUE), new Attribute(

                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);

         command

                 .execute(new File(outputFileName), CATCHMENT_ID, DataType.FLOAT, Arrays.asList(new Attribute(

                         "long_name", CATCHMENT_ID_LONG_NAME), new Attribute("_FillValue", CATCHMENT_ID_MISSING_VALUE), new Attribute(

                         "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);

         command

                 .execute(new File(outputFileName), REPORTING_REGION, DataType.FLOAT, Arrays.asList(new Attribute(

                         "long_name", REPORTING_REGION_LONG_NAME), new Attribute("_FillValue", REPORTING_REGION_MISSING_VALUE), new Attribute(

                                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */,

                         false/* fillValue */);

     }

     /**

      * Defines the rainFall variable

      * 

      * @param outputFileName

      * @throws IllegalArgumentException

      * @throws IOException

      * @throws ParseException

      */

     private void defineVariable(String outputFileName, String dimensions, int variableIndex)

             throws IllegalArgumentException, IOException, ParseException

     {

         NcDefineVariable command = new NcDefineVariable();

         command.execute(new File(outputFileName), variableNames[variableIndex], variableDataTypes[variableIndex],

                 Arrays.asList(new Attribute("units", variableUnits[variableIndex]), new Attribute("long_name",

                         variableLongNames[variableIndex]), new Attribute("missing_value", Float

                         .valueOf(variableMissingValues[variableIndex])), new Attribute("_FillValue", Float

                         .valueOf(variableFillValues[variableIndex])),  new Attribute("grid_mapping", 

                         GRID_MAPPING)), dimensions, false /* isLargeFileSupport */,

                 false/* fillValue */);

     }

     /**

      * Fills the coordinate variables, e.g. latitude, longitude, time.

      * 

      * @param outputFileName

      * @param sortedLongitudes

      * @param sortedLatitudes

      * @param dates

      * @param timeRange 

      * @param timeSize 

      * @param blockLongRange 

      * @param blockLongSize 

      * @param blockLatRange 

      * @param blockLatSize 

      * @throws UnsupportedEncodingException

      * @throws IOException

      * @throws java.text.ParseException

      */

     private void fillCoordinateVariables(String outputFileName, Set<String> sortedLongitudes,

             Set<String> sortedLatitudes, List<String> dates, String blockLatRange, String blockLongRange, String timeRange) throws UnsupportedEncodingException, IOException,

             java.text.ParseException

     {

         NcWriteVariable command = new NcWriteVariable();

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill longitude variable

         // create String of latitudes to populate the variable

         StringBuffer longitudeStringBuffer = new StringBuffer();

         for (Iterator<String> iter = sortedLongitudes.iterator(); iter.hasNext();)

         {

             longitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));

         }

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), LONG, blockLongRange, new ByteArrayInputStream(longitudeStringBuffer

                 .toString().getBytes(ENCODING)), false);

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill latitude variable

         // create String of latitudes to populate the variable

         StringBuffer latitudeStringBuffer = new StringBuffer();

         for (Iterator<String> iter = sortedLatitudes.iterator(); iter.hasNext();)

         {

             latitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));

         }

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), LAT, blockLatRange, new ByteArrayInputStream(latitudeStringBuffer

                 .toString().getBytes(ENCODING)), false);

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill time and time_bnds variables

         StringBuffer timeBoundsStringBuffer = new StringBuffer();

         StringBuffer dateStringBuffer = new StringBuffer();

         boolean first = true;

         for (String date : dates)

         {

             dateStringBuffer.append(date).append(System.getProperty("line.separator"));

             if (!first)

             {

                 timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));

             }

             timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));

             first = false;

         }

         int dayAfterLastDay = Integer.parseInt(dates.get(dates.size()-1)) +1;

         String finalString = String.valueOf(dayAfterLastDay);

         timeBoundsStringBuffer.append(finalString).append(System.getProperty("line.separator"));

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), TIME, timeRange, new ByteArrayInputStream(dateStringBuffer

                 .toString().getBytes(ENCODING)), false);

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), TIME_BOUNDS, timeRange+","+NV_RANGE, new ByteArrayInputStream(

                 timeBoundsStringBuffer.toString().getBytes(ENCODING)), false);

     }

     /**

      * Populates the spatial variables with data

      * 

      * @param outputFileName

      *            The netCDF file to be updated

      * @param sortedCellData

      *            The list of cell data in the same order as the spatial coordinates

      * @throws IOException

      *             If the data cannot be read

      */

     private void fillSpatialVariables(String outputFileName, List<CellData> sortedCellData) throws IOException

     {

         NcWriteVariable command = new NcWriteVariable();

         Set<String> latitudes = getSortedLatitudes(sortedCellData);

         Set<String> longitudes = getSortedLongitudes(sortedCellData);

         String spatialRange = "0-" + (latitudes.size() - 1) + ",0-" + (longitudes.size() - 1);

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill elevation variable

         // create String of elevations to populate the variable

         StringBuffer dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes,

                 CellDataVariable.ELEVATION, "-999.99");

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), ELEVATION, spatialRange, new ByteArrayInputStream(dataBuffer

                 .toString().getBytes(ENCODING)), false);

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill catchment id variable

         // create String of catchment ids to populate the variable

         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.CATCHMENTID, "0");

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), CATCHMENT_ID, spatialRange, new ByteArrayInputStream(dataBuffer

                 .toString().getBytes(ENCODING)), false);

         // ///////////////////////////////////////////////////////////////////////////////////////////

         // fill reporting region id variable

         // create String of reporting region ids to populate the variable

         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.REPORTINGREGIONID,

                 "0");

         // call command to actually perform fill the file

         command.execute(new File(outputFileName), REPORTING_REGION, spatialRange, new ByteArrayInputStream(dataBuffer

                 .toString().getBytes(ENCODING)), false);

     }

     /**

      * @param sortedCellData

      * @param latitudes

      * @param longitudes

      * @param varNum

      * @param fillValue

      * @return

      */

     private StringBuffer fillSpatialDataBuffer(List<CellData> sortedCellData, Set<String> latitudes,

             Set<String> longitudes, CellDataVariable varNum, String fillValue)

     {

         StringBuffer dataBuffer = new StringBuffer();

         int cellNum = 0;

         for (String latVal : latitudes)

         {

             for (String longVal : longitudes)

             {

                 if (cellNum < sortedCellData.size())

                 {

                     CellData cellData = sortedCellData.get(cellNum);

                     if (cellData.latitude.equals(latVal) && cellData.longitude.equals(longVal))

                     {

                         switch (varNum)

                         {

                         case ELEVATION:

                             dataBuffer.append(cellData.elevation);

                             break;

                         case CATCHMENTID:

                             dataBuffer.append(cellData.catchmentId);

                             break;

                         case REPORTINGREGIONID:

                             dataBuffer.append(cellData.reportingRegionId);

                             break;

                         }

                         cellNum++;

                     }

                     else

                     {

                         dataBuffer.append(fillValue);

                     }

                 }

                 else

                 {

                     dataBuffer.append(fillValue);

                 }

                 dataBuffer.append(System.getProperty("line.separator"));

             }

         }

         return dataBuffer;

     }

     /**

      * Populate the data variable with fill values. Used to ensure cells within 

      * the rectangular bounds but for which we have no data are set to the fill value. 

      *   

      * @param varIndex The variable being filled

      * @param sortedLongitudes The longitudes to fill.

      * @param sortedLatitudes The latitudes to fill

      * @param dates The dates to fill

      * @param dimensionOrdering The order of the dimensions in the files

      * @throws IOException If the data cannot be written

      */

     private void fillDataForLatitude(String filename, int varIndex, Set<String> sortedLongitudes, Set<String> sortedLatitudes,

             List<String> dates, List<String> dimensionOrdering) throws IOException

     {

         StringBuffer fillBlock = new StringBuffer();

         String fillLine = variableFillValues[varIndex] + "\n";

         for (int i = 0; i < dates.size(); i++)

         {

             fillBlock.append(fillLine);

         }

         NcWriteVariable writeVarCmd = new NcWriteVariable();

         for (String latitude : sortedLatitudes)

         {

             for (String longitude : sortedLongitudes)

             {

                 String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, dates.size());

                 writeVarCmd.execute(new File(filename), variableNames[varIndex], fillRange,

                         new ByteArrayInputStream(fillBlock.toString().getBytes(ENCODING)), false);

             }

         }

     }

     /**

      * Fills the variables with data for a particular latitude.

      * 

      * @param variableFileNames The filenames for this latitude for each of the variables

      * @param triples The details of the cells that occur in this latitude 

      * @param dimensionOrdering The order of the dimensions in the files

      * @throws IOException If the data cannot be written

      */

     private void fillDataByLatitudeVariables(String[] variableFileNames, List<CellData> triples, List<String> dimensionOrdering)

             throws IOException

     {

         NcWriteVariable command = new NcWriteVariable();

         int counter = 0;

         for (CellData triple : triples)

         {

             if (++counter % 1000 == 0)

             {

                 LOG.warn("   Num files processed: " + counter);

             }

             String csvFileName = triple.getCellId();

             String latitude = triple.getLatitude();

             String longitude = triple.getLongitude();

             String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, TIME_SIZE);

             try

             {

                 StringBuffer[] valuesStringBuffer = new StringBuffer[numVariables];

                 for (int i = 0; i < valuesStringBuffer.length; i++)

                 {

                     valuesStringBuffer[i] = new StringBuffer();

                 }

                 Scanner s = null;

                 try

                 {

                     s = new Scanner(new BufferedReader(new FileReader(INPUT_CSV_DIRECTORY + csvFileName + ".csv")))

                             .useDelimiter(Pattern.compile("[\n,]"));

                     // remove header line

                     for (int i = 0; i < NUM_CSV_COLUMNS; i++)

                     {

                         s.next();

                     }

                     while (s.hasNext())

                     {

                         s.next(); // date column

                         for (int i = 0; i < numVariables; i++)

                         {

                             valuesStringBuffer[i].append(s.next());

                             // Last variable will already have newline at its end.

                             if (i < numVariables - 1)

                             {

                                 valuesStringBuffer[i].append(System.getProperty("line.separator"));

                             }

                         }

                     }

                 }

                 finally

                 {

                     if (s != null)

                     {

                         s.close();

                     }

                 }

                 try

                 {

                     for (int i = 0; i < numVariables; i++)

                     {

                         command.execute(new File(variableFileNames[i]), variableNames[i], fillRange,

                                 new ByteArrayInputStream(valuesStringBuffer[i].toString().getBytes(ENCODING)), false);

                     }

                 }

                 catch (IllegalArgumentException iae)

                 {

                     LOG.error(iae);

                     LOG.error("fillRange (date, lat, long): " + fillRange);

                 }

             }

             catch (FileNotFoundException fnfe)

             {

                 LOG.info(fnfe.getMessage());

                 //System.out.println(fnfe.getMessage());

             }

         }

     }

     /**

      * @param dimensionOrdering

      * @param latitude

      * @param longitude

      * @param dates

      * @return

      */

     private String constructFillRange(List<String> dimensionOrdering, String latitude, String longitude, int dateSize)

     {

         String time_range = "0-" + String.valueOf(dateSize - 1);

         String latitude_range = "lookup(" + latitude + ")-lookup(" + latitude + ")";

         String longitude_range = "lookup(" + longitude + ")-lookup(" + longitude + ")";

         String fillRange = new String();

         for (Iterator<String> iter = dimensionOrdering.iterator(); iter.hasNext();)

         {

             String dimension = iter.next();

             if (dimension.equals(TIME))

             {

                 fillRange += time_range;

             }

             else if (dimension.equals(LAT))

             {

                 fillRange += latitude_range;

             }

             else if (dimension.equals(LONG))

             {

                 fillRange += longitude_range;

             }

             if (iter.hasNext())

                 fillRange += ",";

         }

         return fillRange;

     }

     /**

      * Groups triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file) by degrees of latitude.

      * 

      * @param locationLookupValues

      * @return

      */

     private Map<String, List<CellData>> getTriplesGroupedByLatitude(String[][] locationLookupValues)

     {

         Map<String, List<CellData>> triplesGroupedByLatitude = new HashMap<String, List<CellData>>();

         for (int i = 0; i < locationLookupValues.length; i++)

         {

             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];

             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];

             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];

             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];

             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];

             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];

             String latitudeDegree = String.valueOf(Float.valueOf(latitude).intValue());

             List<CellData> triplesInLatitude = triplesGroupedByLatitude.get(latitudeDegree);

             if (triplesInLatitude == null)

             {

                 triplesInLatitude = new ArrayList<CellData>();

                 triplesGroupedByLatitude.put(latitudeDegree, triplesInLatitude);

             }

             final CellData cellData = new CellData(cellId, longitude, latitude);

             cellData.elevation = elevation;

             cellData.catchmentId = catchmentId;

             cellData.reportingRegionId = reportingRegionId;

             triplesInLatitude.add(cellData);

         }

         // check that we have actually grouped all rows by latitude

         int count = 0;

         for (Iterator<String> iter = triplesGroupedByLatitude.keySet().iterator(); iter.hasNext();)

         {

             count += triplesGroupedByLatitude.get(iter.next()).size();

         }

         if (count != locationLookupValues.length)

         {

             LOG.error("latitudes where not grouped by latitude correctly");

             System.exit(-1);

         }

         return triplesGroupedByLatitude;

     }

     /**

      * Groups dates, e.g. 1895-01-01 by decade.

      * 

      * @param dates

      * @return

      */

     public Map<Integer, List<String>> getDatesByDecade(List<String> dates)

     {

         Map<Integer, List<String>> datesGroupedByDecade = new HashMap<Integer, List<String>>();

         for (String date : dates)

         {

             int year = Integer.valueOf(date.substring(0, 4));

             int decade = year / 10;

             List<String> datesInDecade = datesGroupedByDecade.get(decade);

             if (datesInDecade == null)

             {

                 datesInDecade = new ArrayList<String>();

                 datesGroupedByDecade.put(decade, datesInDecade);

             }

             datesInDecade.add(date);

         }

         // check that we have actually grouped all rows by latitude

         int count = 0;

         for (Iterator<Integer> iter = datesGroupedByDecade.keySet().iterator(); iter.hasNext();)

         {

             count += datesGroupedByDecade.get(iter.next()).size();

         }

         if (count != dates.size())

         {

             LOG.error("dates where not grouped decade correctly");

             System.exit(-1);

         }

         return datesGroupedByDecade;

     }

     /**

      * Loads all triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file).

      * 

      * @param locationLookupValues

      * @return

      */

     private List<CellData> getAllTriples(String[][] locationLookupValues)

     {

         List<CellData> allTriples = new ArrayList<CellData>();

         for (int i = 0; i < locationLookupValues.length; i++)

         {

             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];

             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];

             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];

             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];

             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];

             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];

             CellData cellData = new CellData(cellId, longitude, latitude);

             cellData.elevation = elevation;

             cellData.catchmentId = catchmentId;

             cellData.reportingRegionId = reportingRegionId;

             allTriples.add(cellData);

         }

         // check that we have actually grouped all rows by latitude

         if (allTriples.size() != locationLookupValues.length)

         {

             LOG.error("lookup values where not read correctly");

             System.exit(-1);

         }

         return allTriples;

     }

     /**

      * Parses the WRON lookup file into an row-by-column string matrix.

      * 

      * @throws IOException

      */

     private String[][] readLookupFile() throws IOException

     {

         RandomAccessFile raf = new RandomAccessFile(new File(LOCATION_LOOKUP_FILE), "r");

         raf.readLine();// abandon the first line since it is a header line.

         List<String[]> lineList = new ArrayList<String[]>();

         String line = raf.readLine();

         while (line != null)

         {

             String[] cols = new CSVTokenizer(line).getAllColumns();

             lineList.add(cols);

             line = raf.readLine();

         }

         raf.close();

         String[][] lineCols = lineList.toArray(new String[][]{});

         return lineCols;

     }

     /**

      * Transpose a matrix, e.g. [a][b] -> [b][a]

      * 

      * @param gridValues

      * @return

      */

     private String[][] transpose(String[][] gridValues)

     {

         String[][] transposed = new String[gridValues[0].length][gridValues.length];

         for (int rows = 0; rows < gridValues.length; rows++)

         {

             for (int cols = 0; cols < gridValues[0].length; cols++)

             {

                 transposed[cols][rows] = gridValues[rows][cols];

             }

         }

         return transposed;

     }

     /**

      * Retrieve an ordered set containing all the latitude values from a list of triples.

      * 

      * @param triples

      * @return

      */

     private Set<String> getSortedLatitudes(List<CellData> triples)

     {

         Set<String> latitudes = new TreeSet<String>();

         for (CellData triple : triples)

         {