View Javadoc

1   /**
2    * Copyright 2010, CSIRO Australia.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  package au.csiro.netcdf.wron;
18  
19  import java.io.BufferedReader;
20  import java.io.ByteArrayInputStream;
21  import java.io.File;
22  import java.io.FileInputStream;
23  import java.io.FileNotFoundException;
24  import java.io.FileReader;
25  import java.io.IOException;
26  import java.io.PrintWriter;
27  import java.io.RandomAccessFile;
28  import java.io.StringWriter;
29  import java.io.UnsupportedEncodingException;
30  import java.text.DateFormat;
31  import java.text.NumberFormat;
32  import java.text.SimpleDateFormat;
33  import java.util.ArrayList;
34  import java.util.Arrays;
35  import java.util.Calendar;
36  import java.util.Collections;
37  import java.util.Comparator;
38  import java.util.Date;
39  import java.util.HashMap;
40  import java.util.Iterator;
41  import java.util.List;
42  import java.util.Map;
43  import java.util.Scanner;
44  import java.util.Set;
45  import java.util.TreeSet;
46  import java.util.regex.Pattern;
47  
48  import org.apache.commons.cli.BasicParser;
49  import org.apache.commons.cli.CommandLine;
50  import org.apache.commons.cli.HelpFormatter;
51  import org.apache.commons.cli.MissingOptionException;
52  import org.apache.commons.cli.Option;
53  import org.apache.commons.cli.OptionBuilder;
54  import org.apache.commons.cli.Options;
55  import org.apache.commons.cli.ParseException;
56  import org.apache.log4j.Logger;
57  
58  import ucar.ma2.DataType;
59  import ucar.nc2.Attribute;
60  import ucar.nc2.units.DateFormatter;
61  import au.csiro.netcdf.NcDefineAttributes;
62  import au.csiro.netcdf.NcDefineDimension;
63  import au.csiro.netcdf.NcDefineVariable;
64  import au.csiro.netcdf.NcWriteVariable;
65  import au.csiro.netcdf.cli.CommandLineOptionsComparator;
66  import au.csiro.netcdf.util.CSVTokenizer;
67  import au.csiro.netcdf.util.NetCDFUtils;
68  
69  /**
70   * The class is a converter control class to convert Murray Darling Basin 
71   * Sustainable Yields CSV files into netCDF files. This class will process 
72   * the Scenario C historical dataset files.  
73   * 
74   * Copyright 2010, CSIRO Australia 
75   * All rights reserved.
76   * 
77   * @author Robert Bridle on 17/04/2010
78   * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $
79   */
80  public class MdbsyScenarioCConverter
81  {
82      // netCDF file naming components
83      private static String COLLECTION = "Collection-MDBSY_Climate";
84      private static String SCENARIO = "";
85      private static String[] MODEL_NAMES;
86      private static String CASE = "";
87  
88      /**
89       * The file containing the location lookup values.
90       */
91      private static String LOCATION_LOOKUP_FILE = "";
92  
93      /**
94       * The file containing the metadata attributes (if any).
95       */
96      private static String METADATA_FILE = "";
97      
98      /**
99       * The directory containing the Scenario C csv data files.
100      */
101     private static String INPUT_CSV_DIRECTORY = "";
102 
103     /**
104      * The netCDF file to be created.
105      */
106     private static String OUTPUT_NETCDF_DIRECTORY = "";
107 
108     /**
109      * NetCDF file extension.
110      */
111     private static final String NETCDF_FILE_EXTENSION = ".nc";
112 
113     /**
114      * Encoding used to fill variables.
115      */
116     private static final String ENCODING = "UTF-8";
117     
118     /**
119      * The 15 climate models used in the MDBSY project.
120      */
121     private static Map<String, String> CLIMATE_MODELS = new HashMap<String, String>();
122     static
123     {
124         CLIMATE_MODELS.put("cccma_t47", "CCCMA T47");
125         CLIMATE_MODELS.put("cccma_t63", "CCCMA T63");
126         CLIMATE_MODELS.put("cnrm", "CNRM");
127         CLIMATE_MODELS.put("csiro", "CSIRO-MK3.0");
128         CLIMATE_MODELS.put("gfdl", "GFDL 2.0");
129         CLIMATE_MODELS.put("giss_aom", "GISS-AOM");
130         CLIMATE_MODELS.put("iap", "IAP");
131         CLIMATE_MODELS.put("inmcm", "INMCM");
132         CLIMATE_MODELS.put("ipsl", "IPSL");
133         CLIMATE_MODELS.put("miroc", "MIROC-M");
134         CLIMATE_MODELS.put("miub", "MIUB");
135         CLIMATE_MODELS.put("mpi", "MPI-ECHAM5");
136         CLIMATE_MODELS.put("mri", "MRI");
137         CLIMATE_MODELS.put("ncar_ccsm", "NCAR-CCSM");
138         CLIMATE_MODELS.put("ncar_pcm", "NCAR-PCM1");
139     }
140     /**
141      * Index of model names in {@link MdbsyScenarioCConverter#MODEL_NAMES}.
142      */
143     private static final int FILE_NAME_MODEL_NAME = 0;
144     private static final int IPCC_MODEL_NAME = 1;
145 
146     // Variables
147     private static final String[] variableNames = new String[] { "rainfall", "APET" };
148     private static final String[] variableUnits = new String[] { "mm", "mm" };
149     private static final String[] variableLongNames = new String[] { "rainfall", "APET" };
150     private static final String[] variableFillValues = new String[] { "-9999.0f", "-9999.0f" };
151     private static final String[] variableMissingValues = variableFillValues;
152     private static final DataType[] variableDataTypes = new DataType[] { DataType.FLOAT, DataType.FLOAT };
153     static final int numVariables = variableNames.length;
154     private static final String GRID_MAPPING = "crs";
155 
156     // time constants
157     private static final String TIME = "time";
158     private static String TIME_RANGE = "0-40906";
159     private static final int TIME_SIZE = 40907;
160     private static final String TIME_LONG_NAME = "reference time";
161     private static final String TIME_STANDARD_NAME = "time";
162     private static final String TIME_UNITS = "days since 1895-01-01 0:0:0";
163     private static final String TIME_CALENDAR = "Gregorian";
164     private static final String TIME_AXIS = "T";
165     private static final String TIME_BOUNDS = "time_bnds";
166 
167     // longitude constants
168     private static final String LONG = "longitude";
169     private static final int LONG_COLUMN_INDEX = 1;
170     private static final String LONG_RANGE = "0-296";
171     private static final int LONG_SIZE = 297;
172     private static final String LONG_UNITS = "degrees_east";
173     private static final String LONG_STANDARD_NAME = "longitude";
174     private static final String LONG_AXIS = "X";
175     private static final String LONG_LONG_NAME = "longitude";
176 
177     // latitude constants
178     private static final String LAT = "latitude";
179     private static final int LAT_COLUMN_INDEX = 2;
180     private static final String LAT_RANGE = "0-278";
181     private static final int LAT_SIZE = 279;
182     private static final String LAT_UNITS = "degrees_north";
183     private static final String LAT_STANDARD_NAME = "latitude";
184     private static final String LAT_AXIS = "Y";
185     private static final String LAT_LONG_NAME = "latitude";
186 
187     // num values constants
188     private static final String NV = "nv";
189     private static final String NV_RANGE = "0-1";
190     private static final int NV_SIZE = 2;
191 
192     // elevation constants
193     private static String ELEVATION = "elev";
194     private static int ELEVATION_COLUMN_INDEX = 3;
195     private static final String ELEVATION_UNITS = "m";
196     private static final String ELEVATION_STANDARD_NAME = "altitude";
197     private static final String ELEVATION_LONG_NAME = "Elevation above sea level";
198     private static final float ELEVATION_MISSING_VALUE = -999.99f;
199 
200     // catchment id constants
201     private static String CATCHMENT_ID = "catchmentId";
202     private static int CATCHMENT_ID_COLUMN_INDEX = 4;
203     private static final String CATCHMENT_ID_LONG_NAME = "MDB Catchment Id";
204     private static final float CATCHMENT_ID_MISSING_VALUE = 0f;
205 
206     // reporting region id constants
207     private static String REPORTING_REGION = "repRegionId";
208     private static int REPORTING_REGION_ID_COLUMN_INDEX = 5;
209     private static String REPORTING_REGION_LONG_NAME = "MDB Reporting Region Id";
210     private static final float REPORTING_REGION_MISSING_VALUE = 0f;
211 
212     // crs constants
213     private static String CRS_GRID_MAPPING_NAME = "latitude_longitude";  
214     private static float CRS_LONGITUDE_OF_PRIME_MERIDIAN = 0.0f;
215     private static float CRS_SEMI_MAJOR_AXIS = 6378137.0f;
216     private static float CRS_INVERSE_FLATTENING = 298.257222101f;
217     
218     // cell id
219     private static final int CELL_ID_COLUMN_INDEX = 0;
220 
221     // Other cell data
222 
223     // date conversion constants
224     private static final String EPOC_DATE_STRING = "1895-01-01";
225     private static final DateFormatter isoDateTimeFormat = new DateFormatter();
226     private static final int DATE_COLUMN_INDEX = 0;
227 
228     // number of time values to read from all the csv files and load into memory.
229     private static int ROW_CHUNK_SIZE = 1000;
230 
231     /** The number of columns expected in the CSV data files */
232     private static final int NUM_CSV_COLUMNS = 1 + numVariables;
233 
234     /**
235      * Constant that defines the logger to be used.
236      */
237     private static final Logger LOG = Logger.getLogger(MdbsyScenarioCConverter.class.getName());
238 
239     /** Conversion utilities  */
240     private ConversionUtils convUtils = new ConversionUtils();
241 
242     /**
243      * @param args
244      */
245     @SuppressWarnings("static-access")
246     public static void main(String[] args) throws Exception
247     {
248         Options options = new Options();
249         try
250         {
251             Option inputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(
252                     " 1: the directory containing the csv files for the scenario.").isRequired(true).withLongOpt(
253                     "inputDirectory").create("i");
254 
255             Option lookupFile = OptionBuilder.withArgName("file").hasArg()
256                     .withDescription(" 2: the cellId lookup file.").isRequired(true).withLongOpt("lookupFile").create(
257                             "f");
258 
259             Option outputDirectory = OptionBuilder.withArgName("dir").hasArg().withDescription(
260                     " 3: the output directory.").isRequired(true).withLongOpt("outputDirectory").create("o");
261 
262             Option scenario = OptionBuilder.withArgName("text").hasArg().withDescription(
263                     " 4: the modelling scenario used to name the netCDF file.").isRequired(true).withLongOpt("scenario")
264                     .create("s");
265             
266             Option model = OptionBuilder.withArgName("text").hasArg().withDescription(
267                     " 5: the global climate change model used to name the netCDF file.").isRequired(true).withLongOpt(
268                     "model").create("e");
269 
270             Option modelCase = OptionBuilder.withArgName("text").hasArg().withDescription(
271                     " 6: the model case or sensitivity used to name the netCDF file.").isRequired(true).withLongOpt(
272                     "case").create("c");
273 
274             Option metadataFile = OptionBuilder.withArgName("file").hasArg().withDescription(
275                     " 7: a file of global metadata attributes to be added to the netCDF file.").isRequired(true)
276                     .withLongOpt("metadataFile").create("m");
277 
278             Option numRows = OptionBuilder.withArgName("int").hasArg().withDescription(
279                     " 8: the number of rows to read at a time.").isRequired(true).withLongOpt("rows").create("r");
280 
281             Option byDecade = OptionBuilder.withDescription(
282                     " 9: OPTIONAL, set if we want to convert into netCDF files by decade.").isRequired(false)
283                     .withLongOpt("decade").create("d");
284 
285             Option byLatitude = OptionBuilder.withDescription(
286                     " 10: OPTIONAL, set if we want to convert into netCDF files by degrees of latitude.").isRequired(
287                     false).withLongOpt("latitude").create("l");
288 
289             Option startLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(
290                     " 11: OPTIONAL, The latitude at which to start processing. For byDecade, this option will "
291                             + "assume already existing netCDF files.").isRequired(false).create("startLat");
292 
293             Option endLat = OptionBuilder.withArgName("latitude").hasArg().withDescription(
294                     " 12: OPTIONAL, The latitude at which to end processing (inclusive).").isRequired(false).create(
295                     "endLat");
296 
297             options.addOption(inputDirectory);
298             options.addOption(lookupFile);
299             options.addOption(outputDirectory);
300             options.addOption(scenario);
301             options.addOption(model);
302             options.addOption(modelCase);
303             options.addOption(metadataFile);
304             options.addOption(numRows);
305             options.addOption(byDecade);
306             options.addOption(byLatitude);
307             options.addOption(startLat);
308             options.addOption(endLat);
309 
310             // parse the command line arguments
311             CommandLine parsedCommandLine = new BasicParser().parse(options, args);
312 
313             INPUT_CSV_DIRECTORY = (parsedCommandLine.hasOption("inputDirectory")) ? parsedCommandLine
314                     .getOptionValue("inputDirectory") : "";
315             LOCATION_LOOKUP_FILE = (parsedCommandLine.hasOption("lookupFile")) ? parsedCommandLine
316                     .getOptionValue("lookupFile") : "";
317             OUTPUT_NETCDF_DIRECTORY = (parsedCommandLine.hasOption("outputDirectory")) ? parsedCommandLine
318                     .getOptionValue("outputDirectory") : "";
319             SCENARIO = (parsedCommandLine.hasOption("scenario")) ? parsedCommandLine.getOptionValue("scenario") : "";
320             METADATA_FILE = (parsedCommandLine.hasOption("metadataFile")) ? parsedCommandLine
321                     .getOptionValue("metadataFile") : "";
322             MODEL_NAMES = getModelMapping((parsedCommandLine.hasOption("model")) ? parsedCommandLine.getOptionValue("model") : "");
323             CASE = (parsedCommandLine.hasOption("case")) ? parsedCommandLine.getOptionValue("case") : "";
324             ROW_CHUNK_SIZE = (parsedCommandLine.hasOption("rows")) ? Integer.valueOf(parsedCommandLine
325                     .getOptionValue("rows")) : ROW_CHUNK_SIZE;
326             String startingLatitude = (parsedCommandLine.hasOption("startLat")) ? parsedCommandLine
327                     .getOptionValue("startLat") : "";
328             String endingLatitude = (parsedCommandLine.hasOption("endLat")) ? parsedCommandLine
329                     .getOptionValue("endLat") : "";
330 
331             boolean doSplitByDecade = parsedCommandLine.hasOption("decade");
332             boolean doSplitByLatitude = parsedCommandLine.hasOption("latitude");
333 
334             MdbsyScenarioCConverter converter = new MdbsyScenarioCConverter();
335 
336             if (doSplitByDecade)
337             {
338                 long start = System.currentTimeMillis();
339                 converter.splitByDecade(startingLatitude, endingLatitude);
340                 long end = System.currentTimeMillis() - start;
341                 LOG.warn("Split by decade in: " + end + " ms.");
342             }
343 
344             if (doSplitByLatitude)
345             {
346                 long start = System.currentTimeMillis();
347                 converter.splitByLatitude(startingLatitude, endingLatitude);
348                 long end = System.currentTimeMillis() - start;
349                 LOG.warn("Split by latitude in: " + end + " ms.");
350             }
351         }
352         catch (MissingOptionException moe)
353         {
354             LOG.error(moe.getMessage());
355 
356             StringWriter sw = new StringWriter();
357             HelpFormatter formatter = new HelpFormatter();
358             formatter.setOptionComparator(new CommandLineOptionsComparator());
359             formatter.printHelp(new PrintWriter(sw), 80, "-", "header", options, 0, 1, "footer");
360             System.out.println(sw.toString());
361         }
362     }
363 
364     /**
365      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by degrees of latitude.
366      * 
367      * @throws Exception
368      */
369     public void splitByLatitude(String startingLatitude, String endingLatitude) throws Exception
370     {
371         // group each row in lookup file according to latitude
372         Map<String, List<CellData>> triplesGroupedByLatitude = this.getTriplesGroupedByLatitude(this.readLookupFile());
373 
374         // sort by latitude
375         Set<String> latitudeKeySet = triplesGroupedByLatitude.keySet();
376         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(latitudeKeySet, startingLatitude, endingLatitude);
377         List<String> latitudeKeyList = new ArrayList<String>(limitedLatitudes);
378         Collections.sort(latitudeKeyList, new Comparator<String>()
379         {
380             @Override
381             public int compare(String o1, String o2)
382             {
383                 return Integer.valueOf(o2).compareTo(Integer.valueOf(o1));
384             }
385         });
386 
387         // for each latitude create a netCDF file.
388         for (String latitudeKey : latitudeKeyList)
389         {
390             LOG.warn("Started work on latitude: " + latitudeKey + ".");
391             String latitudeDegree = latitudeKey;
392             List<CellData> triples = triplesGroupedByLatitude.get(latitudeDegree);
393 
394             // get the values to fill the coordinate variables.
395             Set<String> sortedLatitudes = this.getSortedLatitudes(triples);
396             int blockLatSize = sortedLatitudes.size();
397             String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);
398 
399             Set<String> sortedLongitudes = this.getSortedLongitudes(triples);
400             int blockLongSize = sortedLongitudes.size();
401             String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);
402 
403             // create file names
404             String filenames[] = new String[numVariables];
405             List<String> dates = this.getDatesAsDaysSinceEpoc(this.getDates());
406             for (int i = 0; i < filenames.length; i++)
407             {
408                 filenames[i] = this.generateLatitudeFilename(latitudeDegree, variableNames[i]);
409                 this.createVariableFile(filenames[i], Arrays.asList(LAT, LONG, TIME), i, TIME_SIZE, blockLatSize, blockLongSize);
410                 this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,
411                         blockLongRange, TIME_RANGE);
412                 this.fillSpatialVariables(filenames[i], triples);
413                 this.fillDataForLatitude(filenames[i], i, sortedLongitudes, sortedLatitudes, dates, Arrays.asList(LAT,
414                         LONG, TIME));
415             }
416 
417             // for each cellId csv file we read, we will have available the values for both rainFall and PVET, therefore
418             // it makes sense to fill these variables at the same time.
419             this.fillDataByLatitudeVariables(filenames, triples, Arrays.asList(LAT, LONG, TIME));
420         }
421     }
422 
423     /**
424      * Splits a directory of WRON csv files into corresponding netCDF files that are grouped by decade.
425      * 
426      * @return
427      * 
428      * @throws Exception
429      */
430     public void splitByDecade(String startingLatitude, String endingLatitude) throws Exception
431     {
432         // get all rows in lookup file
433         List<CellData> triples = this.getAllTriples(this.readLookupFile());
434 
435         // get the values to fill the coordinate variables.
436         Set<String> sortedLatitudes = this.getSortedLatitudes(triples);
437         String blockLatRange = "0-" + String.valueOf(sortedLatitudes.size() - 1);
438 
439         Set<String> sortedLongitudes = this.getSortedLongitudes(triples);
440         String blockLongRange = "0-" + String.valueOf(sortedLongitudes.size() - 1);
441         String startingLongitude = new ArrayList<String>(sortedLongitudes).get(0);
442 
443         // groups dates by decade
444         Map<Integer, List<String>> datesByDecade = this.getDatesByDecade(this.getDates());
445 
446         // sort by decade
447         Set<Integer> decadeKeySet = datesByDecade.keySet();
448         List<Integer> decadeKeyList = new ArrayList<Integer>(decadeKeySet);
449         Collections.sort(decadeKeyList);
450         Map<Integer, String[]> allFilenames = new HashMap<Integer, String[]>();
451         
452         // for each decade create a netCDF file.
453         int start = 0;
454         int end = start;
455         int filenameStart = 0;
456         for (Integer decadeKey : decadeKeyList)
457         {
458             LOG.warn("Started work on decade: " + decadeKey + "0s.");
459             System.gc();
460             LOG.warn("  Mem: " + getMemDisplay());
461             List<String> dates = datesByDecade.get(decadeKey);
462             end = end + dates.size();
463 
464             int blockTimeSize = dates.size();
465             String blockTimeRange = "0-" + String.valueOf(dates.size() - 1);
466 
467             // create file names
468             String decadeStr = String.valueOf(decadeKey + "0");
469 
470             String filenames[] = new String[numVariables];
471             dates = this.getDatesAsDaysSinceEpoc(dates);
472             for (int i = 0; i < filenames.length; i++)
473             {
474                 filenames[i] = this.generateDecadeFilename(decadeStr, variableNames[i]);
475                 if ("".equals(startingLatitude))
476                 {
477                     this.createVariableFile(filenames[i], Arrays.asList(TIME, LAT, LONG), i, blockTimeSize, LAT_SIZE, LONG_SIZE);
478                     this.fillCoordinateVariables(filenames[i], sortedLongitudes, sortedLatitudes, dates, blockLatRange,
479                             blockLongRange, blockTimeRange);
480                     this.fillSpatialVariables(filenames[i], triples);
481                 }
482             }
483             
484             allFilenames.put(decadeKey, filenames);
485             filenameStart += numVariables;
486         }
487 
488         LOG.warn(" Latitude limits : " + startingLatitude + " to " + endingLatitude + ".");
489         
490         int numBlocks = LAT_SIZE / ROW_CHUNK_SIZE + (LAT_SIZE % ROW_CHUNK_SIZE > 0 ? 1 : 0);
491         Calendar baseCal = convUtils.getCalendar(new Date());
492         baseCal.clear();
493         baseCal.set(1895, 0, 1);
494         
495         Set<String> limitedLatitudes = convUtils.getLimitedLatitudes(sortedLatitudes, startingLatitude, endingLatitude);
496         for (int blockNum = 0; blockNum < numBlocks; blockNum++)
497         {
498             LOG.warn(" Processing block : " + blockNum + ".");
499             LOG.warn("  Mem: " + getMemDisplay());
500             // Grab the next set of latitudes to be read
501             List<String> lats = convUtils.getLatitudeBlock(limitedLatitudes, blockNum, ROW_CHUNK_SIZE);
502             if (lats.isEmpty())
503             {
504                 LOG.warn(" Skipping block as it is outside the latitude range.");
505             }
506             else
507             {
508                 LOG.warn(" Processing latitudes : " + lats + ".");
509                 
510                 // Build a set of files covering those latitudes
511                 List<CellData> targetCells = convUtils.buildCsvFilenamesForLatitudes(lats, triples, INPUT_CSV_DIRECTORY, ".csv");
512                 
513                 // Read all data from the file set
514                 LOG.warn(" Reading data from CSVs...");
515                 Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(targetCells, lats,
516                         numVariables, TIME_SIZE, LONG_SIZE, variableFillValues, startingLongitude);
517                 
518                 // Write out the data in latitude blocks
519                 LOG.warn(" Writing data to netCDF...");
520                 LOG.warn("  Mem: " + getMemDisplay());
521                 convUtils.writeLatDataByDecade(latData, allFilenames, variableNames, baseCal.getTime());
522     
523                 LOG.warn(" Writing finished.");
524             }
525         }
526     }
527 
528     public String getMemDisplay() 
529     {
530         Runtime rt = Runtime.getRuntime();
531         NumberFormat numFmt = NumberFormat.getNumberInstance();
532         numFmt.setMaximumFractionDigits(0);
533         StringBuffer sb = new StringBuffer();
534         sb.append(numFmt.format(rt.totalMemory() / 1024.0));
535         sb.append(" Kb total, ");
536         sb.append(numFmt.format(rt.freeMemory() / 1024.0));
537         sb.append(" Kb free, ");
538         sb.append(numFmt.format(rt.maxMemory() / 1024.0));
539         sb.append(" Kb max.");
540         return sb.toString();
541     }
542     
543     /**
544      * Defines a netCDF file for the rainFall variable, but does not fill the variables
545      * 
546      * @param outputFileName
547      * @throws Exception
548      */
549     private void createVariableFile(String outputFileName, List<String> dimensionOrdering, int variableIndex, int blockTimeSize, int blockLatSize, int blockLongSize)
550             throws Exception
551     {
552         this.defineDimensions(outputFileName, dimensionOrdering, blockTimeSize, blockLatSize, blockLongSize);
553         this.defineCoordinateVariables(outputFileName);
554         this.defineSpatialVariables(outputFileName);
555 
556         // Add attributes
557         DateFormat utcDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
558         Attribute converter = new Attribute("History", utcDateTime.format(new Date()) + " Converted to netCDF by "
559                 + "$Id: MdbsyScenarioCConverter.java 84 2010-08-25 05:56:46Z robertbridle $"); 
560         Attribute ipccClimateModel = new Attribute("IPCC Climate Model Name", MODEL_NAMES[IPCC_MODEL_NAME]);
561         List<Attribute> globalAttrs = new ArrayList<Attribute>();
562         globalAttrs.add(converter);
563         globalAttrs.add(ipccClimateModel);
564         if (METADATA_FILE.length() > 0)
565         {
566             globalAttrs.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(METADATA_FILE)));
567         }
568         NcDefineAttributes ncDefineAttr = new NcDefineAttributes();
569         ncDefineAttr.execute(outputFileName, globalAttrs, false /* isLargeFileSupport */);
570 
571         String dimensions = "";
572         for (String dimension : dimensionOrdering)
573         {
574             dimensions += dimension + " ";
575         }
576         this.defineVariable(outputFileName, dimensions, variableIndex);
577     }
578 
579     /**
580      * Define the dimensions for a WRON netCDF file, e.g. latitude, longitude, time.
581      * 
582      * @param outputFileName
583      * @throws IllegalArgumentException
584      * @throws ParseException
585      * @throws IOException
586      */
587     private void defineDimensions(String outputFileName, List<String> dimensionOrdering, int blockTimeSize, int blockLatSize, int blockLongSize)
588             throws IllegalArgumentException, ParseException, IOException
589     {
590         NcDefineDimension command = new NcDefineDimension();
591 
592         for (String dimName : dimensionOrdering)
593         {
594             if (dimName.equals(TIME))
595             {
596                 command.execute(outputFileName, TIME, blockTimeSize, false /* isUnlimited */, false/* fillValue */);
597             }
598             else if (dimName.equals(LAT))
599             {
600                 command.execute(outputFileName, LAT, blockLatSize, false /* isUnlimited */, false/* fillValue */);
601             }
602             else if (dimName.equals(LONG))
603             {
604                 command.execute(outputFileName, LONG, blockLongSize, false /* isUnlimited */, false/* fillValue */);
605             }
606         }
607         
608         // Add NV as the final dimension
609         command.execute(outputFileName, NV, NV_SIZE, false /* isUnlimited */, false/* fillValue */);
610     }
611 
612     /**
613      * Defines the coordinate variables for a WRON netCDF file, e.g. latitude, longitude, time.
614      * 
615      * @param outputFileName
616      * @throws IllegalArgumentException
617      * @throws ParseException
618      * @throws IOException
619      */
620     private void defineCoordinateVariables(String outputFileName) throws IllegalArgumentException, ParseException,
621             IOException
622     {
623         NcDefineVariable command = new NcDefineVariable();
624 
625         command.execute(new File(outputFileName), LAT, DataType.FLOAT, Arrays.asList(new Attribute("units", LAT_UNITS),
626                 new Attribute("long_name", LAT_LONG_NAME), new Attribute("standard_name", LAT_STANDARD_NAME), new Attribute("axis", LAT_AXIS)), LAT,
627                 false /* isLargeFileSupport */, false/* fillValue */);
628         command.execute(new File(outputFileName), LONG, DataType.FLOAT, Arrays.asList(
629                 new Attribute("long_name", LONG_LONG_NAME), new Attribute("units", LONG_UNITS), new Attribute("standard_name", LONG_STANDARD_NAME), new Attribute(
630                         "axis", LONG_AXIS)), LONG, false /* isLargeFileSupport */, false/* fillValue */);
631         command.execute(new File(outputFileName), TIME, DataType.INT, Arrays.asList(new Attribute("units", TIME_UNITS),
632                 new Attribute("long_name", TIME_LONG_NAME), new Attribute("standard_name", TIME_STANDARD_NAME),
633                 new Attribute("axis", TIME_AXIS), new Attribute("calendar", TIME_CALENDAR), new Attribute("bounds",
634                         TIME_BOUNDS)), TIME, false /* isLargeFileSupport */, false/* fillValue */);
635         command.execute(new File(outputFileName), TIME_BOUNDS, DataType.INT, new ArrayList<Attribute>(), TIME + " "
636                 + NV, false /* isLargeFileSupport */, false/* fillValue */);
637         command.execute(new File(outputFileName), GRID_MAPPING, DataType.INT, Arrays.asList(new Attribute(
638                 "grid_mapping_name", CRS_GRID_MAPPING_NAME), new Attribute("longitude_of_prime_meridian",
639                 CRS_LONGITUDE_OF_PRIME_MERIDIAN), new Attribute("semi_major_axis", CRS_SEMI_MAJOR_AXIS),
640                 new Attribute("inverse_flattening", CRS_INVERSE_FLATTENING)), "",
641                 false /* isLargeFileSupport */, true/* fillValue */);
642     }
643 
644     /**
645      * Defines the spatial metadata variables for a WRON netCDF file, e.g. elevation, catchment id etc.
646      * 
647      * @param outputFileName
648      *            The file to written to
649      * @throws IOException
650      *             If the definitions cannot be written
651      */
652     private void defineSpatialVariables(String outputFileName) throws IOException
653     {
654         NcDefineVariable command = new NcDefineVariable();
655 
656         String spatialDim = LAT + " " + LONG;
657         command.execute(new File(outputFileName), ELEVATION, DataType.FLOAT, Arrays.asList(new Attribute("units",
658                 ELEVATION_UNITS), new Attribute("standard_name", ELEVATION_STANDARD_NAME), new Attribute("long_name",
659                 ELEVATION_LONG_NAME), new Attribute("_FillValue", ELEVATION_MISSING_VALUE), new Attribute(
660                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);
661         command
662                 .execute(new File(outputFileName), CATCHMENT_ID, DataType.FLOAT, Arrays.asList(new Attribute(
663                         "long_name", CATCHMENT_ID_LONG_NAME), new Attribute("_FillValue", CATCHMENT_ID_MISSING_VALUE), new Attribute(
664                         "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */, false/* fillValue */);
665         command
666                 .execute(new File(outputFileName), REPORTING_REGION, DataType.FLOAT, Arrays.asList(new Attribute(
667                         "long_name", REPORTING_REGION_LONG_NAME), new Attribute("_FillValue", REPORTING_REGION_MISSING_VALUE), new Attribute(
668                                 "grid_mapping", GRID_MAPPING)), spatialDim, false /* isLargeFileSupport */,
669                         false/* fillValue */);
670     }
671 
672     /**
673      * Defines the rainFall variable
674      * 
675      * @param outputFileName
676      * @throws IllegalArgumentException
677      * @throws IOException
678      * @throws ParseException
679      */
680     private void defineVariable(String outputFileName, String dimensions, int variableIndex)
681             throws IllegalArgumentException, IOException, ParseException
682     {
683         NcDefineVariable command = new NcDefineVariable();
684         command.execute(new File(outputFileName), variableNames[variableIndex], variableDataTypes[variableIndex],
685                 Arrays.asList(new Attribute("units", variableUnits[variableIndex]), new Attribute("long_name",
686                         variableLongNames[variableIndex]), new Attribute("missing_value", Float
687                         .valueOf(variableMissingValues[variableIndex])), new Attribute("_FillValue", Float
688                         .valueOf(variableFillValues[variableIndex])),  new Attribute("grid_mapping", 
689                         GRID_MAPPING)), dimensions, false /* isLargeFileSupport */,
690                 false/* fillValue */);
691     }
692 
693     /**
694      * Fills the coordinate variables, e.g. latitude, longitude, time.
695      * 
696      * @param outputFileName
697      * @param sortedLongitudes
698      * @param sortedLatitudes
699      * @param dates
700      * @param timeRange 
701      * @param timeSize 
702      * @param blockLongRange 
703      * @param blockLongSize 
704      * @param blockLatRange 
705      * @param blockLatSize 
706      * @throws UnsupportedEncodingException
707      * @throws IOException
708      * @throws java.text.ParseException
709      */
710     private void fillCoordinateVariables(String outputFileName, Set<String> sortedLongitudes,
711             Set<String> sortedLatitudes, List<String> dates, String blockLatRange, String blockLongRange, String timeRange) throws UnsupportedEncodingException, IOException,
712             java.text.ParseException
713     {
714         NcWriteVariable command = new NcWriteVariable();
715 
716         // ///////////////////////////////////////////////////////////////////////////////////////////
717         // fill longitude variable
718         // create String of latitudes to populate the variable
719         StringBuffer longitudeStringBuffer = new StringBuffer();
720         for (Iterator<String> iter = sortedLongitudes.iterator(); iter.hasNext();)
721         {
722             longitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));
723         }
724 
725         // call command to actually perform fill the file
726         command.execute(new File(outputFileName), LONG, blockLongRange, new ByteArrayInputStream(longitudeStringBuffer
727                 .toString().getBytes(ENCODING)), false);
728 
729         // ///////////////////////////////////////////////////////////////////////////////////////////
730         // fill latitude variable
731         // create String of latitudes to populate the variable
732         StringBuffer latitudeStringBuffer = new StringBuffer();
733         for (Iterator<String> iter = sortedLatitudes.iterator(); iter.hasNext();)
734         {
735             latitudeStringBuffer.append(iter.next()).append(System.getProperty("line.separator"));
736         }
737 
738         // call command to actually perform fill the file
739         command.execute(new File(outputFileName), LAT, blockLatRange, new ByteArrayInputStream(latitudeStringBuffer
740                 .toString().getBytes(ENCODING)), false);
741 
742         // ///////////////////////////////////////////////////////////////////////////////////////////
743         // fill time and time_bnds variables
744         StringBuffer timeBoundsStringBuffer = new StringBuffer();
745         StringBuffer dateStringBuffer = new StringBuffer();
746         boolean first = true;
747         for (String date : dates)
748         {
749             dateStringBuffer.append(date).append(System.getProperty("line.separator"));
750             if (!first)
751             {
752                 timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));
753             }
754             timeBoundsStringBuffer.append(date).append(System.getProperty("line.separator"));
755             first = false;
756         }
757         int dayAfterLastDay = Integer.parseInt(dates.get(dates.size()-1)) +1;
758         String finalString = String.valueOf(dayAfterLastDay);
759         timeBoundsStringBuffer.append(finalString).append(System.getProperty("line.separator"));
760 
761         // call command to actually perform fill the file
762         command.execute(new File(outputFileName), TIME, timeRange, new ByteArrayInputStream(dateStringBuffer
763                 .toString().getBytes(ENCODING)), false);
764                 
765         // call command to actually perform fill the file
766         command.execute(new File(outputFileName), TIME_BOUNDS, timeRange+","+NV_RANGE, new ByteArrayInputStream(
767                 timeBoundsStringBuffer.toString().getBytes(ENCODING)), false);
768         
769     }
770 
771     /**
772      * Populates the spatial variables with data
773      * 
774      * @param outputFileName
775      *            The netCDF file to be updated
776      * @param sortedCellData
777      *            The list of cell data in the same order as the spatial coordinates
778      * @throws IOException
779      *             If the data cannot be read
780      */
781     private void fillSpatialVariables(String outputFileName, List<CellData> sortedCellData) throws IOException
782     {
783         NcWriteVariable command = new NcWriteVariable();
784 
785         Set<String> latitudes = getSortedLatitudes(sortedCellData);
786         Set<String> longitudes = getSortedLongitudes(sortedCellData);
787         String spatialRange = "0-" + (latitudes.size() - 1) + ",0-" + (longitudes.size() - 1);
788 
789         // ///////////////////////////////////////////////////////////////////////////////////////////
790         // fill elevation variable
791         // create String of elevations to populate the variable
792         StringBuffer dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes,
793                 CellDataVariable.ELEVATION, "-999.99");
794 
795         // call command to actually perform fill the file
796         command.execute(new File(outputFileName), ELEVATION, spatialRange, new ByteArrayInputStream(dataBuffer
797                 .toString().getBytes(ENCODING)), false);
798 
799         // ///////////////////////////////////////////////////////////////////////////////////////////
800         // fill catchment id variable
801         // create String of catchment ids to populate the variable
802         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.CATCHMENTID, "0");
803 
804         // call command to actually perform fill the file
805         command.execute(new File(outputFileName), CATCHMENT_ID, spatialRange, new ByteArrayInputStream(dataBuffer
806                 .toString().getBytes(ENCODING)), false);
807 
808         // ///////////////////////////////////////////////////////////////////////////////////////////
809         // fill reporting region id variable
810         // create String of reporting region ids to populate the variable
811         dataBuffer = fillSpatialDataBuffer(sortedCellData, latitudes, longitudes, CellDataVariable.REPORTINGREGIONID,
812                 "0");
813 
814         // call command to actually perform fill the file
815         command.execute(new File(outputFileName), REPORTING_REGION, spatialRange, new ByteArrayInputStream(dataBuffer
816                 .toString().getBytes(ENCODING)), false);
817     }
818 
819     /**
820      * @param sortedCellData
821      * @param latitudes
822      * @param longitudes
823      * @param varNum
824      * @param fillValue
825      * @return
826      */
827     private StringBuffer fillSpatialDataBuffer(List<CellData> sortedCellData, Set<String> latitudes,
828             Set<String> longitudes, CellDataVariable varNum, String fillValue)
829     {
830         StringBuffer dataBuffer = new StringBuffer();
831         int cellNum = 0;
832         for (String latVal : latitudes)
833         {
834             for (String longVal : longitudes)
835             {
836                 if (cellNum < sortedCellData.size())
837                 {
838                     CellData cellData = sortedCellData.get(cellNum);
839                     if (cellData.latitude.equals(latVal) && cellData.longitude.equals(longVal))
840                     {
841                         switch (varNum)
842                         {
843                         case ELEVATION:
844                             dataBuffer.append(cellData.elevation);
845                             break;
846                         case CATCHMENTID:
847                             dataBuffer.append(cellData.catchmentId);
848                             break;
849                         case REPORTINGREGIONID:
850                             dataBuffer.append(cellData.reportingRegionId);
851                             break;
852                         }
853                         cellNum++;
854                     }
855                     else
856                     {
857                         dataBuffer.append(fillValue);
858                     }
859                 }
860                 else
861                 {
862                     dataBuffer.append(fillValue);
863                 }
864 
865                 dataBuffer.append(System.getProperty("line.separator"));
866             }
867 
868         }
869         return dataBuffer;
870     }
871 
872     /**
873      * Populate the data variable with fill values. Used to ensure cells within 
874      * the rectangular bounds but for which we have no data are set to the fill value. 
875      *   
876      * @param varIndex The variable being filled
877      * @param sortedLongitudes The longitudes to fill.
878      * @param sortedLatitudes The latitudes to fill
879      * @param dates The dates to fill
880      * @param dimensionOrdering The order of the dimensions in the files
881      * @throws IOException If the data cannot be written
882      */
883     private void fillDataForLatitude(String filename, int varIndex, Set<String> sortedLongitudes, Set<String> sortedLatitudes,
884             List<String> dates, List<String> dimensionOrdering) throws IOException
885     {
886         StringBuffer fillBlock = new StringBuffer();
887         String fillLine = variableFillValues[varIndex] + "\n";
888         for (int i = 0; i < dates.size(); i++)
889         {
890             fillBlock.append(fillLine);
891         }
892 
893         NcWriteVariable writeVarCmd = new NcWriteVariable();
894         for (String latitude : sortedLatitudes)
895         {
896             for (String longitude : sortedLongitudes)
897             {
898                 String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, dates.size());
899                 writeVarCmd.execute(new File(filename), variableNames[varIndex], fillRange,
900                         new ByteArrayInputStream(fillBlock.toString().getBytes(ENCODING)), false);
901             }
902         }
903     }
904 
905     /**
906      * Fills the variables with data for a particular latitude.
907      * 
908      * @param variableFileNames The filenames for this latitude for each of the variables
909      * @param triples The details of the cells that occur in this latitude 
910      * @param dimensionOrdering The order of the dimensions in the files
911      * @throws IOException If the data cannot be written
912      */
913     private void fillDataByLatitudeVariables(String[] variableFileNames, List<CellData> triples, List<String> dimensionOrdering)
914             throws IOException
915     {
916         NcWriteVariable command = new NcWriteVariable();
917 
918         int counter = 0;
919         for (CellData triple : triples)
920         {
921             if (++counter % 1000 == 0)
922             {
923                 LOG.warn("   Num files processed: " + counter);
924             }
925 
926             String csvFileName = triple.getCellId();
927             String latitude = triple.getLatitude();
928             String longitude = triple.getLongitude();
929 
930             String fillRange = this.constructFillRange(dimensionOrdering, latitude, longitude, TIME_SIZE);
931 
932             try
933             {
934                 StringBuffer[] valuesStringBuffer = new StringBuffer[numVariables];
935                 for (int i = 0; i < valuesStringBuffer.length; i++)
936                 {
937                     valuesStringBuffer[i] = new StringBuffer();
938                 }
939 
940                 Scanner s = null;
941                 try
942                 {
943                     s = new Scanner(new BufferedReader(new FileReader(INPUT_CSV_DIRECTORY + csvFileName + ".csv")))
944                             .useDelimiter(Pattern.compile("[\n,]"));
945 
946                     // remove header line
947                     for (int i = 0; i < NUM_CSV_COLUMNS; i++)
948                     {
949                         s.next();
950                     }
951 
952                     while (s.hasNext())
953                     {
954                         s.next(); // date column
955 
956                         for (int i = 0; i < numVariables; i++)
957                         {
958                             valuesStringBuffer[i].append(s.next());
959                             // Last variable will already have newline at its end.
960                             if (i < numVariables - 1)
961                             {
962                                 valuesStringBuffer[i].append(System.getProperty("line.separator"));
963                             }
964                         }
965                     }
966                 }
967                 finally
968                 {
969                     if (s != null)
970                     {
971                         s.close();
972                     }
973                 }
974 
975                 try
976                 {
977                     for (int i = 0; i < numVariables; i++)
978                     {
979                         command.execute(new File(variableFileNames[i]), variableNames[i], fillRange,
980                                 new ByteArrayInputStream(valuesStringBuffer[i].toString().getBytes(ENCODING)), false);
981                     }
982                 }
983                 catch (IllegalArgumentException iae)
984                 {
985                     LOG.error(iae);
986                     LOG.error("fillRange (date, lat, long): " + fillRange);
987                 }
988 
989             }
990             catch (FileNotFoundException fnfe)
991             {
992                 LOG.info(fnfe.getMessage());
993                 //System.out.println(fnfe.getMessage());
994             }
995         }
996     }
997 
998     /**
999      * @param dimensionOrdering
1000      * @param latitude
1001      * @param longitude
1002      * @param dates
1003      * @return
1004      */
1005     private String constructFillRange(List<String> dimensionOrdering, String latitude, String longitude, int dateSize)
1006     {
1007         String time_range = "0-" + String.valueOf(dateSize - 1);
1008         String latitude_range = "lookup(" + latitude + ")-lookup(" + latitude + ")";
1009         String longitude_range = "lookup(" + longitude + ")-lookup(" + longitude + ")";
1010 
1011         String fillRange = new String();
1012         for (Iterator<String> iter = dimensionOrdering.iterator(); iter.hasNext();)
1013         {
1014             String dimension = iter.next();
1015             if (dimension.equals(TIME))
1016             {
1017                 fillRange += time_range;
1018             }
1019             else if (dimension.equals(LAT))
1020             {
1021                 fillRange += latitude_range;
1022             }
1023             else if (dimension.equals(LONG))
1024             {
1025                 fillRange += longitude_range;
1026             }
1027             if (iter.hasNext())
1028                 fillRange += ",";
1029         }
1030         return fillRange;
1031     }
1032 
1033     /**
1034      * Groups triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file) by degrees of latitude.
1035      * 
1036      * @param locationLookupValues
1037      * @return
1038      */
1039     private Map<String, List<CellData>> getTriplesGroupedByLatitude(String[][] locationLookupValues)
1040     {
1041         Map<String, List<CellData>> triplesGroupedByLatitude = new HashMap<String, List<CellData>>();
1042         for (int i = 0; i < locationLookupValues.length; i++)
1043         {
1044             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];
1045             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];
1046             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];
1047             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];
1048             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];
1049             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];
1050 
1051             String latitudeDegree = String.valueOf(Float.valueOf(latitude).intValue());
1052 
1053             List<CellData> triplesInLatitude = triplesGroupedByLatitude.get(latitudeDegree);
1054             if (triplesInLatitude == null)
1055             {
1056                 triplesInLatitude = new ArrayList<CellData>();
1057                 triplesGroupedByLatitude.put(latitudeDegree, triplesInLatitude);
1058             }
1059             final CellData cellData = new CellData(cellId, longitude, latitude);
1060             cellData.elevation = elevation;
1061             cellData.catchmentId = catchmentId;
1062             cellData.reportingRegionId = reportingRegionId;
1063             triplesInLatitude.add(cellData);
1064         }
1065 
1066         // check that we have actually grouped all rows by latitude
1067         int count = 0;
1068         for (Iterator<String> iter = triplesGroupedByLatitude.keySet().iterator(); iter.hasNext();)
1069         {
1070             count += triplesGroupedByLatitude.get(iter.next()).size();
1071         }
1072         if (count != locationLookupValues.length)
1073         {
1074             LOG.error("latitudes where not grouped by latitude correctly");
1075             System.exit(-1);
1076         }
1077 
1078         return triplesGroupedByLatitude;
1079     }
1080 
1081     /**
1082      * Groups dates, e.g. 1895-01-01 by decade.
1083      * 
1084      * @param dates
1085      * @return
1086      */
1087     public Map<Integer, List<String>> getDatesByDecade(List<String> dates)
1088     {
1089         Map<Integer, List<String>> datesGroupedByDecade = new HashMap<Integer, List<String>>();
1090 
1091         for (String date : dates)
1092         {
1093             int year = Integer.valueOf(date.substring(0, 4));
1094             int decade = year / 10;
1095 
1096             List<String> datesInDecade = datesGroupedByDecade.get(decade);
1097             if (datesInDecade == null)
1098             {
1099                 datesInDecade = new ArrayList<String>();
1100                 datesGroupedByDecade.put(decade, datesInDecade);
1101             }
1102             datesInDecade.add(date);
1103         }
1104 
1105         // check that we have actually grouped all rows by latitude
1106         int count = 0;
1107         for (Iterator<Integer> iter = datesGroupedByDecade.keySet().iterator(); iter.hasNext();)
1108         {
1109             count += datesGroupedByDecade.get(iter.next()).size();
1110         }
1111         if (count != dates.size())
1112         {
1113             LOG.error("dates where not grouped decade correctly");
1114             System.exit(-1);
1115         }
1116 
1117         return datesGroupedByDecade;
1118     }
1119 
1120     /**
1121      * Loads all triples, i.e. <cellId,longitude,latitude> (a row in the WRON lookup file).
1122      * 
1123      * @param locationLookupValues
1124      * @return
1125      */
1126     private List<CellData> getAllTriples(String[][] locationLookupValues)
1127     {
1128         List<CellData> allTriples = new ArrayList<CellData>();
1129         for (int i = 0; i < locationLookupValues.length; i++)
1130         {
1131             String cellId = locationLookupValues[i][CELL_ID_COLUMN_INDEX];
1132             String longitude = locationLookupValues[i][LONG_COLUMN_INDEX];
1133             String latitude = locationLookupValues[i][LAT_COLUMN_INDEX];
1134             String elevation = locationLookupValues[i][ELEVATION_COLUMN_INDEX];
1135             String catchmentId = locationLookupValues[i][CATCHMENT_ID_COLUMN_INDEX];
1136             String reportingRegionId = locationLookupValues[i][REPORTING_REGION_ID_COLUMN_INDEX];
1137 
1138             CellData cellData = new CellData(cellId, longitude, latitude);
1139             cellData.elevation = elevation;
1140             cellData.catchmentId = catchmentId;
1141             cellData.reportingRegionId = reportingRegionId;
1142             allTriples.add(cellData);
1143         }
1144 
1145         // check that we have actually grouped all rows by latitude
1146         if (allTriples.size() != locationLookupValues.length)
1147         {
1148             LOG.error("lookup values where not read correctly");
1149             System.exit(-1);
1150         }
1151 
1152         return allTriples;
1153     }
1154 
1155     /**
1156      * Parses the WRON lookup file into an row-by-column string matrix.
1157      * 
1158      * @throws IOException
1159      */
1160     private String[][] readLookupFile() throws IOException
1161     {
1162         RandomAccessFile raf = new RandomAccessFile(new File(LOCATION_LOOKUP_FILE), "r");
1163         raf.readLine();// abandon the first line since it is a header line.
1164 
1165         List<String[]> lineList = new ArrayList<String[]>();
1166         String line = raf.readLine();
1167         while (line != null)
1168         {
1169             String[] cols = new CSVTokenizer(line).getAllColumns();
1170             lineList.add(cols);
1171             line = raf.readLine();
1172         }
1173 
1174         raf.close();
1175 
1176         String[][] lineCols = lineList.toArray(new String[][]{});
1177         return lineCols;
1178     }
1179 
1180     /**
1181      * Transpose a matrix, e.g. [a][b] -> [b][a]
1182      * 
1183      * @param gridValues
1184      * @return
1185      */
1186     private String[][] transpose(String[][] gridValues)
1187     {
1188         String[][] transposed = new String[gridValues[0].length][gridValues.length];
1189         for (int rows = 0; rows < gridValues.length; rows++)
1190         {
1191             for (int cols = 0; cols < gridValues[0].length; cols++)
1192             {
1193                 transposed[cols][rows] = gridValues[rows][cols];
1194             }
1195         }
1196         return transposed;
1197     }
1198 
1199     /**
1200      * Retrieve an ordered set containing all the latitude values from a list of triples.
1201      * 
1202      * @param triples
1203      * @return
1204      */
1205     private Set<String> getSortedLatitudes(List<CellData> triples)
1206     {
1207         Set<String> latitudes = new TreeSet<String>();
1208         for (CellData triple : triples)
1209         {
1210             latitudes.add(triple.getLatitude());
1211 //            if (latitudes.size() >= LAT_SIZE)
1212 //            {
1213 //                break;
1214 //            }
1215         }
1216         return latitudes;
1217     }
1218 
1219     /**
1220      * Retrieve an ordered set containing all the longitude values from a list of triples.
1221      * 
1222      * @param triples
1223      * @return
1224      */
1225     private Set<String> getSortedLongitudes(List<CellData> triples)
1226     {
1227         Set<String> longitudes = new TreeSet<String>();
1228         for (CellData triple : triples)
1229         {
1230             longitudes.add(triple.getLongitude());
1231 //            if (longitudes.size() >= LONG_SIZE)
1232 //            {
1233 //                break;
1234 //            }
1235         }
1236         return longitudes;
1237     }
1238 
1239     /**
1240      * @return
1241      * @throws IOException
1242      */
1243     private List<String> getDates() throws IOException
1244     {
1245         List<String> dates = new ArrayList<String>();
1246 
1247         RandomAccessFile raf = null;
1248         try
1249         {
1250             // open any csv file from scenario C directory and read in the date dimension
1251             File dir = new File(INPUT_CSV_DIRECTORY);
1252             String[] children = dir.list();
1253             if (children != null && children.length > 0)
1254             {
1255                 raf = new RandomAccessFile(new File(INPUT_CSV_DIRECTORY + children[0]), "r");
1256                 raf.readLine();// abandon the first line since it is a header line.
1257 
1258                 List<String[]> lineList = new ArrayList<String[]>();
1259                 String line = raf.readLine();
1260                 while (line != null)
1261                 {
1262                     String[] cols = new CSVTokenizer(line).getAllColumns();
1263                     lineList.add(cols);
1264                     line = raf.readLine();
1265                 }
1266                 String[][] lineCols = lineList.toArray(new String[][]{});
1267                 dates = Arrays.asList(transpose(lineCols)[DATE_COLUMN_INDEX]); // get the date column
1268             }
1269         }
1270         finally
1271         {
1272             if (raf != null)
1273                 raf.close();
1274         }
1275 
1276         return dates;
1277     }
1278 
1279     /**
1280      * Convert the date values into a newline delimited string of days since a given epoc.
1281      * 
1282      * @param dates
1283      * @return
1284      * @throws IOException
1285      * @throws java.text.ParseException
1286      */
1287     private List<String> getDatesAsDaysSinceEpoc(List<String> dates) throws IOException, java.text.ParseException
1288     {
1289         return daysSinceSinceEpocAsString(dates, isoDateTimeFormat.dateOnlyFormat(EPOC_DATE_STRING));
1290     }
1291 
1292     /**
1293      * Convert a string array representing dates to a list of strings representing the number of days since since a
1294      * given epoc.
1295      * 
1296      * @param dates
1297      * @param epoc
1298      * @return
1299      * @throws java.text.ParseException
1300      */
1301     private List<String> daysSinceSinceEpocAsString(List<String> dates, Date epoc) throws java.text.ParseException
1302     {
1303         List<String> convertedDates = new ArrayList<String>();
1304         for (String date : dates)
1305         {
1306             Date currentDate = isoDateTimeFormat.dateOnlyFormat(date);
1307             convertedDates.add(String.valueOf(daysBetween(epoc, currentDate)));
1308         }
1309         return convertedDates;
1310     }
1311 
1312     /**
1313      * @param d1
1314      *            the initial date.
1315      * @param d2
1316      *            the later date.
1317      * @return (d2 - d1) in days
1318      */
1319     public int daysBetween(Date d1, Date d2)
1320     {
1321         return (int) ((d2.getTime() - d1.getTime()) / (1000 * 60 * 60 * 24));
1322     }
1323 
1324     private String generateLatitudeFilename(String latitudeDegree, String variable)
1325     {
1326         String filename = OUTPUT_NETCDF_DIRECTORY + COLLECTION + "." + SCENARIO + ".";
1327         if (MODEL_NAMES[FILE_NAME_MODEL_NAME].length() > 0)
1328         {
1329             filename += MODEL_NAMES[FILE_NAME_MODEL_NAME] + ".";
1330         }
1331         if (CASE.length() > 0)
1332         {
1333             filename += CASE + ".";
1334         }
1335         filename += "Latitude-";
1336         if (latitudeDegree.startsWith("-"))
1337         {
1338             filename += latitudeDegree.substring(1) + "S."; 
1339         }
1340         else
1341         {
1342             filename += latitudeDegree + "N.";
1343         }
1344         filename += "Var-" + variable + NETCDF_FILE_EXTENSION;
1345         return filename;
1346     }
1347 
1348     private String generateDecadeFilename(String decadeStr, String variable)
1349     {
1350         String filename = OUTPUT_NETCDF_DIRECTORY + COLLECTION + "." + SCENARIO + ".";
1351         if (MODEL_NAMES[FILE_NAME_MODEL_NAME].length() > 0)
1352         {
1353             filename += MODEL_NAMES[FILE_NAME_MODEL_NAME] + ".";
1354         }
1355         if (CASE.length() > 0)
1356         {
1357             filename += CASE + ".";
1358         }
1359         filename += "Decade-" + decadeStr +  ".Var-" + variable + NETCDF_FILE_EXTENSION;
1360         return filename;
1361     }
1362 
1363     /**
1364      * Displays the contents of the in-memory data structure/
1365      * 
1366      * @param lookup
1367      */
1368     @SuppressWarnings("unused")
1369     private void printLookupInMemory(Map<Integer, Map<String, LongitudeRange>> lookup)
1370     {
1371         for (Iterator<Integer> dateIter = lookup.keySet().iterator(); dateIter.hasNext();)
1372         {
1373             Integer dateKey = dateIter.next();
1374             System.out.println(dateKey);
1375 
1376             for (Iterator<String> latIter = lookup.get(dateKey).keySet().iterator(); latIter.hasNext();)
1377             {
1378                 String latitude = latIter.next();
1379                 System.out.println("\t" + latitude);
1380 
1381                 LongitudeRange longitudeRange = lookup.get(dateKey).get(latitude);
1382                 System.out.println("\t\t" + longitudeRange.getStartLongitudeRange() + "-"
1383                         + longitudeRange.getEndLongitudeRange());
1384             }
1385         }
1386     }
1387 
1388     public enum CellDataVariable
1389     {
1390         CELLID, LONGITUDE, LATITUDE, ELEVATION, CATCHMENTID, REPORTINGREGIONID
1391     }
1392     
1393     /**
1394      * Maps a climate model name used for file naming to an IPCC standard climate model name.
1395      * 
1396      * @param directoryModelName
1397      *            a climate model name used for file naming.
1398      * @return a String[] which forms a mapping from a climate model name used for file naming to an IPCC standard
1399      *         climate model name.
1400      */
1401     private static String[] getModelMapping(String directoryModelName)
1402     {
1403         // strip "Model-" prefix that may be used.
1404         final String MODEL_NAME_PREFIX = "Model-";
1405         int beginIndex = directoryModelName.indexOf(MODEL_NAME_PREFIX);
1406         if (beginIndex != -1)
1407         {
1408             directoryModelName = directoryModelName.substring(beginIndex + (MODEL_NAME_PREFIX.length()));
1409         }
1410 
1411         // determine the mapping between the climate model used to name the file and the IPCC standard climate
1412         // model name stored as metadata.
1413         if (CLIMATE_MODELS.containsKey(directoryModelName))
1414         {
1415             String[] modelNameMapping = new String[2];
1416             modelNameMapping[FILE_NAME_MODEL_NAME] = MODEL_NAME_PREFIX + directoryModelName;
1417             modelNameMapping[IPCC_MODEL_NAME] = CLIMATE_MODELS.get(directoryModelName);
1418             return modelNameMapping;
1419         }
1420         throw new IllegalArgumentException("Unknown model: " + directoryModelName + ". Valid models are: "
1421                 + CLIMATE_MODELS.keySet());
1422     }
1423 }