/**
    * Copyright 2010, CSIRO Australia.
    *
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    *
    *         http://www.apache.org/licenses/LICENSE-2.0
    *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package au.csiro.netcdf.wron;
  
  import java.io.BufferedWriter;
  import java.io.File;
  import java.io.FileWriter;
  import java.io.IOException;
  import java.io.PrintWriter;
  import java.io.RandomAccessFile;
  import java.io.StringWriter;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.List;
  
  import org.apache.commons.cli.BasicParser;
  import org.apache.commons.cli.CommandLine;
  import org.apache.commons.cli.HelpFormatter;
  import org.apache.commons.cli.MissingOptionException;
  import org.apache.commons.cli.Option;
  import org.apache.commons.cli.OptionBuilder;
  import org.apache.commons.cli.Options;
  import org.apache.commons.cli.ParseException;
  import org.apache.log4j.Logger;
  
  import ucar.ma2.Array;
  import ucar.ma2.InvalidRangeException;
  import ucar.nc2.Attribute;
  import ucar.nc2.NetcdfFile;
  import ucar.nc2.Variable;
  import ucar.nc2.units.DateFormatter;
  import au.csiro.netcdf.cli.CommandLineOptionsComparator;
  import au.csiro.netcdf.util.CSVTokenizer;
  import au.csiro.netcdf.util.Util;
  
  /**
   * The class is a converter control class to convert Murray Darling Basin Sustainable Yields netCDF files into CSV
   * files.
   * 
   * Copyright 2010, CSIRO Australia
   * 
   * @author Robert Bridle on 07/07/2010
   * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $
   */
  public class MdbsyNetCDF2CSVConverter
  {
      /**
       * Constant that defines the logger to be used.
       */
      private static final Logger LOG = Logger.getLogger(MdbsyNetCDF2CSVConverter.class.getName());
  
      /**
       * The latitude variable name used in the netCDF files.
       */
      private static final String LATITUDE_VARIABLE_NAME = "lat";
  
      /**
       * The longitude variable name used in the netCDF files.
       */
      private static final String LONGITUDE_VARIABLE_NAME = "long";
  
      /**
       * The time variable name used in the netCDF files.
       */
      private static final String TIME_VARIABLE_NAME = "time";
  
      /**
       * The number of milliseconds in a 24 hours.
       */
      private static final Long MILLISECONDS_IN_A_DAY = new Long("86400000"); // milliseconds in a day.
  
      /**
       * The date column text that is found in all csv files.
       */
      // private static final String DATE_COLUMN_TEXT = writeOutDateColumn();
  
      /**
       * The name of the command line option used for specifying the input directory containing the netCDF files.
       */
      public static final String INPUT_DIR = "inputDir";
  
      /**
       * The name of the command line option used for specifying the directory to create the csv files in.
       */
      public static final String OUTPUT_DIR = "outputDir";
 
     /**
      * The pattern for accepted file names where multiple files are accepted.
      */
     public static final String FILENAME_PATTERN = "pattern";
 
     /**
      * The column that the variable name is stored in the csv file.
      */
     private static final int VARIABLE_COLUMN = 0;
 
     /**
      * The only variables that we are interested in writing out to csv files are those that are in three dimensions
      * (i.e. rank=3)
      */
     private static final int RANK_THREE = 3;
 
     /**
      * NetCDf file extension.
      */
     private static final Object CSV_FILE_EXTENSION = ".csv";
 
     /**
      * CSV file extension.
      */
     private static final Object NETCDF_FILE_EXTENSION = ".nc";
 
     /**
      * A string containing the date column text to be written out as the first column in all the csv files.
      */
     private static String DATE_COLUMN_TEXT = null;
 
     /**
      * The attribute name used to store a variable's type.
      */
     private static final String UNITS = "units";
 
     /**
      * Command line entry point for the MdbsyNetCDF2CSVConverter class. Gets the input and output directory names 
      * from the command line and runs a conversion.
      * 
      * @param args The command line arguments
      * @throws ParseException If an error occurs when parsing the arguments.
      * @throws IOException If the files cannot be read or written to.
      * @throws InvalidRangeException if an invalid range is attempted to be read from a netCDf variable
      */
     @SuppressWarnings("static-access")
     public static void main(String[] args) throws ParseException, IOException, InvalidRangeException
     {
         Options options = new Options();
         try
         {
             Option inputDirectoryName = OptionBuilder.withArgName("dir").hasArg().withDescription(
                     "1. the directory path containing the netCDF files (split by latitude) to be converted into csv.")
                     .isRequired(true).withLongOpt(INPUT_DIR).create("i");
 
             Option outputDirectoryName = OptionBuilder.withArgName("dir").hasArg().withDescription(
                     "2: the directory path to place the new csv files.").isRequired(true).withLongOpt(OUTPUT_DIR)
                     .create("o");
 
             options.addOption(inputDirectoryName);
             options.addOption(outputDirectoryName);
 
             // parse the command line arguments
             CommandLine parsedCommandLine = new BasicParser().parse(options, args);
 
             String inputDir = parsedCommandLine.getOptionValue(INPUT_DIR);
             String outputDir = parsedCommandLine.getOptionValue(OUTPUT_DIR);
 
             MdbsyNetCDF2CSVConverter converter = new MdbsyNetCDF2CSVConverter();
 
             long start = System.currentTimeMillis();
             converter.execute(inputDir, outputDir);
             long end = System.currentTimeMillis() - start;
             LOG.warn("Successfully converted all netcdf files to csv files in: " + end + " ms.");
         }
         catch (MissingOptionException moe)
         {
             LOG.error(moe.getMessage());
 
             // generate the help/usage statement
             String header = "Recreate ";
             String footer = "\nExample: --inputDir \"C:\\input\" --outputDir \"C:\\output\" \n"
                     + "Will convert the netCDF files (which are split by latitude) in the input directory to csv files in the output directory.";
             StringWriter sw = new StringWriter();
             HelpFormatter formatter = new HelpFormatter();
             formatter.setOptionComparator(new CommandLineOptionsComparator());
             final int lineWidth = 80;
             formatter.printHelp(new PrintWriter(sw), lineWidth, "-", header, options, 0, 1, footer);
             System.out.println(sw.toString());
         }
     }
 
     /**
      * Performs the conversion.
      * 
      * @param inputDir
      *            the directory containing the Scenario A or C netCDF files.
      * @param outputDir
      *            the directory in which the csv file will be placed.
      * @throws InvalidRangeException
      *             thrown if an invalid range is attempted to be read from a netCDf variable.
      * @throws IOException 
      *             If the files cannot be read or written to.
      */
     public void execute(String inputDir, String outputDir) throws IOException, InvalidRangeException
     {
         // perform an initial conversion from netCDF to csv, the csv files will look like this:
         // Date, 1985-01-01, 1895-01-02, 1985-01-03, ...
         // APET, 0.1, 0.3, 0.0, ....
         // rainfall,0.0, 0.4, 0.1, ...
         System.out.println("Starting intial convert of NetCDF 2 CSV.");
         LOG.info("Starting intial convert of NetCDF 2 CSV.");
         writeCSVFiles(inputDir, outputDir);
         System.out.println("Finished intial convert of NetCDF 2 CSV.");
         LOG.info("Finished intial convert of NetCDF 2 CSV.");
 
         // transpose the csv files so they will look like this:
         // Date, APET, rainfall
         // 1895-01-01, 0.1, 0.0
         // 1895-01-02, 0.3, 0.4
         // 1895-01-03, 0.0, 0.1
         System.out.println("Started transposition of CSV.");
         LOG.info("Started transposition of CSV.");
         transposeCSVFiles(outputDir);
         System.out.println("Finished transposition of CSV.");
         LOG.info("Finished transposition of CSV.");
     }
 
     /**
      * For each netCDF file in the input directory:
      * <p>
      * Create a csv file for every latitude and longitude position.
      * <p>
      * Write out the value of a variable at every time interval, at a specific latitude and longitude position.
      * 
      * @param inputDir
      *            the directory containing the Scenario A or C netCDF files.
      * @param outputDir
      *            the directory in which the csv file will be placed.
      * @throws InvalidRangeException
      *             thrown if an invalid range is attempted to be read from a netCDf variable.
      * @throws IOException
      */
     private void writeCSVFiles(String inputDir, String outputDir) throws IOException, InvalidRangeException
     {
         File dir = new File(inputDir);
         File[] files = dir.listFiles();
         if (files == null)
         {
             System.out.println("The input directory does not exist: " + inputDir);
             LOG.error("The input directory does not exist: " + inputDir);
             return;
         }
 
         for (int fileIndex = 0; fileIndex < files.length; fileIndex++)
         {
             if (files[fileIndex].isFile()
                     && NETCDF_FILE_EXTENSION.equals(Util.getFileExtension(files[fileIndex].getName())))
             {
                 NetcdfFile nc = null;
                 try
                 {
                     nc = NetcdfFile.open(files[fileIndex].getAbsolutePath());
 
                     Array latitudes = getLatitudeValues(nc);
                     Array longitudes = getLongitudeValues(nc);
 
                     LOG.info("latitude coordinate variable size: " + latitudes.getSize());
                     LOG.info("longitude coordinate variable size: " + longitudes.getSize());
 
                     for (int latIndex = 0; latIndex < latitudes.getSize(); latIndex++)
                     {
                         for (int longIndex = 0; longIndex < longitudes.getSize(); longIndex++)
                         {
                             String fileName = createFileNameBasedOnLatLong(nc, latIndex, longIndex);
                             String filePath = outputDir + System.getProperty("file.separator") + fileName + ".csv";
 
                             LOG.info("\tWorking on file: " + filePath);
 
                             // if the csv file already exists, then append the next variable.
                             if (Util.fileExists(filePath))
                             {
                                 File file = new File(filePath);
                                 PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file, true /*
                                                                                                                 * append
                                                                                                                 * mode
                                                                                                                 */)));
 
                                 List<String> variablesAlreadyWrittenOut = getVariablesAlreadyWrittenOut(file);
                                 List<String> variablesThatCanBeWrittenOut = getVariablesThatCanBeWrittenOut(nc);
 
                                 variablesThatCanBeWrittenOut.removeAll(variablesAlreadyWrittenOut);
 
                                 try
                                 {
                                     writeOutVariable(nc, out, variablesThatCanBeWrittenOut, latIndex, longIndex);
                                 }
                                 finally
                                 {
                                     if (out != null)
                                         out.close();
                                 }
                             }
                             else
                             // if the csv file does not exist, create a new csv file and write all available variables.
                             {
                                 PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(new File(filePath),
                                         false /* append mode */)));
 
                                 List<String> variablesThatCanBeWrittenOut = getVariablesThatCanBeWrittenOut(nc);
 
                                 try
                                 {
                                     out.println(writeOutDateColumn(nc)); // note: we only write out the date column when
                                                                          // a new csv file is created.
 
                                     writeOutVariable(nc, out, variablesThatCanBeWrittenOut, latIndex, longIndex);
                                 }
                                 finally
                                 {
                                     if (out != null)
                                         out.close();
                                 }
                             }
                         }
                     }
                 }
                 catch (java.io.FileNotFoundException fnfe)
                 {
                     System.out.println("file not found= " + files[fileIndex]);
                     LOG.error(fnfe);
                 }
                 finally
                 {
                     if (nc != null)
                         nc.close();
                 }
             }
         }
     }
 
     /**
      * Writes out the value of a variable at every time interval, at a specific latitude and longitude position.
      * 
      * @throws InvalidRangeException
      * @throws IOException
      * 
      */
     private void writeOutVariable(NetcdfFile nc, PrintWriter out, List<String> variablesThatCanBeWrittenOut,
             int latIndex, int longIndex) throws IOException, InvalidRangeException
     {
         for (String variableName : variablesThatCanBeWrittenOut)
         {
             Array subsection = getVariableAcrossTime(nc, variableName, latIndex, longIndex);
             out.println(variableName + "," + subsection.toString().replaceAll(" ", ","));
         }
     }
 
     /**
      * Transpose the csv files so that columns represent a variable's value over time.
      * 
      * @param outputDir
      *            the directory in which the csv file will be placed.
      * @throws IOException
      */
     private void transposeCSVFiles(String outputDir) throws IOException
     {
         File dir = new File(outputDir);
         File[] files = dir.listFiles();
         if (files == null)
         {
             System.out.println("The output directory does not exist: " + outputDir);
             LOG.error("The output directory does not exist: " + outputDir);
             return;
         }
 
         for (int fileIndex = 0; fileIndex < files.length; fileIndex++)
         {
             if (files[fileIndex].isFile()
                     && CSV_FILE_EXTENSION.equals(Util.getFileExtension(files[fileIndex].getName())))
             {
                 String[][] matrix = readLookupFile(files[fileIndex]);
 
                 // write over-the-top of the csv file.
                 PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(files[fileIndex], false /*
                                                                                                              * append
                                                                                                              * mode
                                                                                                              */)));
 
                 // transpose String[][] matrix and write back out to csv
                 String[][] transposed = transpose(matrix);
 
                 try
                 {
                     for (int row = 0; row < transposed.length; row++)
                     {
                         for (int col = 0; col < transposed[0].length; col++)
                         {
                             out.print(transposed[row][col]);
                             if (col < (transposed[0].length - 1))
                             {
                                 out.print(", ");
                             }
                         }
                         out.println();
                     }
                 }
                 finally
                 {
                     if (out != null)
                         out.close();
                 }
             }
         }
     }
 
     /**
      * Returns the names of the variables in the netCDF file that we need to write out for a given latitude x longitude
      * position.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @return the names of the variables in the netCDF file that we need to write out for a given latitude x longitude
      *         position.
      */
     private List<String> getVariablesThatCanBeWrittenOut(NetcdfFile nc)
     {
         List<String> variablesThatCanBeWrittenOut = new ArrayList<String>();
 
         for (Variable variable : nc.getVariables())
         {
             if (variable.getRank() == RANK_THREE)
             {
                 variablesThatCanBeWrittenOut.add(variable.getName().trim());
             }
         }
 
         return variablesThatCanBeWrittenOut;
     }
 
     /**
      * Returns the name of the variables that have already be written out to a csv file.
      * 
      * @param file
      *            the csv file being written out too.
      * @return the name of the variables that have already be written out to a csv file.
      * @throws IOException
      */
     private List<String> getVariablesAlreadyWrittenOut(File file) throws IOException
     {
         List<String> variablesAlreadyWrittenOut = new ArrayList<String>();
 
         String[][] matrix = readLookupFile(file);
         for (int rows = 0; rows < matrix.length; rows++)
         {
             variablesAlreadyWrittenOut.add(matrix[rows][VARIABLE_COLUMN].trim());
         }
 
         return variablesAlreadyWrittenOut;
     }
 
     /**
      * Creates a file name based on a given latitude and longitude.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @param latIndex
      *            the latitude index.
      * @param longIndex
      *            the longitude index.
      * @return a file name based on latitude and longitude position.
      * @throws InvalidRangeException
      * @throws IOException
      */
     private String createFileNameBasedOnLatLong(NetcdfFile nc, int latIndex, int longIndex) throws IOException,
             InvalidRangeException
     {
         return "region_" + nc.readSection("lat(" + latIndex + ":" + latIndex + ")").toString().trim() + "_"
                 + nc.readSection("long(" + longIndex + ":" + longIndex + ")").toString().trim();
     }
 
     /**
      * Parses a csv file into an row-by-column string matrix.
      * 
      * @param file
      *            a csv file.
      * @return row-by-column string matrix of the csv file.
      * @throws IOException
      */
     public static String[][] readLookupFile(File file) throws IOException
     {
         RandomAccessFile raf = new RandomAccessFile(file, "r");
         List<String[]> lineList = new ArrayList<String[]>();
 
         try
         {
             String line = raf.readLine();
             while (line != null)
             {
                 String[] cols = new CSVTokenizer(line).getAllColumns();
                 lineList.add(cols);
                 line = raf.readLine();
             }
         }
         finally
         {
             raf.close();
         }
 
         String[][] lineCols = lineList.toArray(new String[][] {});
         return lineCols;
     }
 
     /**
      * Transpose a matrix, e.g. [a][b] -> [b][a]
      * 
      * @param values
      *            a String[][] matrix.
      */
     private String[][] transpose(String[][] values)
     {
         if (values.length == 0)
         {
             return new String[0][0];
         }
 
         String[][] transposed = new String[values[0].length][values.length];
         for (int row = 0; row < values.length; row++)
         {
             for (int col = 0; col < values[0].length; col++)
             {
                 transposed[col][row] = values[row][col];
             }
         }
         return transposed;
     }
 
     /**
      * Returns the value of a variable at every time interval, at a specific latitude and longitude position.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @param variableName
      *            the variable whose values are to be read.
      * @param latIndex
      *            the latitude index.
      * @param longIndex
      *            the longitude index.
      * @return the value of a variable at every time interval, at a specific latitude and longitude position.
      * @throws IOException
      * @throws InvalidRangeException
      */
     private Array getVariableAcrossTime(NetcdfFile nc, String variableName, int latIndex, int longIndex)
             throws IOException, InvalidRangeException
     {
         try
         {
             return nc.readSection(variableName + "(" + latIndex + ":" + latIndex + "," + longIndex + ":" + longIndex
                     + ",:)");
         }
         catch (InvalidRangeException ire)
         {
             ire.printStackTrace();
             LOG.error("Could not read section: " + variableName + "(" + latIndex + ":" + latIndex + "," + longIndex
                     + ":" + longIndex + ",:)", ire);
         }
         return null;
     }
 
     /**
      * Returns the longitude coordinate variable values.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @return the longitude coordinate variable values.
      * @throws IOException
      * @throws IllegalArgumentException
      */
     private Array getLongitudeValues(NetcdfFile nc) throws IOException, IllegalArgumentException
     {
         List<Variable> variables = nc.getVariables();
         if (variables != null)
         {
             for (Variable v : variables)
             {
                 if (v.isCoordinateVariable() && LONGITUDE_VARIABLE_NAME.equals(v.getName()))
                 {
                     return v.read();
                 }
             }
         }
         LOG.error("Could not find coordinate variable: " + LONGITUDE_VARIABLE_NAME);
         throw new IllegalStateException("Could not find coordinate variable: " + LONGITUDE_VARIABLE_NAME);
     }
 
     /**
      * Returns the latitude coordinate variable values.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @return the latitude coordinate variable values.
      * @throws IOException
      * @throws IllegalArgumentException
      */
     private Array getLatitudeValues(NetcdfFile nc) throws IOException, IllegalArgumentException
     {
         List<Variable> variables = nc.getVariables();
         if (variables != null)
         {
             for (Variable v : variables)
             {
                 if (v.isCoordinateVariable() && LATITUDE_VARIABLE_NAME.equals(v.getName()))
                 {
                     return v.read();
                 }
             }
         }
         LOG.error("Could not find coordinate variable: " + LATITUDE_VARIABLE_NAME);
         throw new IllegalStateException("Could not find coordinate variable: " + LATITUDE_VARIABLE_NAME);
     }
 
     /**
      * Returns the time variable.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @return the time variable.
      * @throws IllegalArgumentException
      */
     private static Variable getTimeVariable(NetcdfFile nc) throws IllegalArgumentException
     {
         List<Variable> variables = nc.getVariables();
         if (variables != null)
         {
             for (Variable v : variables)
             {
                 if (v.isCoordinateVariable() && TIME_VARIABLE_NAME.equals(v.getName()))
                 {
                     return v;
                 }
             }
         }
         LOG.error("Could not find coordinate variable: " + TIME_VARIABLE_NAME);
         throw new IllegalStateException("Could not find coordinate variable: " + TIME_VARIABLE_NAME);
     }
 
     /**
      * Converts a "days since" time coordinate variable into a list of formatted dates.
      * 
      * @param nc
      *            a netCDF file {@link NetcdfFile}
      * @return a list of formatted dates.
      */
     private static String writeOutDateColumn(NetcdfFile nc)
     {
         // we only need to create the date column text once, it is the same for all csv files.
         if (DATE_COLUMN_TEXT == null)
         {
             StringBuffer strBuf = new StringBuffer();
 
             DateFormatter dateFormatter = new DateFormatter();
             Date epocDate;
             try
             {
                 Variable time = getTimeVariable(nc);
 
                 String timeUnits = getVariablesUnits(time);
                 String epocString = timeUnits.substring(timeUnits.indexOf("days since") + 11); // we expect the time
                                                                                                // value to be in units
                                                                                                // of:
                                                                                                // "days since yyyy-mm-dd h:m:s"
 
                 epocDate = dateFormatter.dateOnlyFormat(epocString);
                 strBuf.append("Date,");
 
                 long timeVariableSize = time.getSize();
                 for (int i = 0; i < timeVariableSize; i++)
                 {
                     strBuf.append(dateFormatter.toDateOnlyString(
                             new Date(epocDate.getTime() + (i * MILLISECONDS_IN_A_DAY))).trim());
 
                     if (i < (timeVariableSize - 1))
                     {
                         strBuf.append(",");
                     }
                 }
 
                 DATE_COLUMN_TEXT = strBuf.toString();
             }
             catch (java.text.ParseException pe)
             {
                 pe.printStackTrace();
                 LOG.error("Error parsing date", pe);
             }
         }
         return DATE_COLUMN_TEXT;
     }
 
     /**
      * Finds the string value of the attribute that is used to store a variable's type.
      * 
      * @param variable
      *            a variable {@link Variable}
      * @return the string value of the variable's type.
      */
     private static String getVariablesUnits(Variable variable)
     {
         List<Attribute> attributes = variable.getAttributes();
         if (attributes != null)
         {
             for (Attribute attribute : attributes)
             {
                 if (UNITS.equals(attribute.getName()))
                 {
                     return attribute.getStringValue();
                 }
             }
         }
         LOG
                 .error("Could not find the value for the variable: " + variable.getName() + " -\"" + UNITS
                         + "\" attribute.");
         throw new IllegalStateException("Could not find the value for the variable: " + variable.getName() + " -\""
                 + UNITS + "\" attribute.");
     }
 }