/**

  * Copyright 2010, CSIRO Australia.

  *

  * Licensed under the Apache License, Version 2.0 (the "License");

  * you may not use this file except in compliance with the License.

  * You may obtain a copy of the License at

  *

  *         http://www.apache.org/licenses/LICENSE-2.0

  *

  * Unless required by applicable law or agreed to in writing, software

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  * See the License for the specific language governing permissions and

  * limitations under the License.

  */

 /**

  * 

  */

 package au.csiro.netcdf;

 import java.io.BufferedInputStream;

 import java.io.BufferedReader;

 import java.io.DataInputStream;

 import java.io.EOFException;

 import java.io.File;

 import java.io.FileInputStream;

 import java.io.IOException;

 import java.io.InputStream;

 import java.io.InputStreamReader;

 import java.io.PrintWriter;

 import java.io.StringWriter;

 import java.util.ArrayList;

 import java.util.List;

 import org.apache.commons.cli.BasicParser;

 import org.apache.commons.cli.CommandLine;

 import org.apache.commons.cli.HelpFormatter;

 import org.apache.commons.cli.Option;

 import org.apache.commons.cli.OptionBuilder;

 import org.apache.commons.cli.Options;

 import org.apache.commons.cli.ParseException;

 import org.apache.log4j.Logger;

 import ucar.ma2.Array;

 import ucar.ma2.InvalidRangeException;

 import ucar.ma2.Range;

 import ucar.nc2.NetcdfFile;

 import ucar.nc2.NetcdfFileWriteable;

 import ucar.nc2.Variable;

 import au.csiro.netcdf.cli.Command;

 import au.csiro.netcdf.cli.CommandLineOptionsComparator;

 import au.csiro.netcdf.util.NetCDFUtils;

 import au.csiro.netcdf.util.Util;

 /**

  * The <strong>ncwriteVar</strong> command writes a specified portion of data to a {@link Variable} 

  * object in a netCDF file.

  * 

  * Copyright 2010, CSIRO Australia

  * All rights reserved.

  * 

  * @author      James Dempsey on 18/03/2010

  * @version     $Revision: 86 $  $Date: 2010-08-30 17:31:03 +1000 (Mon, 30 Aug 2010) $

  */

 public class NcWriteVariable implements Command

 {

     /**

      * Constant that defines the logger to be used.

      */

     private static final Logger LOG = Logger.getLogger(NcWriteVariable.class.getName());

     /**

      * The command name

      */

     public static final String NC_WRITE_VAR_COMMAND_NAME = "ncwriteVar";

     /**

      * The name of the command line option used for specifying the output netCDF file name.

      */

     private static final String OUTPUT_FILE = "outputFileName";

     /**

      * The name of the command line option used for specifying the input text file name.

      */

     private static final String INPUT_FILE = "inputFileName";

     /**

      * The name of the command line option used for specifying the name of the variable.

      */

     private static final String VARIABLE_NAME = "variableName";

     /**

      * The name of the command line option used for specifying the range to be filled.

      */

     private static final String FILL_RANGE = "fillRange";

     /**

      * The name of the command line option used for specifying the data is binary.

      */

     private static final String BINARY = "binary";

     /** Command options  */

     private Options options;

     /**

      * Default constructor.

      */

     public NcWriteVariable()

     {

         this.options = createOptions();

     }

     /* (non-Javadoc)

      * @see au.csiro.netcdf.cli.Command#execute(java.lang.String[])

      */

     @Override

     public void execute(String[] args) throws ParseException, IOException

     {

         // parse the command line arguments

         CommandLine parsedCommandLine = new BasicParser().parse(options, args);

         // get the command line argument values

         String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine

                 .getOptionValue(OUTPUT_FILE) : "";

         String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine

                 .getOptionValue(VARIABLE_NAME) : "";

         String fillRangeArg = (parsedCommandLine.hasOption(FILL_RANGE)) ? parsedCommandLine

                 .getOptionValue(FILL_RANGE) : "";

         String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine

                 .getOptionValue(INPUT_FILE) : "";

         boolean binary = parsedCommandLine.hasOption(BINARY);

         // convert command line parameters to their associated types, note these can throw IllegalArgumentExceptions.

         File outputFile = Util.getExistingFile(outputFilenameArg);

         InputStream input;

         if (inputFilenameArg.length()>0)

         {

             input = new FileInputStream(inputFilenameArg);

         }

         else

         {

             input = System.in;

         }

         this.execute(outputFile, variableNameArg, fillRangeArg, input, binary);

         if (inputFilenameArg.length()>0)

         {

             input.close();

         }

     }

     /**

      * Write data to a variable. This is a library interface used by the command line 

      * ncwriteVar command to put data into a netCDF file. The fill range must be a comma 

      * separated list with values for each dimension of the variable. Values may be either 

      * a range n-n or a value n. e.g. 5-10, 3  

      * 

      * @param outputFile The netCDF file to be updated. 

      * @param variableName The variable to be updated.

      * @param fillRange The range in which the data is to be written. Should be of the form n-n, n-n, n...

      * @param dataStream The stream from which input data is read. Assumed to contain text with one value per line. 

      * @param binary True if the dataStream contains binary data, false if it is text

      * @throws IOException If either the netCDF file or the input stream cannot be accessed.

      */

     public void execute(File outputFile, String variableName, String fillRange, InputStream dataStream, boolean binary)

             throws IOException

     {

         // the netcdf file to be written.

         NetcdfFileWriteable ncfile = null;

         ncfile = NetcdfFileWriteable.openExisting(outputFile.getPath(), false); 

         try

         {

             Variable variable = ncfile.findVariable(NetcdfFile.escapeName(variableName));

             // Check the variable exists

             if (variable == null)

             {

                 throw new IllegalArgumentException("The variable: " + variableName + " does not exist in the file: "

                         + ncfile.getLocation());

             }

             List<Range> ranges = variable.getRanges();

             // Check the ranges match

             String fillRangeVal[] = fillRange.split(",");

             if (ranges.size() != fillRangeVal.length)

             {

                 throw new IllegalArgumentException("Expected " + ranges.size() + " fillRange arguments but only got "

                         + fillRangeVal.length);

             }

             // Calculate the range blocks

             Range fillRanges[] = convertStringsToRanges(fillRangeVal, variable, ncfile);

             List<int[]> originList = createOriginList(fillRanges);

             // Make the array that will buffer the data for a block

             Array buffer = createArray(fillRanges, variable);

             // Read the data

             if (binary)

             {

                 DataInputStream dataIn = new DataInputStream(new BufferedInputStream(dataStream));

                 // Write each block to the variable

                 for (int[] origin : originList)

                 {

                     readBinaryData(dataIn, buffer);

                     ncfile.write(variableName, origin, buffer);

                 }

                 dataIn.close();

             }

             else

             {

                 BufferedReader dataIn = new BufferedReader(new InputStreamReader(dataStream));

                 // Write each block to the variable

                 for (int[] origin : originList)

                 {

                     readData(dataIn, buffer);

                     ncfile.write(variableName, origin, buffer);

                 }

                 dataIn.close();

             }

         }

         catch (InvalidRangeException e)

         {

             LOG.error("Failed to write data due to illegal shaped values array.", e);

             throw new IllegalArgumentException("Failed to write data due to illegal shaped values array.", e);

         }

         finally

         {

             ncfile.close();

         }

     }

     /* (non-Javadoc)

      * @see au.csiro.netcdf.cli.Command#getCommandName()

      */

     @Override

     public String getCommandName()

     {

         return NC_WRITE_VAR_COMMAND_NAME;

     }

     /* (non-Javadoc)

      * @see java.lang.Object#toString()

      */

     @Override

     public String toString()

     {

         // generate the help/usage statement

         String header = "Write data to variable in a netCDF file.";

         String footer = "\nExample: ncwritevar -outputFileName ABC.nc -variableName Lat " + "-fillRange 0-559\n"

                 + "Will write data from stdin to the variable Lat in the file ABC.nc. The data will "

                 + "be written to all 560 cells in the variable. The file must already exist and "

                 + "the variable must be already defined in the file. Data would normally be piped in "

                 + "from a command such as ncextractcsv.\n"

                 + "\nExample: ncwritevar -outputFileName ABC.nc -variableName rainfall "

                 + "-fillRange lookup(-42.8)-lookup(-35.3)\n"

                 + "Will write data from stdin to the variable rainfall in the file ABC.nc. The data will "

                 + "only be written to the range between the two listed values. \n"

                 + "The lookup function allows you to convert a value to an index for a particular dimension. "

                 + "For the lookup to work, the dimension must have a coordinate variable defined (a variable "

                 + "associated with the dimension and having the same name as the dimension). For best performance "

                 + "the values in the coordinate variable should be in ascending order.";

         StringWriter sw = new StringWriter();

         HelpFormatter formatter = new HelpFormatter();

         formatter.setOptionComparator(new CommandLineOptionsComparator());

         formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NC_WRITE_VAR_COMMAND_NAME, header, options, 0, 1, footer);

         return sw.toString();

     }

     /* (non-Javadoc)

      * @see au.csiro.netcdf.cli.Command#getUsageString()

      */

     @Override

     public String getUsageString()

     {

         return toString();

     }

     /* (non-Javadoc)

      * @see au.csiro.netcdf.cli.Command#createOptions()

      */

     @SuppressWarnings("static-access")

     @Override

     public Options createOptions()

     {

         Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(

                 "1. the filename of the netCDF file to be updated.").isRequired(true).withLongOpt(OUTPUT_FILE).create(

                 "o");

         Option inputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(

                 "2. the filename of the text file to be loaded. "

                         + "Stdin will be used if this option is not present. OPTIONAL ").isRequired(false).withLongOpt(

                 INPUT_FILE).create("i");

         Option variableName = OptionBuilder.withArgName("text").hasArg().withDescription("3. the name of the variable.")

                 .isRequired(true).withLongOpt(VARIABLE_NAME).create("v");

         Option fillRange = OptionBuilder.withArgName("text").hasArg().withDescription(

                 "4. the range of the variable to be filled. Must be a comma "

                         + "separated list with values for each dimension of the variable. "

                         + "Values may be a range n-n or a value n. e.g. 5-10, 3").isRequired(true).withLongOpt(

                 FILL_RANGE).create("f");

         Option binary = OptionBuilder.withDescription("5. indicates the data is in a binary format.").isRequired(false)

                 .withLongOpt(BINARY).create("b");

         Options options = new Options();

         options.addOption(outputFileName);

         options.addOption(variableName);

         options.addOption(fillRange);

         options.addOption(inputFileName);

         options.addOption(binary);

         return options;

     }

     /* (non-Javadoc)

      * @see au.csiro.netcdf.cli.Command#validCommand(java.lang.String[])

      */

     @Override

     public String validCommand(String[] args)

     {

         String errorMsg = "";

         // validate the command line parameters

         try

         {

             // try parsing the command line, where ParseException can be thrown from

             CommandLine parsedCommandLine = new BasicParser().parse(options, args);

             // try converting command line parameters to their associated types, where IllegalArgumentException can be

             // thrown from

             String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine

                     .getOptionValue(OUTPUT_FILE) : "";

             String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine

                     .getOptionValue(INPUT_FILE) : "";

             String fillRangeArg = (parsedCommandLine.hasOption(FILL_RANGE)) ? parsedCommandLine

                     .getOptionValue(FILL_RANGE) : "";

             Util.getExistingFile(outputFilenameArg);

             if (inputFilenameArg.length() > 0)

             {

                 Util.getExistingFile(inputFilenameArg);

             }

             validateRangeFormat(fillRangeArg);

             // Variables will be validated when processed and the file is available

         }

         catch (ParseException pe)

         {

             errorMsg = errorMsg + pe.getMessage();

         }

         catch (IllegalArgumentException iae)

         {

             errorMsg = errorMsg + iae.getMessage();

         }

         return errorMsg;

     }

     /**

      * Fill range must be a comma separated list of ranges, each of which are either a 

      * lookup or numeric. Throws an IllegalArgumentException if any problems are found.

      * 

      * @param fillRangeArg The fill range value to be validated

      */

     private void validateRangeFormat(String fillRangeArg)

     {

         final String lookupPattern = "(lookup\\(.*\\))|(\\d+)(-((lookup\\(.*\\))|(\\d+)))?";

         String ranges[] = fillRangeArg.split(",");

         List<String> errors = new ArrayList<String>();

         for (String range : ranges)

         {

             if (!range.trim().matches(lookupPattern))

             {

                 errors.add(range + " is not a valid range. It must be of the form "

                         + "0, 0-9, lookup(0) or lookup(0)-lookup(9) specifying "

                         + "a start and optional end to the range.");

             }

         }

         if (!errors.isEmpty())

         {

             String err = "";

             for (String string : errors)

             {

                 err += "\n" + string;

             }

             throw new IllegalArgumentException(FILL_RANGE + " value is not a comma separated String: " + fillRangeArg

                     + err);

         }

     }

     /**

      * Convert an array of strings defining fill ranges to range objects. Each string 

      * is assumed to be in the format <code>n-n</code> or the format <code>n</code> 

      * where n is a number or a lookup instruction.

      *   

      * @param rangeStrings The string fill ranges

      * @return An array of range objects

      * @throws IOException If the data cannot be read to perform a lookup

      * @throws NumberFormatException If numeric values are not valid numbers.

      */

     Range[] convertStringsToRanges(String[] rangeStrings, Variable variable, NetcdfFile file)

             throws NumberFormatException, IOException

     {

         final String lookupPattern = "^lookup\\(.*\\)$";

         final int lookupPrefixLen = 7;

         Range[] ranges = new Range[rangeStrings.length];

         for (int i = 0; i < rangeStrings.length; i++)

         {

             String string = rangeStrings[i];

             String[] digits = splitRange(string);

             int first;

             // Deal with the index lookup functions

             if (digits[0].matches(lookupPattern))

             {

                 first = NetCDFUtils.lookupIndex(file, variable.getDimension(i), digits[0].substring(lookupPrefixLen,

                         digits[0].length() - 1));

             }

             else

             {

                 first = Integer.parseInt(digits[0]);

             }

             int last = first;

             if (digits.length > 1)

             {

                 if (digits[1].matches(lookupPattern))

                 {

                     last = NetCDFUtils.lookupIndex(file, variable.getDimension(i), digits[1].substring(lookupPrefixLen,

                             digits[1].length() - 1));

                 }

                 else

                 {

                     last = Integer.parseInt(digits[1]);

                 }

             }

             try

             {

                 ranges[i] = new Range(first, last, 1);

             }

             catch (InvalidRangeException e)

             {

                 LOG.error("Invalid range: " + string, e);

                 throw new IllegalArgumentException("Invalid range: " + string);

             }

         }

         return ranges;

     }

     /**

      * Split a range string with regard to the possibility of negative lookup values.

      * e.g. lookup(-52)-lookup(-40) get returned as lookup(-52) and lookup(-40).

      * @param rangeStr The range string to be split

      * @return The array of range start and end strings

      */

     String[] splitRange(String rangeStr)

     {

         String safeString = rangeStr.trim().replaceAll("(\\()-([^\\)]*\\))"  , "$1~$2");

         String[] result =  safeString.split("-");

         for (int i = 0; i < result.length; i++)

         {

             result[i] = result[i].trim().replaceAll("(\\()~([^\\)]*\\))"  , "$1-$2");

         }

         return result;

     }

     /**

      * Produce a list of origin points, being all combinations of points in 

      * the range other than the final range. The final range will always be 

      * written as a single block. Each range represents the section of a 

      * dimension that is to be populated.

      *  

      * @param fillRanges The ranges of each dimension that is to be populated. 

      * @return A list of origin points to be written to.   

      */

     List<int[]> createOriginList(Range[] fillRanges)

     {

         // A single range is a special case, get it out of the way first

         if (fillRanges.length == 1)

         {

             int[] origin = new int[]{fillRanges[0].first()};

             List<int[]> singleOrigin = new ArrayList<int[]>();

             singleOrigin.add(origin);

             return singleOrigin;

         }

         // The list of origin points for all ranges up to the current one

         List<List<int[]>> originForRange = new ArrayList<List<int[]>>();

         // Create the list of all origin points for the first range

         List<int[]> originList = new ArrayList<int[]>();

         for (int j = fillRanges[0].first(); j <= fillRanges[0].last(); j+=fillRanges[0].stride())

         {

             originList.add(new int[]{j});

         }

         originForRange.add(originList);

         // Loop over the later ranges expanding the list of origin combinations

         for (int rangeNum = 1; rangeNum < fillRanges.length-1; rangeNum++)

         {

             Range currRange = fillRanges[rangeNum];

             // Grab the combo of origin points for all ranges up to the previous one

             List<int[]> prevOrigins = originForRange.get(rangeNum-1); 

             List<int[]> currOrigins = new ArrayList<int[]>();

             for (int[] prevPoint : prevOrigins)

             {

                 for (int j = currRange.first(); j <= currRange.last(); j+=currRange.stride())

                 {

                     int[] origin = new int[rangeNum+1];

                     for (int j2 = 0; j2 < prevPoint.length; j2++)

                     {

                         origin[j2] = prevPoint[j2];

                     }

                     origin[rangeNum] = j;

                     currOrigins.add(origin);

                 }

             }

             originForRange.add(currOrigins);

         }

         // Add the start of the last range to generated origins

         List<int[]> origins = originForRange.get(fillRanges.length-2);

         List<int[]> finalOrigins = new ArrayList<int[]>(origins.size());

         int startFinalRange = fillRanges[fillRanges.length-1].first();

         for (int[] origin : origins)

         {

             int[] newOrigin = new int[fillRanges.length];

             for (int i = 0; i < origin.length; i++)

             {

                 newOrigin[i] = origin[i];

             }

             newOrigin[fillRanges.length-1] = startFinalRange;

             finalOrigins.add(newOrigin);

         }

         // Return the final entry, being the full set of combinations

         return finalOrigins;

     }

     /**

      * Create an array to hold a block of data.

      * @param fillRanges The range definition

      * @param variable The variable to be written

      * @return An array suitable to load data into

      */

     Array createArray(Range[] fillRanges, Variable variable)

     {

         int shape[] = new int[fillRanges.length];

         for (int i = 0; i < fillRanges.length; i++)

         {

             if (i<fillRanges.length-1)

             {

                 shape[i] = 1;

             }

             else

             {

                 Range lastRange = fillRanges[fillRanges.length-1]; 

                 shape[i] = lastRange.last() - lastRange.first() + 1;

             }

         }

         Array array = Array.factory(variable.getDataType(), shape);

         return array;

     }

     /**

      * Read in data form a stream containing text.

      * @param dataIn The text stream to be read

      * @param buffer The Array to be populated

      * @throws IOException If data cannot be read from the stream

      */

     @SuppressWarnings("unchecked")

     void readData(BufferedReader dataIn, Array buffer) throws IOException

     {

         Class cls = buffer.getElementType();// getClass();

         boolean isFloat = cls.getName().equalsIgnoreCase("Float");

         boolean isDouble = cls.getName().equalsIgnoreCase("Double");

         boolean isLong = cls.getName().equalsIgnoreCase("Long");

         boolean isInt = cls.getName().equalsIgnoreCase("Int");

         boolean isChar = cls.getName().equalsIgnoreCase("Char");

         boolean isByte = cls.getName().equalsIgnoreCase("Byte");

         boolean isBoolean = cls.getName().equalsIgnoreCase("Boolean");

         boolean isShort = cls.getName().equalsIgnoreCase("Short");

         int i = 0;

         while (i < buffer.getSize())

         {

             String line = dataIn.readLine();

             if (line == null)

             {

                 String error = "Expected block of " + buffer.getSize() + " values but ran out after " + i + ".";

                 System.err.println(error);

                 throw new IllegalArgumentException(error);

             }

             if (isFloat)

             {

                 buffer.setFloat(i, Float.parseFloat(line));

             }

             else if (isDouble)

             {

                 buffer.setDouble(i, Double.parseDouble(line));

             }

             else if (isLong)

             {

                 buffer.setLong(i, Long.parseLong(line));

             }

             else if (isInt)

             {

                 buffer.setInt(i, Integer.parseInt(line));

             }

             else if (isChar)

             {

                 buffer.setChar(i, line.charAt(0));

             }

             else if (isByte)

             {

                 buffer.setByte(i, Byte.parseByte(line));

             }

             else if (isBoolean)

             {

                 buffer.setBoolean(i, Boolean.parseBoolean(line));

             }

             else if (isShort)

             {

                 buffer.setShort(i, Short.parseShort(line));

             }

             i++;

         }

     }

     /**

      * Read in data form a stream containing binary data.

      * @param dataIn The data stream to be read

      * @param buffer The Array to be populated

      * @throws IOException If data cannot be read from the stream

      */

     @SuppressWarnings("unchecked")

     void readBinaryData(DataInputStream dataIn, Array buffer) throws IOException

     {

         Class cls = buffer.getElementType();// getClass();

         boolean isFloat = cls.getName().equalsIgnoreCase("Float");

         boolean isDouble = cls.getName().equalsIgnoreCase("Double");

         boolean isLong = cls.getName().equalsIgnoreCase("Long");

         boolean isInt = cls.getName().equalsIgnoreCase("Int");

         boolean isChar = cls.getName().equalsIgnoreCase("Char");

         boolean isByte = cls.getName().equalsIgnoreCase("Byte");

         boolean isBoolean = cls.getName().equalsIgnoreCase("Boolean");

         boolean isShort = cls.getName().equalsIgnoreCase("Short");

         int i = 0;

         while (i < buffer.getSize())

         {

             try

             {

                 if (isFloat)

                 {

                     buffer.setFloat(i, dataIn.readFloat());

                 }

                 else if (isDouble)

                 {

                     buffer.setDouble(i, dataIn.readDouble());

                 }

                 else if (isLong)

                 {

                     buffer.setLong(i, dataIn.readLong());

                 }

                 else if (isInt)

                 {

                     buffer.setInt(i, dataIn.readInt());

                 }

                 else if (isChar)

                 {

                     buffer.setChar(i, dataIn.readChar());

                 }

                 else if (isByte)

                 {

                     buffer.setByte(i, dataIn.readByte());

                 }

                 else if (isBoolean)

                 {

                     buffer.setBoolean(i, dataIn.readBoolean());

                 }

                 else if (isShort)

                 {

                     buffer.setShort(i, dataIn.readShort());

                 }

             }

             catch (EOFException e)

             {

                 String error = "Expected block of " + buffer.getSize() + " values but ran out after " + i + ".";

                 System.err.println(error);

                 throw new IllegalArgumentException(error);

             }

             i++;

         }

     }

 }