/**

  * Copyright 2010, CSIRO Australia.

  *

  * Licensed under the Apache License, Version 2.0 (the "License");

  * you may not use this file except in compliance with the License.

  * You may obtain a copy of the License at

  *

  *         http://www.apache.org/licenses/LICENSE-2.0

  *

  * Unless required by applicable law or agreed to in writing, software

  * distributed under the License is distributed on an "AS IS" BASIS,

  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  * See the License for the specific language governing permissions and

  * limitations under the License.

  */

 /**

  * 

  */

 package au.csiro.netcdf;

 import java.io.File;

 import java.io.FileInputStream;

 import java.io.IOException;

 import java.io.PrintWriter;

 import java.io.StringWriter;

 import java.util.ArrayList;

 import java.util.HashSet;

 import java.util.List;

 import java.util.Set;

 import java.util.regex.PatternSyntaxException;

 import org.apache.commons.cli.BasicParser;

 import org.apache.commons.cli.CommandLine;

 import org.apache.commons.cli.HelpFormatter;

 import org.apache.commons.cli.Option;

 import org.apache.commons.cli.OptionBuilder;

 import org.apache.commons.cli.Options;

 import org.apache.commons.cli.ParseException;

 import org.apache.log4j.Logger;

 import ucar.ma2.DataType;

 import ucar.nc2.Attribute;

 import ucar.nc2.NetcdfFileWriteable;

 import ucar.nc2.Variable;

 import au.csiro.netcdf.cli.Command;

 import au.csiro.netcdf.cli.CommandLineOptionsComparator;

 import au.csiro.netcdf.util.NetCDFUtils;

 import au.csiro.netcdf.util.Util;

 /**

  * The <strong>ncdefineVar</strong> command defines a {@link Variable} object in a netCDF file.

  * <p>

  * Copyright 2010, CSIRO Australia 

  * All rights reserved.

  * 

  * @author $Author: robertbridle $ on 17/03/2010

  * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $ $Id: NcDefineVariable.java 6525

  *          2010-03-17 23:39:33Z che256 $

  */

 public class NcDefineVariable implements Command

 {

     /**

      * The command name

      */

     public static final String NC_DEFINE_VAR_COMMAND_NAME = "ncdefineVar";

     /**

      * The name of the command line option used for specifying the output netCDF file name.

      */

     public static final String OUTPUT_FILE = "outputFileName";

     /**

      * The name of the command line option used for specifying the input text file name.

      */

     public static final String INPUT_FILE = "inputFileName";

     /**

      * The name of the command line option used for specifying the input from standard input.

      */

     private static final String STANDARD_INPUT = "standardInput";    

     /**

      * The name of the command line option used for specifying the name of the variable.

      */

     public static final String VARIABLE_NAME = "variableName";

     /**

      * The name of the command line option used for specifying the data type of the variable.

      */

     public static final String VARIABLE_DATA_TYPE = "variableDataType";

     /**

      * The name of the command line option used for specifying the attributes of the variable.

      */

     public static final String VARIABLE_ATTRIBUTES = "variableAttributes";

     /**

      * The name of the command line option used for specifying the dimensions of the variable.

      */

     public static final String DIMENSION_NAMES = "dimensionNames";

     /**

      * Whether the netCDF file should be written with large file support, that is, 64-bit addressing for files greater

      * than 2 GB.

      */

     public static final String IS_LARGE_FILE = "largeFileSupport";

     /**

      * Ensure that any variable value not written will have the fill value, otherwise those values will be undefined:

      * possibly zero, or possibly garbage.

      */

     public static final String IS_FILL_VARIABLE = "fillVar";

     /**

      * Dummy variable name used to work around issue XXX with netcdf-java-4.0.41.jar

      */

     private static final String DUMMY_VARIABLE = "dummyVariable";

     /**

      * Constant that defines the logger to be used.

      */

     private static final Logger LOG = Logger.getLogger(NcDefineVariable.class.getName());

     /**

      * Command options

      */

     private Options options = null;

     /**

      * The only data types that can be used in netCDF v3 (classic) files.

      */

     private static Set<DataType> netCDF3DataTypes = new HashSet<DataType>();

     {

         netCDF3DataTypes.add(DataType.DOUBLE);

         netCDF3DataTypes.add(DataType.FLOAT);

         netCDF3DataTypes.add(DataType.INT);

         netCDF3DataTypes.add(DataType.CHAR);

         netCDF3DataTypes.add(DataType.SHORT);

         netCDF3DataTypes.add(DataType.BYTE);

     }

     /**

      * Constructor

      */

     public NcDefineVariable()

     {

         this.options = createOptions();

     }

     /*

      * (non-Javadoc)

      * 

      * @see au.csiro.netcdf.Command#execute(java.lang.String[])

      */

     @Override

     public void execute(String[] args) throws IllegalArgumentException, IOException, ParseException

     {

         // parse the command line arguments

         CommandLine parsedCommandLine = new BasicParser().parse(this.options, args);

         // get the command line argument values

         String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine

                 .getOptionValue(OUTPUT_FILE) : "";

         String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine

                 .getOptionValue(INPUT_FILE) : "";

         boolean isStandardInput = parsedCommandLine.hasOption(STANDARD_INPUT); 

         String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine

                 .getOptionValue(VARIABLE_NAME) : "";

         String variableDataTypeArg = (parsedCommandLine.hasOption(VARIABLE_DATA_TYPE)) ? parsedCommandLine

                 .getOptionValue(VARIABLE_DATA_TYPE) : "";

         String variableAttributesArg = (parsedCommandLine.hasOption(VARIABLE_ATTRIBUTES)) ? parsedCommandLine

                 .getOptionValue(VARIABLE_ATTRIBUTES) : "";

         String dimensionNamesArg = (parsedCommandLine.hasOption(DIMENSION_NAMES)) ? parsedCommandLine

                 .getOptionValue(DIMENSION_NAMES) : "";

         boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);

         boolean isFillVariable = parsedCommandLine.hasOption(IS_FILL_VARIABLE);

         // try getting the specified file

         File outputFile = null;

         try

         {

             outputFile = Util.getExistingFile(outputFilenameArg);

         }

         catch (IllegalArgumentException iae)

         {

             throw new IllegalArgumentException(NcDefineVariable.OUTPUT_FILE + " value refers to non-existent file: "

                     + outputFilenameArg);

         }

         // check whether large file support is needed

         if (Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE && !isLargeFileSupport)

         {

             throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE

                     + " flag.");

         }

         // check that if an input file is specified then it actually exists.

         if (!inputFilenameArg.isEmpty())

         {

             Util.getExistingFile(inputFilenameArg);

         }

         // try getting the dimension names

         String dimensionNamesString = NcDefineVariable.mapStringToDimensionNamesString(dimensionNamesArg);

         // try getting the variable's attributes from the command line

         List<Attribute> variableAttributes = new ArrayList<Attribute>(); // this is non-mandatory option, its default

                                                                          // value is an empty list.

         if (!variableAttributesArg.isEmpty())

         {

             try

             {

                 variableAttributes = NetCDFUtils.mapStringToAttributeValueList(variableAttributesArg);

             }

             catch (IllegalArgumentException iae)

             {

                 throw new IllegalArgumentException(NcDefineVariable.VARIABLE_ATTRIBUTES

                         + " value is not a comma separated String of attribute-value pairs: " + variableAttributesArg);

             }

         }

         // try getting the variable's data type

         DataType variableDataType = NcDefineVariable.mapStringToDataType(variableDataTypeArg);

         // try getting variable's attributes from a file or stdin.

         if (!inputFilenameArg.isEmpty())

         {

             variableAttributes.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(inputFilenameArg)));

         }

         if(isStandardInput)

         {

             variableAttributes.addAll(NetCDFUtils.readAttributesFromStream(System.in));

         }

         this.execute(outputFile, variableNameArg, variableDataType, variableAttributes, dimensionNamesString,

                 isLargeFileSupport, isFillVariable);

     }

     /**

      * Allows the command to be run programmatically, instead of from a command line.

      * 

      * @param outputFilename

      *            the netCDF file in which to define a dimension.

      * @param variableName

      *            the name of the variable, this value will be used to reference the dimension.

      * @param variableDataType

      *            the data type of the variable.

      * @param variableAttributes

      *            a attributes of the variable.

      * @param dimensionNamesString

      *            a whitespace separated list of dimension names, or '*' for Dimension.UNKNOWN. A <tt>null</tt> or empty

      *            String is a scalar.

      * @param isLargeFileSupport

      *            whether the netCDF file should be written with large file support, i.e. 64-bit addressing for files

      *            greater than 2 GB.

      * @param isFillVariable

      *            whether unwritten variable values should have the fill value.

      * @throws IOException

      *             thrown if netCDF can not be written to or read from.

      */

     public void execute(File outputFilename, String variableName, DataType variableDataType,

             List<Attribute> variableAttributes, String dimensionNamesString, boolean isLargeFileSupport,

             boolean isFillVariable) throws IOException

     {

         // the netcdf file to be written.

         NetcdfFileWriteable ncfile = null;

         ncfile = NetcdfFileWriteable.openExisting(outputFilename.getPath(), isFillVariable /*

                                                                                             * Setting fill = true causes

                                                                                             * everything to be written

                                                                                             * twice: first with the fill

                                                                                             * value, then with the data

                                                                                             * values. If you know you

                                                                                             * will write all the data,

                                                                                             * you dont need to use fill.

                                                                                             * If you don't know if all

                                                                                             * the data will be written,

                                                                                             * turning fill on ensures

                                                                                             * that any values not

                                                                                             * written will have the fill

                                                                                             * value. Otherwise those

                                                                                             * values will be undefined:

                                                                                             * possibly zero, or possibly

                                                                                             * garbage.

                                                                                             */);

         try

         {

             // We can not define any variables if the file does not contain any dimensions.

             if (ncfile.getDimensions().isEmpty())

             {

                 throw new IllegalStateException("No dimensions have been defined in the file: "

                         + outputFilename.getPath());

             }

             // The following if-block is used to work around issue XXX with the netcdf-java-4.0.41.jar

             // The issue occurs under the following circumstance:

             // - When trying to edit the header-info of a netCDF file that does not have any variables defined.

             // The issue is caused by:

             // - The netcdf java library parses the header-info of a netCDF file and only records a correct

             // "where to write data from" byte-offset if the header-info contains a variable.

             // To work around this issue we do the following:

             // - If the header-info contains no variables, we create a dummy variable and write the file out.

             // We then read the file back so that the "where to write data from" byte-offset is recorded correctly.

             // Note: we remove the dummy variable before writing the file out again.

             if (ncfile.getVariables().isEmpty())

             {

                 ncfile.setRedefineMode(true);

                 ncfile.setLargeFile(true);

                 Variable variable = new Variable(ncfile, null /* containing group */, null /* parent structure */,

                         DUMMY_VARIABLE);

                 variable.setDataType(DataType.DOUBLE);

                 variable.setCaching(false);

                 variable.setDimensions(ncfile.getDimensions().get(0).getName()); // if we get here, then there has to

                                                                                  // exist at least 1 dimension.

                 ncfile.addVariable(null, variable);

                 ncfile.setRedefineMode(false);

                 ncfile.close();

                 ncfile = NetcdfFileWriteable.openExisting(outputFilename.getPath(), isFillVariable /*

                                                                                                     * Setting fill =

                                                                                                     * true causes

                                                                                                     * everything to be

                                                                                                     * written twice:

                                                                                                     * first with the

                                                                                                     * fill value, then

                                                                                                     * with the data

                                                                                                     * values. If you

                                                                                                     * know you will

                                                                                                     * write all the

                                                                                                     * data, you dont

                                                                                                     * need to use fill.

                                                                                                     * If you don't know

                                                                                                     * if all the data

                                                                                                     * will be written,

                                                                                                     * turning fill on

                                                                                                     * ensures that any

                                                                                                     * values not written

                                                                                                     * will have the fill

                                                                                                     * value. Otherwise

                                                                                                     * those values will

                                                                                                     * be undefined:

                                                                                                     * possibly zero, or

                                                                                                     * possibly garbage.

                                                                                                     */);

                 ncfile.setRedefineMode(true);

                 ncfile.removeVariable(null, DUMMY_VARIABLE); // make sure we remove the dummy variable

             }

             else

             {

                 ncfile.setRedefineMode(true);

             }

             ncfile.setLargeFile(isLargeFileSupport);

             Variable variable = new Variable(ncfile, null /* containing group */, null /* parent structure */,

                     variableName);

             variable.setDataType(variableDataType);

             variable.setCaching(false);

             // add variable's dimensions

             variable.setDimensions(dimensionNamesString);

             // add variable's attributes

             for (Attribute attribute : variableAttributes)

             {

                 variable.addAttribute(attribute);

             }

             ncfile.addVariable(null, variable);

             // close editing header info

             ncfile.setRedefineMode(false);

         }

         finally

         {

             if(ncfile!=null)

             {

                 try

                 {

                     ncfile.close();

                 }

                 catch(Exception e)

                 {

                     LOG.error("Could not close file: " + outputFilename.getPath(), e);

                 }

             }

         }

     }

     /*

      * (non-Javadoc)

      * 

      * @see au.csiro.netcdf.Command#getCommandName()

      */

     @Override

     public String getCommandName()

     {

         return NcDefineVariable.NC_DEFINE_VAR_COMMAND_NAME;

     }

     /*

      * (non-Javadoc)

      * 

      * @see java.lang.Object#toString()

      */

     public String toString()

     {

         // generate the help/usage statement

         String header = "Define a variable in a netCDF file.";

         String footer = "\nExample: ncdefinevar -outputFileName ABC.nc -variableName Lat "

                 + "-variableDataType float -dimensionNames Lat\n"

                 + "Will add a variable called Lat to the file ABC.nc. The variable is linked to the "

                 + "dimension lat and thus is a 'corresponding' variable. The file must already "

                 + "exist and the dimension must be already defined in the file.";

         StringWriter sw = new StringWriter();

         HelpFormatter formatter = new HelpFormatter();

         formatter.setOptionComparator(new CommandLineOptionsComparator());

         formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NcDefineVariable.NC_DEFINE_VAR_COMMAND_NAME, header,

                 this.options, 0, 1, footer);

         return sw.toString();

     }

     /*

      * (non-Javadoc)

      * 

      * @see au.csiro.netcdf.Command#createOptions()

      */

     @SuppressWarnings("static-access")

     @Override

     public Options createOptions()

     {

         Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(

                 "1: the filename of the netCDF file to be modified.").isRequired(true).withLongOpt(OUTPUT_FILE).create(

                 "o");

         Option inputFileName = OptionBuilder

                 .withArgName("file")

                 .hasArg()

                 .withDescription(

                 "2. the filename of a text file containing attributes to be loaded. "

                         + "OPTIONAL, ensure that text containing '=' or ',' characters are delimited by a backslash.")

                 .isRequired(false).withLongOpt(INPUT_FILE).create("i");

         Option standardInput = OptionBuilder.withDescription(

                 "3: OPTIONAL, read attributes from Stdin, ensure that text containing '=' or ',' characters are delimited by a backslash.").isRequired(false)

                 .withLongOpt(STANDARD_INPUT).create("s");

         Option variableName = OptionBuilder.withArgName("text").hasArg().withDescription(

                 "4: the name to be given to the variable.").isRequired(true).withLongOpt(VARIABLE_NAME).create("v");

         Option variableType = OptionBuilder.withArgName("text").hasArg().withDescription(

                 "5: the data type of the variable, e.g. " + NcDefineVariable.netCDF3DataTypes).isRequired(true)

                 .withLongOpt(VARIABLE_DATA_TYPE).create("t");

         Option variableAttributes = OptionBuilder

                 .withArgName("text")

                 .hasArg()

                 .withDescription(

                 "6: a comma separated list of attribute-value pairs, OPTIONAL, e.g. \"units=mm\", ensure that text containing '=' or ',' characters are delimited by a backslash.")

                 .isRequired(false).withLongOpt(VARIABLE_ATTRIBUTES).create("a");

         Option dimensionNames = OptionBuilder.withArgName("text").hasArg().withDescription(

                 "7: OPTIONAL, a comma separated list of the variable's dimensions, e.g. date,latitude,longitude.")

                 .isRequired(false).withLongOpt(DIMENSION_NAMES).create("d");

         Option largeFileSupport = OptionBuilder.withDescription(

                 "8: OPTIONAL, set if more than 2 GB of data will need to be stored in this file.").isRequired(false)

                 .withLongOpt(IS_LARGE_FILE).create("l");

         Option fillVariable = OptionBuilder.withDescription(

                 "9: OPTIONAL, set if unwritten variable values should have the fill value.").isRequired(false)

                 .withLongOpt(IS_FILL_VARIABLE).create("f");

         Options options = new Options();

         options.addOption(outputFileName);

         options.addOption(inputFileName);

         options.addOption(standardInput);

         options.addOption(variableName);

         options.addOption(variableType);

         options.addOption(variableAttributes);

         options.addOption(dimensionNames);

         options.addOption(largeFileSupport);

         options.addOption(fillVariable);

         return options;

     }

     /**

      * @return the usage for this command.

      */

     public String getUsageString()

     {

         return this.toString();

     }

     /*

      * (non-Javadoc)

      * 

      * @see au.csiro.netcdf.cli.Command#validCommand(java.lang.String[])

      */

     public String validCommand(String[] commandLine)

     {

         String errorMsg = "";

         // validate the command line parameters

         try

         {

             // try parsing the command line, where ParseException can be thrown from

             CommandLine parsedCommandLine = new BasicParser().parse(this.options, commandLine);

             // try converting command line parameters to their associated types, where IllegalArgumentException can be

             // thrown from

             String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine

                     .getOptionValue(OUTPUT_FILE) : "";

             String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine

                     .getOptionValue(INPUT_FILE) : "";                      

             String variableDataTypeArg = (parsedCommandLine.hasOption(VARIABLE_DATA_TYPE)) ? parsedCommandLine

                     .getOptionValue(VARIABLE_DATA_TYPE) : "";

             String variableAttributesArg = (parsedCommandLine.hasOption(VARIABLE_ATTRIBUTES)) ? parsedCommandLine

                     .getOptionValue(VARIABLE_ATTRIBUTES) : "";

             String dimensionNamesArg = (parsedCommandLine.hasOption(DIMENSION_NAMES)) ? parsedCommandLine

                     .getOptionValue(DIMENSION_NAMES) : "";

             boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);                    

             // try getting the specified file

             try

             {

                 Util.getExistingFile(outputFilenameArg);

             }

             catch (IllegalArgumentException iae)

             {

                 throw new IllegalArgumentException(NcDefineVariable.OUTPUT_FILE

                         + " value refers to non-existent file: " + outputFilenameArg);

             }

             // check whether large file support is needed

             if (Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE && !isLargeFileSupport)

             {

                 throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE

                         + " flag.");

             }

             // check that if an input file is specified then it actually exists.

             if (!inputFilenameArg.isEmpty())

             {

                 Util.getExistingFile(inputFilenameArg);

             }

             // try getting the dimension names

             NcDefineVariable.mapStringToDimensionNamesString(dimensionNamesArg);

             // try getting the variable's attributes from the command line.  

             if (!variableAttributesArg.isEmpty())

             {

                 try

                 {

                     NetCDFUtils.mapStringToAttributeValueList(variableAttributesArg);

                 }

                 catch (IllegalArgumentException iae)

                 {

                     throw new IllegalArgumentException(NcDefineVariable.VARIABLE_ATTRIBUTES

                             + " value is not a comma separated String of attribute-value pairs: "

                             + variableAttributesArg);

                 }

             }

             // try getting the variable's data type

             NcDefineVariable.mapStringToDataType(variableDataTypeArg);

         }

         catch (ParseException pe)

         {

             errorMsg = errorMsg + "\n" + pe.getMessage();

         }

         catch (IllegalArgumentException iae)

         {

             errorMsg = errorMsg + "\n" + iae.getMessage();

         }

         return errorMsg;

     }

     /**

      * Maps a <code>String</code> into a whitespace separated list of dimension names.

      * 

      * @param dimensionNamesArg

      *            a list of comma separated dimension descriptions, e.g. date,longitude,latitude,...

      * @return whitespace separated list of dimension names, or '*' for Dimension.UNKNOWN. A <tt>null</tt> or empty

      *         String is a scalar.

      * @throws IllegalArgumentException

      *             thrown if the <code>String</code> can not be mapped into a whitespace separated list of dimension

      *             names.

      */

     private static String mapStringToDimensionNamesString(String dimensionNamesArg) throws IllegalArgumentException

     {

         try

         {

             List<String> dimensionNames = Util.tokeniseCommaSeparatedString(dimensionNamesArg);

             StringBuffer strBuf = new StringBuffer();

             for (String dimensionName : dimensionNames)

             {

                 strBuf.append(dimensionName + " ");

             }

             return strBuf.toString();

         }

         catch (PatternSyntaxException pse)

         {

             throw new IllegalArgumentException(NcDefineVariable.DIMENSION_NAMES

                     + " value is not a comma separated String: " + dimensionNamesArg);

         }

     }

     /**

      * Maps a <code>String</code> into a {@link DataType}.

      * 

      * @param variableDataType

      *            a data type description.

      * @return a {@link DataType}

      * @throws IllegalArgumentException

      *             thrown if the <code>String</code> can not be mapped to a {@link DataType}.

      */

     private static DataType mapStringToDataType(String variableDataType) throws IllegalArgumentException

     {

         DataType dataType = DataType.getType(variableDataType);

         if (dataType != null && NcDefineVariable.netCDF3DataTypes.contains(dataType))

         {

             return dataType;

         }

         throw new IllegalArgumentException(NcDefineVariable.VARIABLE_DATA_TYPE + " value is not a valid data type: "

                 + variableDataType + ". Allowed data types are: " + NcDefineVariable.netCDF3DataTypes);

     }

     /**

      * @return the netCDF3DataTypes

      */

     public static Set<DataType> getNetCDF3DataTypes()

     {

         return netCDF3DataTypes;

     }

     /**

      * @param netCDF3DataTypes the netCDF3DataTypes to set

      */

     public static void setNetCDF3DataTypes(Set<DataType> netCDF3DataTypes)

     {

         NcDefineVariable.netCDF3DataTypes = netCDF3DataTypes;

     }

 }