View Javadoc

1   /**
2    * Copyright 2010, CSIRO Australia.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  /**
18   * 
19   */
20  package au.csiro.netcdf;
21  
22  import java.io.File;
23  import java.io.FileInputStream;
24  import java.io.IOException;
25  import java.io.PrintWriter;
26  import java.io.StringWriter;
27  import java.util.ArrayList;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Set;
31  
32  import org.apache.commons.cli.BasicParser;
33  import org.apache.commons.cli.CommandLine;
34  import org.apache.commons.cli.HelpFormatter;
35  import org.apache.commons.cli.Option;
36  import org.apache.commons.cli.OptionBuilder;
37  import org.apache.commons.cli.Options;
38  import org.apache.commons.cli.ParseException;
39  import org.apache.log4j.Logger;
40  
41  import ucar.ma2.DataType;
42  import ucar.nc2.Attribute;
43  import ucar.nc2.NetcdfFileWriteable;
44  import ucar.nc2.Variable;
45  import au.csiro.netcdf.cli.Command;
46  import au.csiro.netcdf.cli.CommandLineOptionsComparator;
47  import au.csiro.netcdf.util.NetCDFUtils;
48  import au.csiro.netcdf.util.Util;
49  
50  /**
51   * The <strong>ncdefineAtt</strong> command defines global or variable {@link Attribute}s in a netCDF file.
52   * <p>
53   * Copyright 2010, CSIRO Australia 
54   * All rights reserved.
55   * 
56   * @author $Author: robertbridle $ on 17/03/2010
57   * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $ $Id: NcDefineDimension.java 6520
58   *          2010-03-17 06:17:50Z bri26e $
59   */
60  public class NcDefineAttributes implements Command
61  {
62      private static final String NULL_VALUE = "null";
63  
64      /**
65       * The command name
66       */
67      public static final String NC_DEFINE_ATT_COMMAND_NAME = "ncdefineAtt";
68  
69      /**
70       * The name of the command line option used for specifying the output netCDF file name.
71       */
72      public static final String OUTPUT_FILE = "outputFileName";
73      
74      /**
75       * The name of the command line option used for specifying the input text file name.
76       */
77      public static final String INPUT_FILE = "inputFileName";
78      
79      /**
80       * The name of the command line option used for specifying the input from standard input.
81       */
82      private static final String STANDARD_INPUT = "standardInput";
83      
84      /**
85       * The name of the command line option used for specifying the attributes of a netCDF file.
86       */
87      public static final String ATTRIBUTES = "attributes";
88      
89      /**
90       * The name of the command line option used for specifying the name of the variable receiving the attributes.
91       */
92      public static final String VARIABLE_NAME = "variable";
93      
94      /**
95       * Whether the netCDF file should be written with large file support, that is, 64-bit addressing for files greater
96       * than 2 GB.
97       */
98      public static final String IS_LARGE_FILE = "largeFileSupport";
99  
100     /**
101      * The pattern for accepted file names where multiple files are accepted.
102      */
103     public static final String FILENAME_PATTERN = "pattern"; 
104 
105     /**
106      * The name of the command line option used for specifying the data type of the attribute.
107      */
108     public static final String ATTRIBUTE_DATA_TYPE = "attributeDataType";
109     
110     /**
111      * Constant that defines the logger to be used.
112      */
113     private static final Logger LOG = Logger.getLogger(NcDefineAttributes.class.getName());
114 
115 
116     /**
117      * The only data types that can be used for attributes in netCDF v3 (classic) files.
118      */
119     private static Set<DataType> netCDF3DataTypes = new HashSet<DataType>();
120     
121     static 
122     {
123         netCDF3DataTypes.add(DataType.DOUBLE);
124         netCDF3DataTypes.add(DataType.FLOAT);
125         netCDF3DataTypes.add(DataType.INT);
126         netCDF3DataTypes.add(DataType.CHAR);
127         netCDF3DataTypes.add(DataType.SHORT);
128         netCDF3DataTypes.add(DataType.BYTE);
129         netCDF3DataTypes.add(DataType.STRING);
130     }
131     
132     /**
133      * Command options
134      */
135     private Options options = null;
136 
137     /**
138      * Constructor
139      */
140     public NcDefineAttributes()
141     {
142         this.options = createOptions();
143     }
144 
145     /*
146      * (non-Javadoc)
147      * 
148      * @see au.csiro.netcdf.Command#execute(java.lang.String[])
149      */
150     @Override
151     public void execute(String[] args) throws ParseException, IOException, IllegalArgumentException
152     {
153         // parse the command line arguments
154         CommandLine parsedCommandLine = new BasicParser().parse(this.options, args);
155 
156         // get the command line argument values
157         String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
158                 .getOptionValue(OUTPUT_FILE) : "";
159         String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
160                 .getOptionValue(INPUT_FILE) : "";
161         boolean isStandardInput = parsedCommandLine.hasOption(STANDARD_INPUT);                
162         String attributesArg = (parsedCommandLine.hasOption(ATTRIBUTES)) ? parsedCommandLine
163                 .getOptionValue(ATTRIBUTES) : "";
164         boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);  
165         String pattern = (parsedCommandLine.hasOption(FILENAME_PATTERN)) ? parsedCommandLine
166                 .getOptionValue(FILENAME_PATTERN) : "";
167         String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine
168                         .getOptionValue(VARIABLE_NAME) : "";
169         String attributeDataTypeArg = (parsedCommandLine.hasOption(ATTRIBUTE_DATA_TYPE)) ? parsedCommandLine
170                 .getOptionValue(ATTRIBUTE_DATA_TYPE) : "String";
171 
172         List<String> targetFileNames = Util.getListOfTargetFiles(outputFilenameArg, pattern);
173                 
174         // check whether large file support is needed
175         for (String outputFileName : targetFileNames)
176         {
177             if (Util.fileExists(outputFileName)
178                     && Util.getExistingFile(outputFileName).length() >= MAX_32BIT_OFFSET_FILE_SIZE
179                     && !isLargeFileSupport)
180             {
181                 throw new IllegalArgumentException("The netCDF file " + outputFileName
182                         + " will be too large, please use " + IS_LARGE_FILE + " flag.");
183             }
184         }
185         
186         
187         // check that if an input file is specified then it actually exists.
188         if (!inputFilenameArg.isEmpty())
189         {
190             Util.getExistingFile(inputFilenameArg);
191         }
192         
193         // try getting attributes from the command line
194         List<Attribute> attributes = new ArrayList<Attribute>(); // this is non-mandatory option, its default
195                                                                        // value is an empty list.                
196         if(!attributesArg.isEmpty())
197         {
198             // try getting the variable's data type
199             DataType attributeDataType = NcDefineAttributes.mapStringToDataType(attributeDataTypeArg);
200 
201             try
202             {
203                 attributes = NetCDFUtils.mapStringToAttributeValueList(attributesArg, attributeDataType);
204             }
205             catch (IllegalArgumentException iae)
206             {
207                 throw new IllegalArgumentException(NcDefineAttributes.ATTRIBUTES
208                         + " value is not a comma separated String of attribute-value pairs: " + attributesArg);
209             }
210         }          
211 
212         // try getting variable's attributes from a file or stdin.
213         if (!inputFilenameArg.isEmpty())
214         {
215             attributes.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(inputFilenameArg)));
216         }
217         if(isStandardInput)
218         {
219             attributes.addAll(NetCDFUtils.readAttributesFromStream(System.in));
220         }
221         
222         for (String outputFileName : targetFileNames)
223         {
224             LOG.debug("Processing file : " + outputFileName);
225             this.execute(outputFileName, attributes, isLargeFileSupport, variableNameArg);
226         }
227     }
228     
229     /**
230      * Allows the command to be run programmatically, instead of from a command line.
231      * 
232      * @param outputFilename
233      *            the netCDF file in which to define a dimension.
234      * @param attributes
235      *            a attributes of the file.
236      * @param isLargeFileSupport
237      *            whether the netCDF file should be written with large file support, i.e. 64-bit addressing for files
238      *            greater than 2 GB.
239      * @throws IOException
240      *             thrown if netCDF can to be written to or read from.
241      * @throws SecurityException
242      *             thrown if a security manager exists and it prevents the netCDF file from being created.
243      */
244     public void execute(String outputFilename, List<Attribute> attributes, boolean isLargeFileSupport) throws IOException, SecurityException
245     {
246         this.execute(outputFilename, attributes, isLargeFileSupport, "");
247     }
248     
249     /**
250      * Allows the command to be run programmatically, instead of from a command line.
251      * 
252      * @param outputFilename
253      *            the netCDF file in which to define a dimension.
254      * @param attributes
255      *            a attributes of the file.
256      * @param isLargeFileSupport
257      *            whether the netCDF file should be written with large file support, i.e. 64-bit addressing for files
258      *            greater than 2 GB.
259      * @param variableName
260      *            the variable name to assign attributes to
261      * @throws IOException
262      *             thrown if netCDF can to be written to or read from.
263      * @throws SecurityException
264      *             thrown if a security manager exists and it prevents the netCDF file from being created.
265      */
266     public void execute(String outputFilename, List<Attribute> attributes, 
267             boolean isLargeFileSupport, String variableName) throws IOException, SecurityException
268     {
269         // the netcdf file to be written.
270         NetcdfFileWriteable ncfile = null;
271 
272         // does output file exist
273         File outputFile = new File(outputFilename);
274         if (outputFile.exists())
275         {
276             ncfile = NetcdfFileWriteable.openExisting(outputFilename, false /*
277                                                                             * Setting fill = true causes everything to
278                                                                             * be written twice: first with the fill
279                                                                             * value, then with the data values. If you
280                                                                             * know you will write all the data, you dont
281                                                                             * need to use fill. If you don't know if all
282                                                                             * the data will be written, turning fill on
283                                                                             * ensures that any values not written will
284                                                                             * have the fill value. Otherwise those
285                                                                             * values will be undefined: possibly zero,
286                                                                             * or possibly garbage.
287                                                                             */);
288             try
289             {
290                 // allow editing header info
291                 ncfile.setRedefineMode(true);
292                 ncfile.setLargeFile(isLargeFileSupport);
293 
294                 // add attributes to file
295                 if (variableName.isEmpty())
296                 {
297                     for(Attribute attribute : attributes)
298                     {
299                         // remove Attribute if the value is "null"                       
300                         if (NULL_VALUE.equalsIgnoreCase(attribute.getStringValue()))
301                         {
302                             ncfile.deleteGlobalAttribute(attribute.getName());
303                         }
304                         else
305                         {
306                             ncfile.addAttribute(null, attribute);
307                         }
308                     }
309                 }
310                 
311                 // add attributes to variable
312                 else
313                 {
314                     Variable variable = ncfile.findVariable(variableName);
315                     if (variable == null)
316                     {
317                         throw new IllegalArgumentException("The variable " +  
318                                 variableName + " doesn't exist in file " + outputFilename + ".");
319                     }
320                     else 
321                     {
322                         for(Attribute attribute : attributes)
323                         {
324                             // remove Attribute if the value is "null"                       
325                             if (NULL_VALUE.equalsIgnoreCase(attribute.getStringValue()))
326                             {
327                                 variable.removeAttributeIgnoreCase(attribute.getName());
328                             }
329                             else
330                             {
331                                 variable.addAttribute(attribute);
332                             }
333                         }
334                     }
335                 }
336                 
337                 // close editing header info
338                 ncfile.setRedefineMode(false);
339             }
340             finally
341             {
342                 if(ncfile!=null)
343                 {
344                     try
345                     {
346                         ncfile.close();
347                     }
348                     catch(Exception e)
349                     {
350                         LOG.error("Could not close file: " + outputFilename, e);
351                     }
352                 }
353             }
354         }
355         else
356         {
357             File parentFile = outputFile.getParentFile();
358             if (parentFile != null)
359             {
360                 parentFile.mkdirs(); // create the file's parent folder structure
361             }
362             ncfile = NetcdfFileWriteable.createNew(outputFilename, false /*
363                                                                          * Setting fill = true causes everything to be
364                                                                          * written twice: first with the fill value,
365                                                                          * then with the data values. If you know you
366                                                                          * will write all the data, you dont need to use
367                                                                          * fill. If you don't know if all the data will
368                                                                          * be written, turning fill on ensures that any
369                                                                          * values not written will have the fill value.
370                                                                          * Otherwise those values will be undefined:
371                                                                          * possibly zero, or possibly garbage.
372                                                                          */);
373             try
374             {
375                 // add attributes
376                 for(Attribute attribute : attributes)
377                 {
378                     ncfile.addAttribute(null, attribute);
379                 }
380 
381                 // when all objects are added, call create(). You cannot read or write data until create() is called.
382                 ncfile.create();
383             }
384             finally
385             {
386                 if(ncfile!=null)
387                 {
388                     try
389                     {
390                         ncfile.close();
391                     }
392                     catch(Exception e)
393                     {
394                         LOG.error("Could not close file: " + outputFilename, e);
395                     }
396                 }
397             }
398         }
399     }
400 
401     /*
402      * (non-Javadoc)
403      * 
404      * @see au.csiro.netcdf.Command#getCommandName()
405      */
406     @Override
407     public String getCommandName()
408     {
409         return NcDefineAttributes.NC_DEFINE_ATT_COMMAND_NAME;
410     }
411 
412     /*
413      * (non-Javadoc)
414      * 
415      * @see java.lang.Object#toString()
416      */
417     public String toString()
418     {
419         // generate the help/usage statement 
420         String header = "Define an attribute in a netCDF file.";
421         String footer = "\nExample: ncdefineatt -outputFileName ABC.nc "
422                 + "-attributes \"netCDF-Java=4.0,Conventions=value with a space,toDelete=null\"\n"
423                 + "Will add the two and delete one listed global attributes to the file ABC.nc. "
424                 + "The file will be created if it doesn't already exist. ";
425         StringWriter sw = new StringWriter();
426         HelpFormatter formatter = new HelpFormatter();
427         formatter.setOptionComparator(new CommandLineOptionsComparator());
428         formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NcDefineAttributes.NC_DEFINE_ATT_COMMAND_NAME, header,
429                 this.options, 0, 1, footer);
430         return sw.toString();
431     }
432 
433     /*
434      * (non-Javadoc)
435      * 
436      * @see au.csiro.netcdf.Command#createOptions()
437      */
438     @SuppressWarnings("static-access")
439     @Override
440     public Options createOptions()
441     {
442         Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(
443                 "1: the filename of the netCDF file to be created.").isRequired(true).withLongOpt(OUTPUT_FILE).create(
444                 "o");
445         
446         Option inputFileName = OptionBuilder
447                 .withArgName("file")
448                 .hasArg()
449                 .withDescription(
450                 "2. the filename of a text file containing attributes to be loaded. "
451                         + "OPTIONAL, ensure that text containing '=' or ',' characters are delimited by a backslash.")
452                 .isRequired(false).withLongOpt(INPUT_FILE).create("i");
453         
454         Option standardInput = OptionBuilder.withDescription(
455                 "3: OPTIONAL, read attributes from Stdin, ensure that text containing '=' or ',' characters are delimited by a backslash.").isRequired(false)
456                 .withLongOpt(STANDARD_INPUT).create("s");
457 
458         Option attributes = OptionBuilder
459                 .withArgName("text")
460                 .hasArg()
461                 .withDescription(
462                 "4: a comma separated list of attribute-value pairs, OPTIONAL, e.g. \"netCDF Java=4.0,Conventions=CF-1.0\", "
463                         + "ensure that text containing '=' or ',' characters are delimited by a backslash. "
464                         + "Set value to null to delete existing attribute.")
465                 .isRequired(false).withLongOpt(ATTRIBUTES).create("a");
466         
467         Option largeFileSupport = OptionBuilder.withDescription(
468                 "5: OPTIONAL, set if more than 2 GB of data will need to be stored in this file.").isRequired(false)
469                 .withLongOpt(IS_LARGE_FILE).create("l");
470 
471         Option pattern = OptionBuilder.withArgName("filename pattern").hasArg().withDescription(
472                 "6: OPTIONAL, a pattern to match multiple existing files, where multiple files should be "
473                         + "processed. If this option is present the outputFilename is expected to be a directory "
474                         + "containing the files to be processed. The wildcard characters * and ? are supported but "
475                         + "may need to be escaped by a \\ to avoid processing by the shell. e.g. \\*.nc ").isRequired(
476                 false).withLongOpt(FILENAME_PATTERN).create("p");
477         Option variableName = OptionBuilder
478                 .withArgName("text")
479                 .hasArg()
480                 .withDescription(
481                 "7: OPTIONAL, the variable to assign attribute(s) to, "
482                         + "set to define variable attribute.") 
483                 .isRequired(false).withLongOpt(VARIABLE_NAME).create("v");     
484 
485         Option attributeType = OptionBuilder.withArgName("text").hasArg().withDescription(
486                 "8: OPTIONAL, the data type of the attribute(s), e.g. " + NcDefineAttributes.netCDF3DataTypes)
487                 .isRequired(false).withLongOpt(ATTRIBUTE_DATA_TYPE).create("t");
488         
489         Options options = new Options();
490 
491         options.addOption(outputFileName);
492         options.addOption(inputFileName);
493         options.addOption(standardInput);
494         options.addOption(attributes);
495         options.addOption(largeFileSupport);
496         options.addOption(pattern);
497         options.addOption(variableName);
498         options.addOption(attributeType);
499 
500         return options;
501     }
502 
503     /**
504      * @return the usage for this command.
505      */
506     public String getUsageString()
507     {
508         return this.toString();
509     }
510 
511     /**
512      * Determine if this command is valid, check its syntax.
513      * 
514      * @param commandLine
515      *            command line arguments.
516      * @return error if found.
517      */
518     public String validCommand(String[] commandLine)
519     {
520         String errorMsg = "";
521 
522         // validate the command line parameters
523         try
524         {
525             // try parsing the command line, where ParseException can be thrown from
526             CommandLine parsedCommandLine = new BasicParser().parse(this.options, commandLine);
527             
528             // get the command line argument values
529             String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
530                     .getOptionValue(OUTPUT_FILE) : ""; 
531             String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
532                     .getOptionValue(INPUT_FILE) : "";                   
533             String attributesArg = (parsedCommandLine.hasOption(ATTRIBUTES)) ? parsedCommandLine
534                     .getOptionValue(ATTRIBUTES) : "";
535             boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);  
536             String pattern = (parsedCommandLine.hasOption(FILENAME_PATTERN)) ? parsedCommandLine
537                     .getOptionValue(FILENAME_PATTERN) : "";
538             String variableDataTypeArg = (parsedCommandLine.hasOption(ATTRIBUTE_DATA_TYPE)) ? parsedCommandLine
539                     .getOptionValue(ATTRIBUTE_DATA_TYPE) : "String";
540             
541             // check whether large file support is needed
542             if (pattern.length() == 0 && Util.fileExists(outputFilenameArg) 
543                     && Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE
544                     && !isLargeFileSupport)
545             {
546                 throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE
547                         + " flag.");
548             }
549             
550             // Check that if a pattern is present that the output file is a directory
551             if (pattern.length() > 0)
552             {
553                 if (!Util.fileExists(outputFilenameArg))
554                 {
555                     throw new IllegalArgumentException("When a pattern is specified " +
556                     		"the output file must be an existing directory.");
557                 }
558                 File file = new File(outputFilenameArg);
559                 if (!file.isDirectory())
560                 {
561                     throw new IllegalArgumentException("When a pattern is specified "
562                             + "the output file must be an existing directory.");
563                 }
564             }
565 
566             // check that if an input file is specified then it actually exists.
567             if (!inputFilenameArg.isEmpty())
568             {
569                 Util.getExistingFile(inputFilenameArg);
570             }
571             
572             // try getting attributes from the command line.     
573             if(!attributesArg.isEmpty())
574             {
575                 try
576                 {
577                     NetCDFUtils.mapStringToAttributeValueList(attributesArg);
578                 }
579                 catch (IllegalArgumentException iae)
580                 {
581                     throw new IllegalArgumentException(NcDefineAttributes.ATTRIBUTES
582                             + " value is not a comma separated String of attribute-value pairs: "
583                             + attributesArg);
584                 }
585             }
586             
587 
588             // try getting the variable's data type
589             NcDefineAttributes.mapStringToDataType(variableDataTypeArg);
590             
591         }
592         catch (ParseException pe)
593         {
594             errorMsg = errorMsg + "\n" + pe.getMessage();
595         }
596         catch (IllegalArgumentException iae)
597         {
598             errorMsg = errorMsg + "\n" + iae.getMessage();
599         }
600 
601         return errorMsg;
602     }
603 
604     /**
605      * Maps a <code>String</code> into a {@link DataType}.
606      * 
607      * @param variableDataType
608      *            a data type description.
609      * @return a {@link DataType}
610      * @throws IllegalArgumentException
611      *             thrown if the <code>String</code> can not be mapped to a {@link DataType}.
612      */
613     public static DataType mapStringToDataType(String variableDataType) throws IllegalArgumentException
614     {
615         DataType dataType = DataType.getType(variableDataType);
616         if (dataType != null && netCDF3DataTypes.contains(dataType))
617         {
618             return dataType;
619         }
620         throw new IllegalArgumentException(NcDefineVariable.VARIABLE_DATA_TYPE + " value is not a valid data type: "
621                 + variableDataType + ". Allowed data types are: " + netCDF3DataTypes);
622     }
623 }