1 /**
2 * Copyright 2010, CSIRO Australia.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 /**
18 *
19 */
20 package au.csiro.netcdf;
21
22 import java.io.File;
23 import java.io.FileInputStream;
24 import java.io.IOException;
25 import java.io.PrintWriter;
26 import java.io.StringWriter;
27 import java.util.ArrayList;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Set;
31 import java.util.regex.PatternSyntaxException;
32
33 import org.apache.commons.cli.BasicParser;
34 import org.apache.commons.cli.CommandLine;
35 import org.apache.commons.cli.HelpFormatter;
36 import org.apache.commons.cli.Option;
37 import org.apache.commons.cli.OptionBuilder;
38 import org.apache.commons.cli.Options;
39 import org.apache.commons.cli.ParseException;
40 import org.apache.log4j.Logger;
41
42 import ucar.ma2.DataType;
43 import ucar.nc2.Attribute;
44 import ucar.nc2.NetcdfFileWriteable;
45 import ucar.nc2.Variable;
46 import au.csiro.netcdf.cli.Command;
47 import au.csiro.netcdf.cli.CommandLineOptionsComparator;
48 import au.csiro.netcdf.util.NetCDFUtils;
49 import au.csiro.netcdf.util.Util;
50
51 /**
52 * The <strong>ncdefineVar</strong> command defines a {@link Variable} object in a netCDF file.
53 * <p>
54 * Copyright 2010, CSIRO Australia
55 * All rights reserved.
56 *
57 * @author $Author: robertbridle $ on 17/03/2010
58 * @version $Revision: 84 $ $Date: 2010-08-25 15:56:46 +1000 (Wed, 25 Aug 2010) $ $Id: NcDefineVariable.java 6525
59 * 2010-03-17 23:39:33Z che256 $
60 */
61 public class NcDefineVariable implements Command
62 {
63 /**
64 * The command name
65 */
66 public static final String NC_DEFINE_VAR_COMMAND_NAME = "ncdefineVar";
67
68 /**
69 * The name of the command line option used for specifying the output netCDF file name.
70 */
71 public static final String OUTPUT_FILE = "outputFileName";
72
73 /**
74 * The name of the command line option used for specifying the input text file name.
75 */
76 public static final String INPUT_FILE = "inputFileName";
77
78 /**
79 * The name of the command line option used for specifying the input from standard input.
80 */
81 private static final String STANDARD_INPUT = "standardInput";
82
83 /**
84 * The name of the command line option used for specifying the name of the variable.
85 */
86 public static final String VARIABLE_NAME = "variableName";
87
88 /**
89 * The name of the command line option used for specifying the data type of the variable.
90 */
91 public static final String VARIABLE_DATA_TYPE = "variableDataType";
92
93 /**
94 * The name of the command line option used for specifying the attributes of the variable.
95 */
96 public static final String VARIABLE_ATTRIBUTES = "variableAttributes";
97
98 /**
99 * The name of the command line option used for specifying the dimensions of the variable.
100 */
101 public static final String DIMENSION_NAMES = "dimensionNames";
102
103 /**
104 * Whether the netCDF file should be written with large file support, that is, 64-bit addressing for files greater
105 * than 2 GB.
106 */
107 public static final String IS_LARGE_FILE = "largeFileSupport";
108
109 /**
110 * Ensure that any variable value not written will have the fill value, otherwise those values will be undefined:
111 * possibly zero, or possibly garbage.
112 */
113 public static final String IS_FILL_VARIABLE = "fillVar";
114
115 /**
116 * Dummy variable name used to work around issue XXX with netcdf-java-4.0.41.jar
117 */
118 private static final String DUMMY_VARIABLE = "dummyVariable";
119
120 /**
121 * Constant that defines the logger to be used.
122 */
123 private static final Logger LOG = Logger.getLogger(NcDefineVariable.class.getName());
124
125 /**
126 * Command options
127 */
128 private Options options = null;
129
130 /**
131 * The only data types that can be used in netCDF v3 (classic) files.
132 */
133 private static Set<DataType> netCDF3DataTypes = new HashSet<DataType>();
134 {
135 netCDF3DataTypes.add(DataType.DOUBLE);
136 netCDF3DataTypes.add(DataType.FLOAT);
137 netCDF3DataTypes.add(DataType.INT);
138 netCDF3DataTypes.add(DataType.CHAR);
139 netCDF3DataTypes.add(DataType.SHORT);
140 netCDF3DataTypes.add(DataType.BYTE);
141 }
142
143 /**
144 * Constructor
145 */
146 public NcDefineVariable()
147 {
148 this.options = createOptions();
149 }
150
151 /*
152 * (non-Javadoc)
153 *
154 * @see au.csiro.netcdf.Command#execute(java.lang.String[])
155 */
156 @Override
157 public void execute(String[] args) throws IllegalArgumentException, IOException, ParseException
158 {
159 // parse the command line arguments
160 CommandLine parsedCommandLine = new BasicParser().parse(this.options, args);
161
162 // get the command line argument values
163 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
164 .getOptionValue(OUTPUT_FILE) : "";
165 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
166 .getOptionValue(INPUT_FILE) : "";
167 boolean isStandardInput = parsedCommandLine.hasOption(STANDARD_INPUT);
168 String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine
169 .getOptionValue(VARIABLE_NAME) : "";
170 String variableDataTypeArg = (parsedCommandLine.hasOption(VARIABLE_DATA_TYPE)) ? parsedCommandLine
171 .getOptionValue(VARIABLE_DATA_TYPE) : "";
172 String variableAttributesArg = (parsedCommandLine.hasOption(VARIABLE_ATTRIBUTES)) ? parsedCommandLine
173 .getOptionValue(VARIABLE_ATTRIBUTES) : "";
174 String dimensionNamesArg = (parsedCommandLine.hasOption(DIMENSION_NAMES)) ? parsedCommandLine
175 .getOptionValue(DIMENSION_NAMES) : "";
176 boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);
177 boolean isFillVariable = parsedCommandLine.hasOption(IS_FILL_VARIABLE);
178
179 // try getting the specified file
180 File outputFile = null;
181 try
182 {
183 outputFile = Util.getExistingFile(outputFilenameArg);
184 }
185 catch (IllegalArgumentException iae)
186 {
187 throw new IllegalArgumentException(NcDefineVariable.OUTPUT_FILE + " value refers to non-existent file: "
188 + outputFilenameArg);
189 }
190
191 // check whether large file support is needed
192 if (Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE && !isLargeFileSupport)
193 {
194 throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE
195 + " flag.");
196 }
197
198 // check that if an input file is specified then it actually exists.
199 if (!inputFilenameArg.isEmpty())
200 {
201 Util.getExistingFile(inputFilenameArg);
202 }
203
204 // try getting the dimension names
205 String dimensionNamesString = NcDefineVariable.mapStringToDimensionNamesString(dimensionNamesArg);
206
207 // try getting the variable's attributes from the command line
208 List<Attribute> variableAttributes = new ArrayList<Attribute>(); // this is non-mandatory option, its default
209 // value is an empty list.
210 if (!variableAttributesArg.isEmpty())
211 {
212 try
213 {
214 variableAttributes = NetCDFUtils.mapStringToAttributeValueList(variableAttributesArg);
215 }
216 catch (IllegalArgumentException iae)
217 {
218 throw new IllegalArgumentException(NcDefineVariable.VARIABLE_ATTRIBUTES
219 + " value is not a comma separated String of attribute-value pairs: " + variableAttributesArg);
220 }
221 }
222
223 // try getting the variable's data type
224 DataType variableDataType = NcDefineVariable.mapStringToDataType(variableDataTypeArg);
225
226 // try getting variable's attributes from a file or stdin.
227 if (!inputFilenameArg.isEmpty())
228 {
229 variableAttributes.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(inputFilenameArg)));
230 }
231 if(isStandardInput)
232 {
233 variableAttributes.addAll(NetCDFUtils.readAttributesFromStream(System.in));
234 }
235 this.execute(outputFile, variableNameArg, variableDataType, variableAttributes, dimensionNamesString,
236 isLargeFileSupport, isFillVariable);
237 }
238
239 /**
240 * Allows the command to be run programmatically, instead of from a command line.
241 *
242 * @param outputFilename
243 * the netCDF file in which to define a dimension.
244 * @param variableName
245 * the name of the variable, this value will be used to reference the dimension.
246 * @param variableDataType
247 * the data type of the variable.
248 * @param variableAttributes
249 * a attributes of the variable.
250 * @param dimensionNamesString
251 * a whitespace separated list of dimension names, or '*' for Dimension.UNKNOWN. A <tt>null</tt> or empty
252 * String is a scalar.
253 * @param isLargeFileSupport
254 * whether the netCDF file should be written with large file support, i.e. 64-bit addressing for files
255 * greater than 2 GB.
256 * @param isFillVariable
257 * whether unwritten variable values should have the fill value.
258 * @throws IOException
259 * thrown if netCDF can not be written to or read from.
260 */
261 public void execute(File outputFilename, String variableName, DataType variableDataType,
262 List<Attribute> variableAttributes, String dimensionNamesString, boolean isLargeFileSupport,
263 boolean isFillVariable) throws IOException
264 {
265 // the netcdf file to be written.
266 NetcdfFileWriteable ncfile = null;
267
268 ncfile = NetcdfFileWriteable.openExisting(outputFilename.getPath(), isFillVariable /*
269 * Setting fill = true causes
270 * everything to be written
271 * twice: first with the fill
272 * value, then with the data
273 * values. If you know you
274 * will write all the data,
275 * you dont need to use fill.
276 * If you don't know if all
277 * the data will be written,
278 * turning fill on ensures
279 * that any values not
280 * written will have the fill
281 * value. Otherwise those
282 * values will be undefined:
283 * possibly zero, or possibly
284 * garbage.
285 */);
286 try
287 {
288 // We can not define any variables if the file does not contain any dimensions.
289 if (ncfile.getDimensions().isEmpty())
290 {
291 throw new IllegalStateException("No dimensions have been defined in the file: "
292 + outputFilename.getPath());
293 }
294
295 // The following if-block is used to work around issue XXX with the netcdf-java-4.0.41.jar
296 // The issue occurs under the following circumstance:
297 // - When trying to edit the header-info of a netCDF file that does not have any variables defined.
298 // The issue is caused by:
299 // - The netcdf java library parses the header-info of a netCDF file and only records a correct
300 // "where to write data from" byte-offset if the header-info contains a variable.
301 // To work around this issue we do the following:
302 // - If the header-info contains no variables, we create a dummy variable and write the file out.
303 // We then read the file back so that the "where to write data from" byte-offset is recorded correctly.
304 // Note: we remove the dummy variable before writing the file out again.
305 if (ncfile.getVariables().isEmpty())
306 {
307 ncfile.setRedefineMode(true);
308 ncfile.setLargeFile(true);
309 Variable variable = new Variable(ncfile, null /* containing group */, null /* parent structure */,
310 DUMMY_VARIABLE);
311 variable.setDataType(DataType.DOUBLE);
312 variable.setCaching(false);
313 variable.setDimensions(ncfile.getDimensions().get(0).getName()); // if we get here, then there has to
314 // exist at least 1 dimension.
315 ncfile.addVariable(null, variable);
316 ncfile.setRedefineMode(false);
317 ncfile.close();
318 ncfile = NetcdfFileWriteable.openExisting(outputFilename.getPath(), isFillVariable /*
319 * Setting fill =
320 * true causes
321 * everything to be
322 * written twice:
323 * first with the
324 * fill value, then
325 * with the data
326 * values. If you
327 * know you will
328 * write all the
329 * data, you dont
330 * need to use fill.
331 * If you don't know
332 * if all the data
333 * will be written,
334 * turning fill on
335 * ensures that any
336 * values not written
337 * will have the fill
338 * value. Otherwise
339 * those values will
340 * be undefined:
341 * possibly zero, or
342 * possibly garbage.
343 */);
344 ncfile.setRedefineMode(true);
345 ncfile.removeVariable(null, DUMMY_VARIABLE); // make sure we remove the dummy variable
346 }
347 else
348 {
349 ncfile.setRedefineMode(true);
350 }
351
352 ncfile.setLargeFile(isLargeFileSupport);
353
354 Variable variable = new Variable(ncfile, null /* containing group */, null /* parent structure */,
355 variableName);
356 variable.setDataType(variableDataType);
357 variable.setCaching(false);
358
359 // add variable's dimensions
360 variable.setDimensions(dimensionNamesString);
361
362 // add variable's attributes
363 for (Attribute attribute : variableAttributes)
364 {
365 variable.addAttribute(attribute);
366 }
367
368 ncfile.addVariable(null, variable);
369
370 // close editing header info
371 ncfile.setRedefineMode(false);
372 }
373 finally
374 {
375 if(ncfile!=null)
376 {
377 try
378 {
379 ncfile.close();
380 }
381 catch(Exception e)
382 {
383 LOG.error("Could not close file: " + outputFilename.getPath(), e);
384 }
385 }
386 }
387 }
388
389 /*
390 * (non-Javadoc)
391 *
392 * @see au.csiro.netcdf.Command#getCommandName()
393 */
394 @Override
395 public String getCommandName()
396 {
397 return NcDefineVariable.NC_DEFINE_VAR_COMMAND_NAME;
398 }
399
400 /*
401 * (non-Javadoc)
402 *
403 * @see java.lang.Object#toString()
404 */
405 public String toString()
406 {
407 // generate the help/usage statement
408 String header = "Define a variable in a netCDF file.";
409 String footer = "\nExample: ncdefinevar -outputFileName ABC.nc -variableName Lat "
410 + "-variableDataType float -dimensionNames Lat\n"
411 + "Will add a variable called Lat to the file ABC.nc. The variable is linked to the "
412 + "dimension lat and thus is a 'corresponding' variable. The file must already "
413 + "exist and the dimension must be already defined in the file.";
414 StringWriter sw = new StringWriter();
415 HelpFormatter formatter = new HelpFormatter();
416 formatter.setOptionComparator(new CommandLineOptionsComparator());
417 formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NcDefineVariable.NC_DEFINE_VAR_COMMAND_NAME, header,
418 this.options, 0, 1, footer);
419 return sw.toString();
420 }
421
422 /*
423 * (non-Javadoc)
424 *
425 * @see au.csiro.netcdf.Command#createOptions()
426 */
427 @SuppressWarnings("static-access")
428 @Override
429 public Options createOptions()
430 {
431 Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(
432 "1: the filename of the netCDF file to be modified.").isRequired(true).withLongOpt(OUTPUT_FILE).create(
433 "o");
434
435 Option inputFileName = OptionBuilder
436 .withArgName("file")
437 .hasArg()
438 .withDescription(
439 "2. the filename of a text file containing attributes to be loaded. "
440 + "OPTIONAL, ensure that text containing '=' or ',' characters are delimited by a backslash.")
441 .isRequired(false).withLongOpt(INPUT_FILE).create("i");
442
443 Option standardInput = OptionBuilder.withDescription(
444 "3: OPTIONAL, read attributes from Stdin, ensure that text containing '=' or ',' characters are delimited by a backslash.").isRequired(false)
445 .withLongOpt(STANDARD_INPUT).create("s");
446
447 Option variableName = OptionBuilder.withArgName("text").hasArg().withDescription(
448 "4: the name to be given to the variable.").isRequired(true).withLongOpt(VARIABLE_NAME).create("v");
449
450 Option variableType = OptionBuilder.withArgName("text").hasArg().withDescription(
451 "5: the data type of the variable, e.g. " + NcDefineVariable.netCDF3DataTypes).isRequired(true)
452 .withLongOpt(VARIABLE_DATA_TYPE).create("t");
453
454 Option variableAttributes = OptionBuilder
455 .withArgName("text")
456 .hasArg()
457 .withDescription(
458 "6: a comma separated list of attribute-value pairs, OPTIONAL, e.g. \"units=mm\", ensure that text containing '=' or ',' characters are delimited by a backslash.")
459 .isRequired(false).withLongOpt(VARIABLE_ATTRIBUTES).create("a");
460
461 Option dimensionNames = OptionBuilder.withArgName("text").hasArg().withDescription(
462 "7: OPTIONAL, a comma separated list of the variable's dimensions, e.g. date,latitude,longitude.")
463 .isRequired(false).withLongOpt(DIMENSION_NAMES).create("d");
464
465 Option largeFileSupport = OptionBuilder.withDescription(
466 "8: OPTIONAL, set if more than 2 GB of data will need to be stored in this file.").isRequired(false)
467 .withLongOpt(IS_LARGE_FILE).create("l");
468
469 Option fillVariable = OptionBuilder.withDescription(
470 "9: OPTIONAL, set if unwritten variable values should have the fill value.").isRequired(false)
471 .withLongOpt(IS_FILL_VARIABLE).create("f");
472
473 Options options = new Options();
474
475 options.addOption(outputFileName);
476 options.addOption(inputFileName);
477 options.addOption(standardInput);
478 options.addOption(variableName);
479 options.addOption(variableType);
480 options.addOption(variableAttributes);
481 options.addOption(dimensionNames);
482 options.addOption(largeFileSupport);
483 options.addOption(fillVariable);
484
485 return options;
486 }
487
488 /**
489 * @return the usage for this command.
490 */
491 public String getUsageString()
492 {
493 return this.toString();
494 }
495
496 /*
497 * (non-Javadoc)
498 *
499 * @see au.csiro.netcdf.cli.Command#validCommand(java.lang.String[])
500 */
501 public String validCommand(String[] commandLine)
502 {
503 String errorMsg = "";
504
505 // validate the command line parameters
506 try
507 {
508 // try parsing the command line, where ParseException can be thrown from
509 CommandLine parsedCommandLine = new BasicParser().parse(this.options, commandLine);
510
511 // try converting command line parameters to their associated types, where IllegalArgumentException can be
512 // thrown from
513 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
514 .getOptionValue(OUTPUT_FILE) : "";
515 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
516 .getOptionValue(INPUT_FILE) : "";
517 String variableDataTypeArg = (parsedCommandLine.hasOption(VARIABLE_DATA_TYPE)) ? parsedCommandLine
518 .getOptionValue(VARIABLE_DATA_TYPE) : "";
519 String variableAttributesArg = (parsedCommandLine.hasOption(VARIABLE_ATTRIBUTES)) ? parsedCommandLine
520 .getOptionValue(VARIABLE_ATTRIBUTES) : "";
521 String dimensionNamesArg = (parsedCommandLine.hasOption(DIMENSION_NAMES)) ? parsedCommandLine
522 .getOptionValue(DIMENSION_NAMES) : "";
523 boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);
524
525 // try getting the specified file
526 try
527 {
528 Util.getExistingFile(outputFilenameArg);
529 }
530 catch (IllegalArgumentException iae)
531 {
532 throw new IllegalArgumentException(NcDefineVariable.OUTPUT_FILE
533 + " value refers to non-existent file: " + outputFilenameArg);
534 }
535
536 // check whether large file support is needed
537 if (Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE && !isLargeFileSupport)
538 {
539 throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE
540 + " flag.");
541 }
542
543 // check that if an input file is specified then it actually exists.
544 if (!inputFilenameArg.isEmpty())
545 {
546 Util.getExistingFile(inputFilenameArg);
547 }
548
549 // try getting the dimension names
550 NcDefineVariable.mapStringToDimensionNamesString(dimensionNamesArg);
551
552 // try getting the variable's attributes from the command line.
553 if (!variableAttributesArg.isEmpty())
554 {
555 try
556 {
557 NetCDFUtils.mapStringToAttributeValueList(variableAttributesArg);
558 }
559 catch (IllegalArgumentException iae)
560 {
561 throw new IllegalArgumentException(NcDefineVariable.VARIABLE_ATTRIBUTES
562 + " value is not a comma separated String of attribute-value pairs: "
563 + variableAttributesArg);
564 }
565 }
566
567 // try getting the variable's data type
568 NcDefineVariable.mapStringToDataType(variableDataTypeArg);
569 }
570 catch (ParseException pe)
571 {
572 errorMsg = errorMsg + "\n" + pe.getMessage();
573 }
574 catch (IllegalArgumentException iae)
575 {
576 errorMsg = errorMsg + "\n" + iae.getMessage();
577 }
578
579 return errorMsg;
580 }
581
582 /**
583 * Maps a <code>String</code> into a whitespace separated list of dimension names.
584 *
585 * @param dimensionNamesArg
586 * a list of comma separated dimension descriptions, e.g. date,longitude,latitude,...
587 * @return whitespace separated list of dimension names, or '*' for Dimension.UNKNOWN. A <tt>null</tt> or empty
588 * String is a scalar.
589 * @throws IllegalArgumentException
590 * thrown if the <code>String</code> can not be mapped into a whitespace separated list of dimension
591 * names.
592 */
593 private static String mapStringToDimensionNamesString(String dimensionNamesArg) throws IllegalArgumentException
594 {
595 try
596 {
597 List<String> dimensionNames = Util.tokeniseCommaSeparatedString(dimensionNamesArg);
598 StringBuffer strBuf = new StringBuffer();
599 for (String dimensionName : dimensionNames)
600 {
601 strBuf.append(dimensionName + " ");
602 }
603 return strBuf.toString();
604 }
605 catch (PatternSyntaxException pse)
606 {
607 throw new IllegalArgumentException(NcDefineVariable.DIMENSION_NAMES
608 + " value is not a comma separated String: " + dimensionNamesArg);
609 }
610 }
611
612 /**
613 * Maps a <code>String</code> into a {@link DataType}.
614 *
615 * @param variableDataType
616 * a data type description.
617 * @return a {@link DataType}
618 * @throws IllegalArgumentException
619 * thrown if the <code>String</code> can not be mapped to a {@link DataType}.
620 */
621 private static DataType mapStringToDataType(String variableDataType) throws IllegalArgumentException
622 {
623 DataType dataType = DataType.getType(variableDataType);
624 if (dataType != null && NcDefineVariable.netCDF3DataTypes.contains(dataType))
625 {
626 return dataType;
627 }
628 throw new IllegalArgumentException(NcDefineVariable.VARIABLE_DATA_TYPE + " value is not a valid data type: "
629 + variableDataType + ". Allowed data types are: " + NcDefineVariable.netCDF3DataTypes);
630 }
631
632 /**
633 * @return the netCDF3DataTypes
634 */
635 public static Set<DataType> getNetCDF3DataTypes()
636 {
637 return netCDF3DataTypes;
638 }
639
640 /**
641 * @param netCDF3DataTypes the netCDF3DataTypes to set
642 */
643 public static void setNetCDF3DataTypes(Set<DataType> netCDF3DataTypes)
644 {
645 NcDefineVariable.netCDF3DataTypes = netCDF3DataTypes;
646 }
647 }