1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package au.csiro.netcdf;
21
22 import java.io.File;
23 import java.io.FileInputStream;
24 import java.io.IOException;
25 import java.io.PrintWriter;
26 import java.io.StringWriter;
27 import java.util.ArrayList;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.commons.cli.BasicParser;
33 import org.apache.commons.cli.CommandLine;
34 import org.apache.commons.cli.HelpFormatter;
35 import org.apache.commons.cli.Option;
36 import org.apache.commons.cli.OptionBuilder;
37 import org.apache.commons.cli.Options;
38 import org.apache.commons.cli.ParseException;
39 import org.apache.log4j.Logger;
40
41 import ucar.ma2.DataType;
42 import ucar.nc2.Attribute;
43 import ucar.nc2.NetcdfFileWriteable;
44 import ucar.nc2.Variable;
45 import au.csiro.netcdf.cli.Command;
46 import au.csiro.netcdf.cli.CommandLineOptionsComparator;
47 import au.csiro.netcdf.util.NetCDFUtils;
48 import au.csiro.netcdf.util.Util;
49
50
51
52
53
54
55
56
57
58
59
60 public class NcDefineAttributes implements Command
61 {
62 private static final String NULL_VALUE = "null";
63
64
65
66
67 public static final String NC_DEFINE_ATT_COMMAND_NAME = "ncdefineAtt";
68
69
70
71
72 public static final String OUTPUT_FILE = "outputFileName";
73
74
75
76
77 public static final String INPUT_FILE = "inputFileName";
78
79
80
81
82 private static final String STANDARD_INPUT = "standardInput";
83
84
85
86
87 public static final String ATTRIBUTES = "attributes";
88
89
90
91
92 public static final String VARIABLE_NAME = "variable";
93
94
95
96
97
98 public static final String IS_LARGE_FILE = "largeFileSupport";
99
100
101
102
103 public static final String FILENAME_PATTERN = "pattern";
104
105
106
107
108 public static final String ATTRIBUTE_DATA_TYPE = "attributeDataType";
109
110
111
112
113 private static final Logger LOG = Logger.getLogger(NcDefineAttributes.class.getName());
114
115
116
117
118
119 private static Set<DataType> netCDF3DataTypes = new HashSet<DataType>();
120
121 static
122 {
123 netCDF3DataTypes.add(DataType.DOUBLE);
124 netCDF3DataTypes.add(DataType.FLOAT);
125 netCDF3DataTypes.add(DataType.INT);
126 netCDF3DataTypes.add(DataType.CHAR);
127 netCDF3DataTypes.add(DataType.SHORT);
128 netCDF3DataTypes.add(DataType.BYTE);
129 netCDF3DataTypes.add(DataType.STRING);
130 }
131
132
133
134
135 private Options options = null;
136
137
138
139
140 public NcDefineAttributes()
141 {
142 this.options = createOptions();
143 }
144
145
146
147
148
149
150 @Override
151 public void execute(String[] args) throws ParseException, IOException, IllegalArgumentException
152 {
153
154 CommandLine parsedCommandLine = new BasicParser().parse(this.options, args);
155
156
157 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
158 .getOptionValue(OUTPUT_FILE) : "";
159 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
160 .getOptionValue(INPUT_FILE) : "";
161 boolean isStandardInput = parsedCommandLine.hasOption(STANDARD_INPUT);
162 String attributesArg = (parsedCommandLine.hasOption(ATTRIBUTES)) ? parsedCommandLine
163 .getOptionValue(ATTRIBUTES) : "";
164 boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);
165 String pattern = (parsedCommandLine.hasOption(FILENAME_PATTERN)) ? parsedCommandLine
166 .getOptionValue(FILENAME_PATTERN) : "";
167 String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine
168 .getOptionValue(VARIABLE_NAME) : "";
169 String attributeDataTypeArg = (parsedCommandLine.hasOption(ATTRIBUTE_DATA_TYPE)) ? parsedCommandLine
170 .getOptionValue(ATTRIBUTE_DATA_TYPE) : "String";
171
172 List<String> targetFileNames = Util.getListOfTargetFiles(outputFilenameArg, pattern);
173
174
175 for (String outputFileName : targetFileNames)
176 {
177 if (Util.fileExists(outputFileName)
178 && Util.getExistingFile(outputFileName).length() >= MAX_32BIT_OFFSET_FILE_SIZE
179 && !isLargeFileSupport)
180 {
181 throw new IllegalArgumentException("The netCDF file " + outputFileName
182 + " will be too large, please use " + IS_LARGE_FILE + " flag.");
183 }
184 }
185
186
187
188 if (!inputFilenameArg.isEmpty())
189 {
190 Util.getExistingFile(inputFilenameArg);
191 }
192
193
194 List<Attribute> attributes = new ArrayList<Attribute>();
195
196 if(!attributesArg.isEmpty())
197 {
198
199 DataType attributeDataType = NcDefineAttributes.mapStringToDataType(attributeDataTypeArg);
200
201 try
202 {
203 attributes = NetCDFUtils.mapStringToAttributeValueList(attributesArg, attributeDataType);
204 }
205 catch (IllegalArgumentException iae)
206 {
207 throw new IllegalArgumentException(NcDefineAttributes.ATTRIBUTES
208 + " value is not a comma separated String of attribute-value pairs: " + attributesArg);
209 }
210 }
211
212
213 if (!inputFilenameArg.isEmpty())
214 {
215 attributes.addAll(NetCDFUtils.readAttributesFromStream(new FileInputStream(inputFilenameArg)));
216 }
217 if(isStandardInput)
218 {
219 attributes.addAll(NetCDFUtils.readAttributesFromStream(System.in));
220 }
221
222 for (String outputFileName : targetFileNames)
223 {
224 LOG.debug("Processing file : " + outputFileName);
225 this.execute(outputFileName, attributes, isLargeFileSupport, variableNameArg);
226 }
227 }
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244 public void execute(String outputFilename, List<Attribute> attributes, boolean isLargeFileSupport) throws IOException, SecurityException
245 {
246 this.execute(outputFilename, attributes, isLargeFileSupport, "");
247 }
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266 public void execute(String outputFilename, List<Attribute> attributes,
267 boolean isLargeFileSupport, String variableName) throws IOException, SecurityException
268 {
269
270 NetcdfFileWriteable ncfile = null;
271
272
273 File outputFile = new File(outputFilename);
274 if (outputFile.exists())
275 {
276 ncfile = NetcdfFileWriteable.openExisting(outputFilename, false
277
278
279
280
281
282
283
284
285
286
287 );
288 try
289 {
290
291 ncfile.setRedefineMode(true);
292 ncfile.setLargeFile(isLargeFileSupport);
293
294
295 if (variableName.isEmpty())
296 {
297 for(Attribute attribute : attributes)
298 {
299
300 if (NULL_VALUE.equalsIgnoreCase(attribute.getStringValue()))
301 {
302 ncfile.deleteGlobalAttribute(attribute.getName());
303 }
304 else
305 {
306 ncfile.addAttribute(null, attribute);
307 }
308 }
309 }
310
311
312 else
313 {
314 Variable variable = ncfile.findVariable(variableName);
315 if (variable == null)
316 {
317 throw new IllegalArgumentException("The variable " +
318 variableName + " doesn't exist in file " + outputFilename + ".");
319 }
320 else
321 {
322 for(Attribute attribute : attributes)
323 {
324
325 if (NULL_VALUE.equalsIgnoreCase(attribute.getStringValue()))
326 {
327 variable.removeAttributeIgnoreCase(attribute.getName());
328 }
329 else
330 {
331 variable.addAttribute(attribute);
332 }
333 }
334 }
335 }
336
337
338 ncfile.setRedefineMode(false);
339 }
340 finally
341 {
342 if(ncfile!=null)
343 {
344 try
345 {
346 ncfile.close();
347 }
348 catch(Exception e)
349 {
350 LOG.error("Could not close file: " + outputFilename, e);
351 }
352 }
353 }
354 }
355 else
356 {
357 File parentFile = outputFile.getParentFile();
358 if (parentFile != null)
359 {
360 parentFile.mkdirs();
361 }
362 ncfile = NetcdfFileWriteable.createNew(outputFilename, false
363
364
365
366
367
368
369
370
371
372 );
373 try
374 {
375
376 for(Attribute attribute : attributes)
377 {
378 ncfile.addAttribute(null, attribute);
379 }
380
381
382 ncfile.create();
383 }
384 finally
385 {
386 if(ncfile!=null)
387 {
388 try
389 {
390 ncfile.close();
391 }
392 catch(Exception e)
393 {
394 LOG.error("Could not close file: " + outputFilename, e);
395 }
396 }
397 }
398 }
399 }
400
401
402
403
404
405
406 @Override
407 public String getCommandName()
408 {
409 return NcDefineAttributes.NC_DEFINE_ATT_COMMAND_NAME;
410 }
411
412
413
414
415
416
417 public String toString()
418 {
419
420 String header = "Define an attribute in a netCDF file.";
421 String footer = "\nExample: ncdefineatt -outputFileName ABC.nc "
422 + "-attributes \"netCDF-Java=4.0,Conventions=value with a space,toDelete=null\"\n"
423 + "Will add the two and delete one listed global attributes to the file ABC.nc. "
424 + "The file will be created if it doesn't already exist. ";
425 StringWriter sw = new StringWriter();
426 HelpFormatter formatter = new HelpFormatter();
427 formatter.setOptionComparator(new CommandLineOptionsComparator());
428 formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NcDefineAttributes.NC_DEFINE_ATT_COMMAND_NAME, header,
429 this.options, 0, 1, footer);
430 return sw.toString();
431 }
432
433
434
435
436
437
438 @SuppressWarnings("static-access")
439 @Override
440 public Options createOptions()
441 {
442 Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(
443 "1: the filename of the netCDF file to be created.").isRequired(true).withLongOpt(OUTPUT_FILE).create(
444 "o");
445
446 Option inputFileName = OptionBuilder
447 .withArgName("file")
448 .hasArg()
449 .withDescription(
450 "2. the filename of a text file containing attributes to be loaded. "
451 + "OPTIONAL, ensure that text containing '=' or ',' characters are delimited by a backslash.")
452 .isRequired(false).withLongOpt(INPUT_FILE).create("i");
453
454 Option standardInput = OptionBuilder.withDescription(
455 "3: OPTIONAL, read attributes from Stdin, ensure that text containing '=' or ',' characters are delimited by a backslash.").isRequired(false)
456 .withLongOpt(STANDARD_INPUT).create("s");
457
458 Option attributes = OptionBuilder
459 .withArgName("text")
460 .hasArg()
461 .withDescription(
462 "4: a comma separated list of attribute-value pairs, OPTIONAL, e.g. \"netCDF Java=4.0,Conventions=CF-1.0\", "
463 + "ensure that text containing '=' or ',' characters are delimited by a backslash. "
464 + "Set value to null to delete existing attribute.")
465 .isRequired(false).withLongOpt(ATTRIBUTES).create("a");
466
467 Option largeFileSupport = OptionBuilder.withDescription(
468 "5: OPTIONAL, set if more than 2 GB of data will need to be stored in this file.").isRequired(false)
469 .withLongOpt(IS_LARGE_FILE).create("l");
470
471 Option pattern = OptionBuilder.withArgName("filename pattern").hasArg().withDescription(
472 "6: OPTIONAL, a pattern to match multiple existing files, where multiple files should be "
473 + "processed. If this option is present the outputFilename is expected to be a directory "
474 + "containing the files to be processed. The wildcard characters * and ? are supported but "
475 + "may need to be escaped by a \\ to avoid processing by the shell. e.g. \\*.nc ").isRequired(
476 false).withLongOpt(FILENAME_PATTERN).create("p");
477 Option variableName = OptionBuilder
478 .withArgName("text")
479 .hasArg()
480 .withDescription(
481 "7: OPTIONAL, the variable to assign attribute(s) to, "
482 + "set to define variable attribute.")
483 .isRequired(false).withLongOpt(VARIABLE_NAME).create("v");
484
485 Option attributeType = OptionBuilder.withArgName("text").hasArg().withDescription(
486 "8: OPTIONAL, the data type of the attribute(s), e.g. " + NcDefineAttributes.netCDF3DataTypes)
487 .isRequired(false).withLongOpt(ATTRIBUTE_DATA_TYPE).create("t");
488
489 Options options = new Options();
490
491 options.addOption(outputFileName);
492 options.addOption(inputFileName);
493 options.addOption(standardInput);
494 options.addOption(attributes);
495 options.addOption(largeFileSupport);
496 options.addOption(pattern);
497 options.addOption(variableName);
498 options.addOption(attributeType);
499
500 return options;
501 }
502
503
504
505
506 public String getUsageString()
507 {
508 return this.toString();
509 }
510
511
512
513
514
515
516
517
518 public String validCommand(String[] commandLine)
519 {
520 String errorMsg = "";
521
522
523 try
524 {
525
526 CommandLine parsedCommandLine = new BasicParser().parse(this.options, commandLine);
527
528
529 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
530 .getOptionValue(OUTPUT_FILE) : "";
531 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
532 .getOptionValue(INPUT_FILE) : "";
533 String attributesArg = (parsedCommandLine.hasOption(ATTRIBUTES)) ? parsedCommandLine
534 .getOptionValue(ATTRIBUTES) : "";
535 boolean isLargeFileSupport = parsedCommandLine.hasOption(IS_LARGE_FILE);
536 String pattern = (parsedCommandLine.hasOption(FILENAME_PATTERN)) ? parsedCommandLine
537 .getOptionValue(FILENAME_PATTERN) : "";
538 String variableDataTypeArg = (parsedCommandLine.hasOption(ATTRIBUTE_DATA_TYPE)) ? parsedCommandLine
539 .getOptionValue(ATTRIBUTE_DATA_TYPE) : "String";
540
541
542 if (pattern.length() == 0 && Util.fileExists(outputFilenameArg)
543 && Util.getExistingFile(outputFilenameArg).length() >= MAX_32BIT_OFFSET_FILE_SIZE
544 && !isLargeFileSupport)
545 {
546 throw new IllegalArgumentException("The netCDF file will be too large, please use " + IS_LARGE_FILE
547 + " flag.");
548 }
549
550
551 if (pattern.length() > 0)
552 {
553 if (!Util.fileExists(outputFilenameArg))
554 {
555 throw new IllegalArgumentException("When a pattern is specified " +
556 "the output file must be an existing directory.");
557 }
558 File file = new File(outputFilenameArg);
559 if (!file.isDirectory())
560 {
561 throw new IllegalArgumentException("When a pattern is specified "
562 + "the output file must be an existing directory.");
563 }
564 }
565
566
567 if (!inputFilenameArg.isEmpty())
568 {
569 Util.getExistingFile(inputFilenameArg);
570 }
571
572
573 if(!attributesArg.isEmpty())
574 {
575 try
576 {
577 NetCDFUtils.mapStringToAttributeValueList(attributesArg);
578 }
579 catch (IllegalArgumentException iae)
580 {
581 throw new IllegalArgumentException(NcDefineAttributes.ATTRIBUTES
582 + " value is not a comma separated String of attribute-value pairs: "
583 + attributesArg);
584 }
585 }
586
587
588
589 NcDefineAttributes.mapStringToDataType(variableDataTypeArg);
590
591 }
592 catch (ParseException pe)
593 {
594 errorMsg = errorMsg + "\n" + pe.getMessage();
595 }
596 catch (IllegalArgumentException iae)
597 {
598 errorMsg = errorMsg + "\n" + iae.getMessage();
599 }
600
601 return errorMsg;
602 }
603
604
605
606
607
608
609
610
611
612
613 public static DataType mapStringToDataType(String variableDataType) throws IllegalArgumentException
614 {
615 DataType dataType = DataType.getType(variableDataType);
616 if (dataType != null && netCDF3DataTypes.contains(dataType))
617 {
618 return dataType;
619 }
620 throw new IllegalArgumentException(NcDefineVariable.VARIABLE_DATA_TYPE + " value is not a valid data type: "
621 + variableDataType + ". Allowed data types are: " + netCDF3DataTypes);
622 }
623 }