1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package au.csiro.netcdf;
21
22 import java.io.BufferedInputStream;
23 import java.io.BufferedReader;
24 import java.io.DataInputStream;
25 import java.io.EOFException;
26 import java.io.File;
27 import java.io.FileInputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.PrintWriter;
32 import java.io.StringWriter;
33 import java.util.ArrayList;
34 import java.util.List;
35
36 import org.apache.commons.cli.BasicParser;
37 import org.apache.commons.cli.CommandLine;
38 import org.apache.commons.cli.HelpFormatter;
39 import org.apache.commons.cli.Option;
40 import org.apache.commons.cli.OptionBuilder;
41 import org.apache.commons.cli.Options;
42 import org.apache.commons.cli.ParseException;
43 import org.apache.log4j.Logger;
44
45 import ucar.ma2.Array;
46 import ucar.ma2.InvalidRangeException;
47 import ucar.ma2.Range;
48 import ucar.nc2.NetcdfFile;
49 import ucar.nc2.NetcdfFileWriteable;
50 import ucar.nc2.Variable;
51 import au.csiro.netcdf.cli.Command;
52 import au.csiro.netcdf.cli.CommandLineOptionsComparator;
53 import au.csiro.netcdf.util.NetCDFUtils;
54 import au.csiro.netcdf.util.Util;
55
56
57
58
59
60
61
62
63
64
65
66 public class NcWriteVariable implements Command
67 {
68
69
70
71 private static final Logger LOG = Logger.getLogger(NcWriteVariable.class.getName());
72
73
74
75
76 public static final String NC_WRITE_VAR_COMMAND_NAME = "ncwriteVar";
77
78
79
80
81 private static final String OUTPUT_FILE = "outputFileName";
82
83
84
85
86 private static final String INPUT_FILE = "inputFileName";
87
88
89
90
91 private static final String VARIABLE_NAME = "variableName";
92
93
94
95
96 private static final String FILL_RANGE = "fillRange";
97
98
99
100
101 private static final String BINARY = "binary";
102
103
104 private Options options;
105
106
107
108
109 public NcWriteVariable()
110 {
111 this.options = createOptions();
112 }
113
114
115
116
117 @Override
118 public void execute(String[] args) throws ParseException, IOException
119 {
120
121 CommandLine parsedCommandLine = new BasicParser().parse(options, args);
122
123
124 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
125 .getOptionValue(OUTPUT_FILE) : "";
126 String variableNameArg = (parsedCommandLine.hasOption(VARIABLE_NAME)) ? parsedCommandLine
127 .getOptionValue(VARIABLE_NAME) : "";
128 String fillRangeArg = (parsedCommandLine.hasOption(FILL_RANGE)) ? parsedCommandLine
129 .getOptionValue(FILL_RANGE) : "";
130 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
131 .getOptionValue(INPUT_FILE) : "";
132 boolean binary = parsedCommandLine.hasOption(BINARY);
133
134
135 File outputFile = Util.getExistingFile(outputFilenameArg);
136 InputStream input;
137 if (inputFilenameArg.length()>0)
138 {
139 input = new FileInputStream(inputFilenameArg);
140 }
141 else
142 {
143 input = System.in;
144 }
145
146 this.execute(outputFile, variableNameArg, fillRangeArg, input, binary);
147 if (inputFilenameArg.length()>0)
148 {
149 input.close();
150 }
151 }
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166 public void execute(File outputFile, String variableName, String fillRange, InputStream dataStream, boolean binary)
167 throws IOException
168 {
169
170 NetcdfFileWriteable ncfile = null;
171
172 ncfile = NetcdfFileWriteable.openExisting(outputFile.getPath(), false);
173 try
174 {
175 Variable variable = ncfile.findVariable(NetcdfFile.escapeName(variableName));
176
177
178 if (variable == null)
179 {
180 throw new IllegalArgumentException("The variable: " + variableName + " does not exist in the file: "
181 + ncfile.getLocation());
182 }
183
184 List<Range> ranges = variable.getRanges();
185
186
187 String fillRangeVal[] = fillRange.split(",");
188 if (ranges.size() != fillRangeVal.length)
189 {
190 throw new IllegalArgumentException("Expected " + ranges.size() + " fillRange arguments but only got "
191 + fillRangeVal.length);
192 }
193
194
195 Range fillRanges[] = convertStringsToRanges(fillRangeVal, variable, ncfile);
196 List<int[]> originList = createOriginList(fillRanges);
197
198
199 Array buffer = createArray(fillRanges, variable);
200
201
202 if (binary)
203 {
204 DataInputStream dataIn = new DataInputStream(new BufferedInputStream(dataStream));
205
206
207 for (int[] origin : originList)
208 {
209 readBinaryData(dataIn, buffer);
210 ncfile.write(variableName, origin, buffer);
211 }
212 dataIn.close();
213 }
214 else
215 {
216 BufferedReader dataIn = new BufferedReader(new InputStreamReader(dataStream));
217
218
219 for (int[] origin : originList)
220 {
221 readData(dataIn, buffer);
222 ncfile.write(variableName, origin, buffer);
223 }
224 dataIn.close();
225 }
226 }
227 catch (InvalidRangeException e)
228 {
229 LOG.error("Failed to write data due to illegal shaped values array.", e);
230 throw new IllegalArgumentException("Failed to write data due to illegal shaped values array.", e);
231 }
232 finally
233 {
234 ncfile.close();
235 }
236 }
237
238
239
240
241 @Override
242 public String getCommandName()
243 {
244 return NC_WRITE_VAR_COMMAND_NAME;
245 }
246
247
248
249
250 @Override
251 public String toString()
252 {
253
254 String header = "Write data to variable in a netCDF file.";
255 String footer = "\nExample: ncwritevar -outputFileName ABC.nc -variableName Lat " + "-fillRange 0-559\n"
256 + "Will write data from stdin to the variable Lat in the file ABC.nc. The data will "
257 + "be written to all 560 cells in the variable. The file must already exist and "
258 + "the variable must be already defined in the file. Data would normally be piped in "
259 + "from a command such as ncextractcsv.\n"
260 + "\nExample: ncwritevar -outputFileName ABC.nc -variableName rainfall "
261 + "-fillRange lookup(-42.8)-lookup(-35.3)\n"
262 + "Will write data from stdin to the variable rainfall in the file ABC.nc. The data will "
263 + "only be written to the range between the two listed values. \n"
264 + "The lookup function allows you to convert a value to an index for a particular dimension. "
265 + "For the lookup to work, the dimension must have a coordinate variable defined (a variable "
266 + "associated with the dimension and having the same name as the dimension). For best performance "
267 + "the values in the coordinate variable should be in ascending order.";
268 StringWriter sw = new StringWriter();
269 HelpFormatter formatter = new HelpFormatter();
270 formatter.setOptionComparator(new CommandLineOptionsComparator());
271 formatter.printHelp(new PrintWriter(sw), PRINT_WIDTH, NC_WRITE_VAR_COMMAND_NAME, header, options, 0, 1, footer);
272 return sw.toString();
273 }
274
275
276
277
278 @Override
279 public String getUsageString()
280 {
281 return toString();
282 }
283
284
285
286
287 @SuppressWarnings("static-access")
288 @Override
289 public Options createOptions()
290 {
291 Option outputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(
292 "1. the filename of the netCDF file to be updated.").isRequired(true).withLongOpt(OUTPUT_FILE).create(
293 "o");
294
295 Option inputFileName = OptionBuilder.withArgName("file").hasArg().withDescription(
296 "2. the filename of the text file to be loaded. "
297 + "Stdin will be used if this option is not present. OPTIONAL ").isRequired(false).withLongOpt(
298 INPUT_FILE).create("i");
299
300 Option variableName = OptionBuilder.withArgName("text").hasArg().withDescription("3. the name of the variable.")
301 .isRequired(true).withLongOpt(VARIABLE_NAME).create("v");
302
303 Option fillRange = OptionBuilder.withArgName("text").hasArg().withDescription(
304 "4. the range of the variable to be filled. Must be a comma "
305 + "separated list with values for each dimension of the variable. "
306 + "Values may be a range n-n or a value n. e.g. 5-10, 3").isRequired(true).withLongOpt(
307 FILL_RANGE).create("f");
308
309 Option binary = OptionBuilder.withDescription("5. indicates the data is in a binary format.").isRequired(false)
310 .withLongOpt(BINARY).create("b");
311
312 Options options = new Options();
313
314 options.addOption(outputFileName);
315 options.addOption(variableName);
316 options.addOption(fillRange);
317 options.addOption(inputFileName);
318 options.addOption(binary);
319
320 return options;
321 }
322
323
324
325
326 @Override
327 public String validCommand(String[] args)
328 {
329 String errorMsg = "";
330
331
332 try
333 {
334
335 CommandLine parsedCommandLine = new BasicParser().parse(options, args);
336
337
338
339 String outputFilenameArg = (parsedCommandLine.hasOption(OUTPUT_FILE)) ? parsedCommandLine
340 .getOptionValue(OUTPUT_FILE) : "";
341 String inputFilenameArg = (parsedCommandLine.hasOption(INPUT_FILE)) ? parsedCommandLine
342 .getOptionValue(INPUT_FILE) : "";
343 String fillRangeArg = (parsedCommandLine.hasOption(FILL_RANGE)) ? parsedCommandLine
344 .getOptionValue(FILL_RANGE) : "";
345
346 Util.getExistingFile(outputFilenameArg);
347 if (inputFilenameArg.length() > 0)
348 {
349 Util.getExistingFile(inputFilenameArg);
350 }
351
352 validateRangeFormat(fillRangeArg);
353
354
355 }
356 catch (ParseException pe)
357 {
358 errorMsg = errorMsg + pe.getMessage();
359 }
360 catch (IllegalArgumentException iae)
361 {
362 errorMsg = errorMsg + iae.getMessage();
363 }
364
365 return errorMsg;
366 }
367
368
369
370
371
372
373
374 private void validateRangeFormat(String fillRangeArg)
375 {
376 final String lookupPattern = "(lookup\\(.*\\))|(\\d+)(-((lookup\\(.*\\))|(\\d+)))?";
377
378 String ranges[] = fillRangeArg.split(",");
379 List<String> errors = new ArrayList<String>();
380 for (String range : ranges)
381 {
382 if (!range.trim().matches(lookupPattern))
383 {
384 errors.add(range + " is not a valid range. It must be of the form "
385 + "0, 0-9, lookup(0) or lookup(0)-lookup(9) specifying "
386 + "a start and optional end to the range.");
387 }
388 }
389 if (!errors.isEmpty())
390 {
391 String err = "";
392 for (String string : errors)
393 {
394 err += "\n" + string;
395 }
396 throw new IllegalArgumentException(FILL_RANGE + " value is not a comma separated String: " + fillRangeArg
397 + err);
398 }
399
400
401 }
402
403
404
405
406
407
408
409
410
411
412
413 Range[] convertStringsToRanges(String[] rangeStrings, Variable variable, NetcdfFile file)
414 throws NumberFormatException, IOException
415 {
416 final String lookupPattern = "^lookup\\(.*\\)$";
417 final int lookupPrefixLen = 7;
418
419 Range[] ranges = new Range[rangeStrings.length];
420 for (int i = 0; i < rangeStrings.length; i++)
421 {
422 String string = rangeStrings[i];
423 String[] digits = splitRange(string);
424 int first;
425
426 if (digits[0].matches(lookupPattern))
427 {
428 first = NetCDFUtils.lookupIndex(file, variable.getDimension(i), digits[0].substring(lookupPrefixLen,
429 digits[0].length() - 1));
430 }
431 else
432 {
433 first = Integer.parseInt(digits[0]);
434 }
435 int last = first;
436 if (digits.length > 1)
437 {
438 if (digits[1].matches(lookupPattern))
439 {
440 last = NetCDFUtils.lookupIndex(file, variable.getDimension(i), digits[1].substring(lookupPrefixLen,
441 digits[1].length() - 1));
442 }
443 else
444 {
445 last = Integer.parseInt(digits[1]);
446 }
447 }
448 try
449 {
450 ranges[i] = new Range(first, last, 1);
451 }
452 catch (InvalidRangeException e)
453 {
454 LOG.error("Invalid range: " + string, e);
455 throw new IllegalArgumentException("Invalid range: " + string);
456 }
457
458 }
459
460 return ranges;
461 }
462
463
464
465
466
467
468
469 String[] splitRange(String rangeStr)
470 {
471 String safeString = rangeStr.trim().replaceAll("(\\()-([^\\)]*\\))" , "$1~$2");
472 String[] result = safeString.split("-");
473 for (int i = 0; i < result.length; i++)
474 {
475 result[i] = result[i].trim().replaceAll("(\\()~([^\\)]*\\))" , "$1-$2");
476 }
477 return result;
478 }
479
480
481
482
483
484
485
486
487
488
489 List<int[]> createOriginList(Range[] fillRanges)
490 {
491
492 if (fillRanges.length == 1)
493 {
494 int[] origin = new int[]{fillRanges[0].first()};
495 List<int[]> singleOrigin = new ArrayList<int[]>();
496 singleOrigin.add(origin);
497 return singleOrigin;
498 }
499
500
501 List<List<int[]>> originForRange = new ArrayList<List<int[]>>();
502
503
504 List<int[]> originList = new ArrayList<int[]>();
505 for (int j = fillRanges[0].first(); j <= fillRanges[0].last(); j+=fillRanges[0].stride())
506 {
507 originList.add(new int[]{j});
508 }
509 originForRange.add(originList);
510
511
512 for (int rangeNum = 1; rangeNum < fillRanges.length-1; rangeNum++)
513 {
514 Range currRange = fillRanges[rangeNum];
515
516 List<int[]> prevOrigins = originForRange.get(rangeNum-1);
517 List<int[]> currOrigins = new ArrayList<int[]>();
518 for (int[] prevPoint : prevOrigins)
519 {
520 for (int j = currRange.first(); j <= currRange.last(); j+=currRange.stride())
521 {
522 int[] origin = new int[rangeNum+1];
523 for (int j2 = 0; j2 < prevPoint.length; j2++)
524 {
525 origin[j2] = prevPoint[j2];
526 }
527 origin[rangeNum] = j;
528 currOrigins.add(origin);
529 }
530 }
531
532 originForRange.add(currOrigins);
533 }
534
535
536 List<int[]> origins = originForRange.get(fillRanges.length-2);
537 List<int[]> finalOrigins = new ArrayList<int[]>(origins.size());
538 int startFinalRange = fillRanges[fillRanges.length-1].first();
539 for (int[] origin : origins)
540 {
541 int[] newOrigin = new int[fillRanges.length];
542 for (int i = 0; i < origin.length; i++)
543 {
544 newOrigin[i] = origin[i];
545 }
546 newOrigin[fillRanges.length-1] = startFinalRange;
547 finalOrigins.add(newOrigin);
548 }
549
550
551 return finalOrigins;
552 }
553
554
555
556
557
558
559
560 Array createArray(Range[] fillRanges, Variable variable)
561 {
562 int shape[] = new int[fillRanges.length];
563 for (int i = 0; i < fillRanges.length; i++)
564 {
565 if (i<fillRanges.length-1)
566 {
567 shape[i] = 1;
568 }
569 else
570 {
571 Range lastRange = fillRanges[fillRanges.length-1];
572 shape[i] = lastRange.last() - lastRange.first() + 1;
573 }
574 }
575 Array array = Array.factory(variable.getDataType(), shape);
576 return array;
577 }
578
579
580
581
582
583
584
585 @SuppressWarnings("unchecked")
586 void readData(BufferedReader dataIn, Array buffer) throws IOException
587 {
588 Class cls = buffer.getElementType();
589 boolean isFloat = cls.getName().equalsIgnoreCase("Float");
590 boolean isDouble = cls.getName().equalsIgnoreCase("Double");
591 boolean isLong = cls.getName().equalsIgnoreCase("Long");
592 boolean isInt = cls.getName().equalsIgnoreCase("Int");
593 boolean isChar = cls.getName().equalsIgnoreCase("Char");
594 boolean isByte = cls.getName().equalsIgnoreCase("Byte");
595 boolean isBoolean = cls.getName().equalsIgnoreCase("Boolean");
596 boolean isShort = cls.getName().equalsIgnoreCase("Short");
597
598 int i = 0;
599 while (i < buffer.getSize())
600 {
601 String line = dataIn.readLine();
602 if (line == null)
603 {
604 String error = "Expected block of " + buffer.getSize() + " values but ran out after " + i + ".";
605 System.err.println(error);
606 throw new IllegalArgumentException(error);
607 }
608 if (isFloat)
609 {
610 buffer.setFloat(i, Float.parseFloat(line));
611 }
612 else if (isDouble)
613 {
614 buffer.setDouble(i, Double.parseDouble(line));
615 }
616 else if (isLong)
617 {
618 buffer.setLong(i, Long.parseLong(line));
619 }
620 else if (isInt)
621 {
622 buffer.setInt(i, Integer.parseInt(line));
623 }
624 else if (isChar)
625 {
626 buffer.setChar(i, line.charAt(0));
627 }
628 else if (isByte)
629 {
630 buffer.setByte(i, Byte.parseByte(line));
631 }
632 else if (isBoolean)
633 {
634 buffer.setBoolean(i, Boolean.parseBoolean(line));
635 }
636 else if (isShort)
637 {
638 buffer.setShort(i, Short.parseShort(line));
639 }
640 i++;
641 }
642
643 }
644
645
646
647
648
649
650
651 @SuppressWarnings("unchecked")
652 void readBinaryData(DataInputStream dataIn, Array buffer) throws IOException
653 {
654 Class cls = buffer.getElementType();
655 boolean isFloat = cls.getName().equalsIgnoreCase("Float");
656 boolean isDouble = cls.getName().equalsIgnoreCase("Double");
657 boolean isLong = cls.getName().equalsIgnoreCase("Long");
658 boolean isInt = cls.getName().equalsIgnoreCase("Int");
659 boolean isChar = cls.getName().equalsIgnoreCase("Char");
660 boolean isByte = cls.getName().equalsIgnoreCase("Byte");
661 boolean isBoolean = cls.getName().equalsIgnoreCase("Boolean");
662 boolean isShort = cls.getName().equalsIgnoreCase("Short");
663
664 int i = 0;
665 while (i < buffer.getSize())
666 {
667 try
668 {
669 if (isFloat)
670 {
671 buffer.setFloat(i, dataIn.readFloat());
672 }
673 else if (isDouble)
674 {
675 buffer.setDouble(i, dataIn.readDouble());
676 }
677 else if (isLong)
678 {
679 buffer.setLong(i, dataIn.readLong());
680 }
681 else if (isInt)
682 {
683 buffer.setInt(i, dataIn.readInt());
684 }
685 else if (isChar)
686 {
687 buffer.setChar(i, dataIn.readChar());
688 }
689 else if (isByte)
690 {
691 buffer.setByte(i, dataIn.readByte());
692 }
693 else if (isBoolean)
694 {
695 buffer.setBoolean(i, dataIn.readBoolean());
696 }
697 else if (isShort)
698 {
699 buffer.setShort(i, dataIn.readShort());
700 }
701 }
702 catch (EOFException e)
703 {
704 String error = "Expected block of " + buffer.getSize() + " values but ran out after " + i + ".";
705 System.err.println(error);
706 throw new IllegalArgumentException(error);
707 }
708 i++;
709 }
710
711 }
712 }