ConversionUtilsTest xref


1   /**
2    * Copyright 2010, CSIRO Australia.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  /**
18   * 
19   */
20  package au.csiro.netcdf.wron;
21  
22  import java.io.File;
23  import java.io.FileWriter;
24  import java.io.IOException;
25  import java.net.URL;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.SortedSet;
32  import java.util.TreeSet;
33  
34  import junit.framework.TestCase;
35  
36  /**
37   * Verify the function of the ConversionUtils functions
38   * 
39   * Copyright 2010, CSIRO Australia All rights reserved.
40   * 
41   * @author James Dempsey on 21/04/2010
42   * @version $Revision: 78 $ $Date: 2010-07-24 16:23:13 +1000 (Sat, 24 Jul 2010) $
43   */
44  public class ConversionUtilsTest extends TestCase
45  {
46  
47      private static final String FILL_VALUE = "-999.99";
48      private ConversionUtils convUtils;
49      private List<CellData> triples = new ArrayList<CellData>();
50      private String lats[] = new String[] { "-24.1", "-24.15", "-24.2", "-24.25", "-24.3", "-24.35", "-24.4", "-24.45",
51              "-24.5", "-24.55", "-24.6", "-24.65", "-24.7", "-24.75", "-24.8", "-24.85", "-24.9", "-24.95", "-25.0",
52              "-25.05", "-25.1" };
53      private String longs[] = new String[] { "146.45", "146.5", "146.55", "146.6", "146.65", "146.7", "146.75", "146.8",
54              "146.85", "146.9", "146.95", "147", "147.05", "147.1", "147.15" };
55      private Set<String> sortedLatitudes = new TreeSet<String>();
56      private Set<String> sortedLongitudes = new TreeSet<String>();
57      int baseCellNum = 470;
58      private File testFile[];
59      private List<String> csvFilenames;
60  
61      /*
62       * (non-Javadoc)
63       * 
64       * @see junit.framework.TestCase#setUp()
65       */
66      protected void setUp() throws Exception
67      {
68          super.setUp();
69          File testFolder = computeTestDataRoot(ConversionUtilsTest.class);
70          boolean result = testFolder.mkdirs();
71          String path = testFolder.getCanonicalPath() + File.separator;
72          // Removed "System.out.println" since it causes issues when running test locally.
73          //System.out.println("Path is " + path + " created:" + result);
74  
75          convUtils = new ConversionUtils();
76          sortedLatitudes.addAll(Arrays.asList(lats));
77          sortedLongitudes.addAll(Arrays.asList(longs));
78          for (int i = 0; i < lats.length; i++)
79          {
80              for (int j = 0; j < longs.length; j++)
81              {
82                  final int cellNum = i * 300 + 170 + j;
83                  CellData cd = new CellData(String.valueOf(cellNum), longs[j], lats[i]);
84                  cd.filename = path + String.valueOf(cellNum) + ".csv";
85                  triples.add(cd);
86              }
87          }
88          
89          csvFilenames = new ArrayList<String>();
90          for (int i = baseCellNum; i < baseCellNum+8; i++)
91          {
92              csvFilenames.add(path + String.valueOf(i) + ".csv");
93          }
94  
95      }
96  
97      /* (non-Javadoc)
98       * @see junit.framework.TestCase#tearDown()
99       */
100     @Override
101     protected void tearDown() throws Exception
102     {
103         if (testFile != null)
104         {
105             for (int i = 0; i < testFile.length; i++)
106             {
107                 File file = testFile[i];
108                 if (file != null && file.exists())
109                 {
110                     file.delete();
111                 }
112             }
113         }
114         super.tearDown();
115     }
116 
117     /**
118      * Test method for {@link au.csiro.netcdf.wron.ConversionUtils#getLatitudeBlock(java.util.Set, int, int)}.
119      */
120     public void testGetLatitudeBlock()
121     {
122         // Grab the next set of latitudes to be read
123         List<String> selLats = convUtils.getLatitudeBlock(sortedLatitudes, 0, 2);
124         assertEquals("Incorrect number of latitudes", 2, selLats.size());
125         for (int i = 0; i < 2; i++)
126         {
127             assertEquals("Incorrect latitude #" + i, lats[i], selLats.get(i));
128         }
129 
130         selLats = convUtils.getLatitudeBlock(sortedLatitudes, 3, 2);
131         assertEquals("Incorrect number of latitudes", 2, selLats.size());
132         for (int i = 0; i < 2; i++)
133         {
134             int latNum = 2 * 3 + i;
135             assertEquals("Incorrect latitude #" + latNum, lats[latNum], selLats.get(i));
136         }
137 
138         selLats = convUtils.getLatitudeBlock(sortedLatitudes, 10, 2);
139         assertEquals("Incorrect number of latitudes", 1, selLats.size());
140         assertEquals("Incorrect latitude #20", lats[20], selLats.get(0));
141 
142         selLats = convUtils.getLatitudeBlock(sortedLatitudes, 11, 2);
143         assertEquals("Incorrect number of latitudes", 0, selLats.size());
144     }
145 
146     /**
147      * Test method for
148      * {@link au.csiro.netcdf.wron.ConversionUtils#buildCsvFilenamesForLatitudes(java.util.List, java.util.List)}.
149      */
150     public void testBuildCsvFilenamesForLatitudes()
151     {
152         List<String> targetLats = new ArrayList<String>();
153         for (int i = 7; i < 10; i++)
154         {
155             targetLats.add(lats[i]);
156         }
157 
158         // Test a mid range set of latitudes
159         String folderName = "/data01/mdbsy/scenA/input/";
160         String extension = ".csv";
161         List<CellData> targetCells = convUtils.buildCsvFilenamesForLatitudes(targetLats, triples, folderName, extension);
162         assertEquals("Incorrect number of filenames returned", targetLats.size() * longs.length, targetCells.size());
163         for (int i = 0; i < targetLats.size(); i++)
164         {
165             for (int j = 0; j < longs.length; j++)
166             {
167                 int index = (i * longs.length) + j;
168                 assertEquals("Incorrect filenames for #" + index, folderName
169                         + String.valueOf(170 + ((i + 7) * 300) + j) + extension, targetCells.get(index).filename);
170             }
171         }
172         
173         // Test a mismatch of latitudes
174         targetLats.clear();
175         targetLats.add("foo");
176         targetLats.add("bar");
177         targetCells = convUtils.buildCsvFilenamesForLatitudes(targetLats, triples, folderName, extension);
178         assertEquals("Incorrect number of filenames returned", 0, targetCells.size());
179         
180     }
181 
182     /**
183      * Test method for
184      * {@link au.csiro.netcdf.wron.ConversionUtils#readDataByLatitudes(java.util.List, java.util.List, java.util.List, int)}
185      * .
186      */
187     public void testReadDataByLatitudesNoFiles()
188     {
189         List<String> csvFilenames = new ArrayList<String>();
190         File testFolder = computeTestDataRoot(ConversionUtilsTest.class);
191         String path = testFolder.getAbsolutePath() + File.separator;
192         for (int i = baseCellNum; i < baseCellNum+8; i++)
193         {
194             csvFilenames.add(path + String.valueOf(i) + ".csv");
195         }
196         int numVariables = 9;
197         List<String> targetLats = new ArrayList<String>();
198         targetLats.add(lats[0]);
199         List<CellData> targetCells = new ArrayList<CellData>();
200         for (CellData cellData : triples)
201         {
202             if (cellData.latitude.equals(lats[0]))
203             {
204                 targetCells.add(cellData);
205             }
206         }
207         int numTimes = 43098;
208         String[] fillData = new String[numVariables];
209         Arrays.fill(fillData, FILL_VALUE);
210         StringBuffer fillOnlyResult = new StringBuffer();
211         for (int i = 0; i < longs.length; i++)
212         {
213             fillOnlyResult.append(fillData[0]).append(System.getProperty("line.separator"));
214         }
215 
216         // Read all data from the file set (Map keys: time id, latitude)
217         Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(targetCells, targetLats,
218                 numVariables, numTimes, longs.length, fillData, longs[0]);
219         assertEquals("Incorrect number of times", numTimes, latData.size());
220         for (Integer timeKey : latData.keySet())
221         {
222             final Map<String, LongitudeRange> latsForTime = latData.get(timeKey);
223             assertEquals("Incorrect number of latitudes for time " + timeKey, targetLats.size(), latsForTime.size());
224             for (String lat : targetLats)
225             {
226                 assertEquals("Incorrect number of variables for time " + timeKey + " and latitude " + lat,
227                         numVariables, latsForTime.get(lat).getNumVariables());
228                 assertEquals("Incorrect variable 2 data for time " + timeKey + " and latitude " + lat,
229                         fillOnlyResult.toString(), latsForTime.get(lat).getValues(2));
230             }
231         }
232         
233     }
234 
235     /**
236      * Test method for
237      * {@link au.csiro.netcdf.wron.ConversionUtils#readDataByLatitudes(java.util.List, java.util.List, java.util.List, int)}
238      * .
239      * @throws IOException 
240      */
241     public void testReadDataByLatitudesWithFiles() throws IOException
242     {
243         int numVariables = 9;
244         List<String> targetLats = new ArrayList<String>();
245         targetLats.add(lats[0]);
246         targetLats.add(lats[1]);
247         List<CellData> targetCells = new ArrayList<CellData>();
248         for (CellData cellData : triples)
249         {
250             if (cellData.latitude.equals(lats[0]) || cellData.latitude.equals(lats[1]))
251             {
252                 targetCells.add(cellData);
253             }
254         }
255         int numTimes = 4398;
256         String[] fillData = new String[numVariables];
257         Arrays.fill(fillData, FILL_VALUE);
258         String[] filenames = new String[]{csvFilenames.get(2)};
259 
260         String[][] csvData = createTestFiles(numVariables, numTimes, filenames);
261         
262         // Read all data from the file set (Map keys: time id, latitude)
263         Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(targetCells, targetLats,
264                 numVariables, numTimes, longs.length, fillData, longs[0]);
265         assertEquals("Incorrect number of times", numTimes, latData.size());
266         int timeIndex = 0;
267         SortedSet<Integer> timeKeysOrdered = new TreeSet<Integer>(latData.keySet()); 
268         for (Integer timeKey : timeKeysOrdered)
269         {
270             final Map<String, LongitudeRange> latsForTime = latData.get(timeKey);
271             assertEquals("Incorrect number of latitudes for time " + timeKey, targetLats.size(), latsForTime.size());
272             LongitudeRange range = latsForTime.get(lats[1]);
273 //            System.out.println("Time:" + timeKey);
274 //            System.out.println(range.debugOutput());
275             for (int varNum = 0; varNum < numVariables; varNum++)
276             {
277                 assertEquals("Wrong value for variable " + varNum + " at time " + timeKey, 
278                         Float.parseFloat(csvData[timeIndex][varNum+1]), Float.parseFloat(range.getSingleValue(varNum, 2)));
279             }
280             timeIndex++;
281         }
282         
283     }
284 
285     public void testReadDataByLatitudesWithPartialCoverage() throws IOException
286     {
287         // Create a small test set - 5 dates, 2 latitudes, 3 longitudes with data only available for the middle longitudes
288         // Expected result for each time and variable
289         //  _, X, _
290         //  _, _. X
291         int numVariables = 2;
292         int numTimes = 5;
293         int numLongitudes = 3;
294         List<CellData> cellList = new ArrayList<CellData>();
295         cellList.add(new CellData(String.valueOf(baseCellNum+2), longs[1], lats[0]));
296         cellList.add(new CellData(String.valueOf(baseCellNum+5), longs[2], lats[1]));
297         List<String> targetLats = new ArrayList<String>();
298         for (CellData cellData : cellList)
299         {
300             targetLats.add(cellData.latitude);
301         }
302         String[] fillData = new String[numVariables];
303         Arrays.fill(fillData, FILL_VALUE);
304 
305         // Generate the testing file
306         String[] filenames = new String[]{csvFilenames.get(2), csvFilenames.get(5)};
307         String[][] csvData = createTestFiles(numVariables, numTimes, filenames);
308         for (int i = 0; i < filenames.length; i++)
309         {
310             cellList.get(i).filename = filenames[i];
311         }
312         
313         // Read all data from the file set (Map keys: time id, latitude)
314         Map<Integer, Map<String, LongitudeRange>> latData = convUtils.readDataByLatitudes(cellList, targetLats,
315                 numVariables, numTimes, numLongitudes, fillData, longs[0]);
316         assertEquals("Incorrect number of times", numTimes, latData.size());
317         int timeIndex = 0;
318         SortedSet<Integer> timeKeysOrdered = new TreeSet<Integer>(latData.keySet()); 
319         for (Integer timeKey : timeKeysOrdered)
320         {
321             final Map<String, LongitudeRange> latsForTime = latData.get(timeKey);
322             assertEquals("Incorrect number of latitudes for time " + timeKey, targetLats.size(), latsForTime.size());
323             int latIndex = 0;
324             for (String lat : targetLats)
325             {
326                 LongitudeRange range = latsForTime.get(lat);
327                 for (int varNum = 0; varNum < numVariables; varNum++)
328                 {
329                     //System.out.println("Got range for variable " + varNum + " at time " + timeKey + " lat " + lat + " of " + range.getValues(varNum));
330                     for (int i = 0; i < numLongitudes; i++)
331                     {
332                         String value = range.getSingleValue(varNum, i);
333                         if (i==latIndex+1)
334                         {
335                             assertEquals("Wrong value for variable " + varNum + " at time " + timeKey + " lat " + lat + " long " + i, 
336                                     Float.parseFloat(csvData[timeIndex][varNum+1]), Float.parseFloat(value));
337                         }
338                         else
339                         {
340                             assertEquals("Wrong value for variable " + varNum + " at time " + timeKey + " lat " + lat + " long " + i, 
341                                     Float.parseFloat(FILL_VALUE), Float.parseFloat(value));
342                         }
343                     }
344                 }
345                 latIndex++;
346             }
347             timeIndex++;
348         }
349         
350     }
351 
352     public void testGetLimitedLatitudes()
353     {
354         Set<String> result = convUtils.getLimitedLatitudes(sortedLatitudes, "", "-24.2");
355         assertEquals(3, result.size());
356         int i=0;
357         for (String string : result)
358         {
359             assertEquals("Incorrect value for lat " + i, lats[i], string);
360             i++;
361         }
362 
363         result = convUtils.getLimitedLatitudes(sortedLatitudes, "-24.9", "");
364         assertEquals(5, result.size());
365         i=16;
366         for (String string : result)
367         {
368             assertEquals("Incorrect value for lat " + i, lats[i], string);
369             i++;
370         }
371 
372         result = convUtils.getLimitedLatitudes(sortedLatitudes, "-24.9", "-25.0");
373         assertEquals(3, result.size());
374         i=16;
375         for (String string : result)
376         {
377             assertEquals("Incorrect value for lat " + i, lats[i], string);
378             i++;
379         }
380     }
381     
382 //    public void testGetTriplesGroupedByLatitude()
383 //    {
384 //        String [][] testData = new String[][] {
385 //                {"476", "143.0", "-25.95", "1", "0", "0"}, 
386 //                {"477", "143.05", "-25.95", "2", "0", "0"}, 
387 //                {"478", "143.10", "-25.95", "3", "0", "0"}, 
388 //                {"509", "143.0", "-26.0", "4", "0", "0"}, 
389 //                {"510", "143.05", "-26.0", "5", "0", "0"}, 
390 //                {"511", "143.10", "-26.0", "6", "0", "0"}, 
391 //                {"569", "143.0", "-26.05", "7", "0", "0"}, 
392 //                {"570", "143.05", "-26.05", "8", "0", "0"}, 
393 //                {"571", "143.10", "-26.05", "9", "0", "0"} 
394 //                };
395 //        Map<String, List<CellData>> latMap = convUtils.getTriplesGroupedByLatitude(testData);
396 //        assertNotNull("Triples map is null", latMap);
397 //        assertEquals("Incorrect number of latitude keys", 2, latMap.keySet().size());
398 //        assertNotNull("Missing latitude -25", latMap.get("-25"));
399 //        assertNotNull("Missing latitude -26", latMap.get("-26"));
400 //        List<CellData> cellList = latMap.get("-25");
401 //        assertEquals(3,cellList.size());
402 //        for (int i = 0; i < 3; i++)
403 //        {
404 //            CellData cell = cellList.get(i);
405 //            assertEquals("Incorrect cell id row" + i, testData[i][0], cell.cellId);
406 //            assertEquals("Incorrect longitude row" + i, testData[i][1], cell.longitude);
407 //            assertEquals("Incorrect latitude row" + i, testData[i][2], cell.latitude);
408 //            assertEquals("Incorrect elevation row" + i, testData[i][3], cell.elevation);
409 //            assertEquals("Incorrect catchment id row" + i, testData[i][4], cell.catchmentId);
410 //            assertEquals("Incorrect reporting region id row" + i, testData[i][5], cell.reportingRegionId);
411 //        }
412 //
413 //        cellList = latMap.get("-26");
414 //        assertEquals(6,cellList.size());
415 //        for (int i = 0; i < 6; i++)
416 //        {
417 //            CellData cell = cellList.get(i);
418 //            assertEquals("Incorrect cell id row" + (i+3), testData[i+3][0], cell.cellId);
419 //            assertEquals("Incorrect longitude row" + (i+3), testData[i+3][1], cell.longitude);
420 //            assertEquals("Incorrect latitude row" + (i+3), testData[i+3][2], cell.latitude);
421 //            assertEquals("Incorrect elevation row" + (i+3), testData[i+3][3], cell.elevation);
422 //            assertEquals("Incorrect catchment id row" + (i+3), testData[i+3][4], cell.catchmentId);
423 //            assertEquals("Incorrect reporting region id row" + (i+3), testData[i+3][5], cell.reportingRegionId);
424 //        }
425 //    }
426     
427     /**
428      * Create and populate the files with test data.
429      * Note: As a side-effect updates the testFile array - assumed to only be called once per test cycle.
430      * 
431      * @param numVariables
432      * @param numTimes
433      * @param filenames
434      * @throws IOException
435      */
436     private String[][] createTestFiles(int numVariables, int numTimes, String[] filenames) throws IOException
437     {
438         String[][] csvData = new String[numTimes][numVariables+1];
439         for (int i = 0; i < csvData.length; i++)
440         {
441             for (int j = 0; j < numVariables+1; j++)
442             {
443                 csvData[i][j] = String.valueOf(i*(j+1));
444             }
445         }
446         testFile = new File[filenames.length];
447         for (int fileNum = 0; fileNum < testFile.length; fileNum++)
448         {
449             testFile[fileNum] = new File(filenames[fileNum]);
450             if (!testFile[fileNum].getParentFile().exists())
451             {
452                 testFile[fileNum].getParentFile().mkdirs();
453             }
454             testFile[fileNum].createNewFile();
455             FileWriter writer = new FileWriter(testFile[fileNum]);
456             writer.append("Date");
457             for (int i = 0; i < numVariables; i++)
458             {
459                 writer.append(",Var " + (i+1));
460                 
461             }
462             writer.append(System.getProperty("line.separator"));
463             for (int i = 0; i < csvData.length; i++)
464             {
465                 StringBuffer line = new StringBuffer();
466                 for (int j = 0; j < csvData[i].length; j++)
467                 {
468                     line.append(csvData[i][j]);
469                     if (j < csvData[i].length-1)
470                     {
471                         line.append(',');
472                     }
473                 }
474                 writer.append(line);
475                 writer.append(System.getProperty("line.separator"));
476             }
477             writer.close();
478         }
479         
480         return csvData;
481     }
482 
483     
484 
485     @SuppressWarnings("unchecked")
486     public static File computeTestDataRoot(Class anyTestClass) {
487         final String clsUri = anyTestClass.getName().replace('.','/') + ".class";
488         final URL url = anyTestClass.getClassLoader().getResource(clsUri);
489         final String clsPath = url.getPath();
490         final File root = new File(clsPath.substring(0, clsPath.length() - clsUri.length()));
491         return new File(root.getParentFile(), "test-data");
492       }
493     
494 }