--- C:/Users/cwardgar/Desktop/origFileWriter.java Sat Apr 10 05:47:46 2010 +++ C:/Users/cwardgar/Desktop/newFileWriter.java Mon Apr 12 04:48:49 2010 @@ -252,43 +252,11 @@ long size = oldVar.getSize() * oldVar.getElementSize(); total += size; - // robert.bridle@csiro.au | www.csiro.au - // 03/29/2010 -///////////// ORIGINAL CODE ////////////////////// - int nelems = (int) (size / maxSize); - if (nelems <= 1) - copyAll(ncfile, oldVar); - else - copySome(ncfile, oldVar, nelems); -//////////////////////////////////////////////////// - -////////////// PROPOSED CODE ////////////////////// -/* if(size > maxSize) - { - int[] shape = oldVar.getShape(); - - // determine the size of all the dimensions, other than the first. - long sizeOfOtherDimensions = 1; - for (int i = 1; i < shape.length; i++) { - if (shape[i] >= 0) - sizeOfOtherDimensions *= shape[i]; + if (size <= maxSize) { + copyAll(ncfile, oldVar); + } else { + copySome(ncfile, oldVar, maxSize); } - - // determine number of bytes in all the dimensions, other than the first. - long bytesInOtherDimensions = sizeOfOtherDimensions * oldVar.getElementSize(); - - // first dimension chunk-size that will fit within maxSize of memory. - int firstDimensionChunkSize = (int) (maxSize/bytesInOtherDimensions); - //System.out.println("We can fit: " + firstDimensionChunkSize + " chunks in: " + maxSize + " bytes of memory."); - - copySome(ncfile, oldVar, firstDimensionChunkSize); - } - else - { - copyAll(ncfile, oldVar); - } */ -//////////////////////////////////////////////////// - } // write record data @@ -344,35 +312,75 @@ } } - private static void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems) throws IOException { - String newName = N3iosp.makeValidNetcdfObjectName( oldVar.getName()); + /** + * An index that computes chunk shapes. It is intended to be used to compute the origins and shapes for a series + * of contiguous writes to a multidimensional array. + */ + public static class ChunkingIndex extends Index { + public ChunkingIndex(int[] shape) { + super(shape); + } - int[] shape = oldVar.getShape(); - int[] origin = new int[oldVar.getRank()]; - int size = shape[0]; + /** + * Computes the shape of the largest possible contiguous chunk, starting at {@link #getCurrentCounter()} + * and with {@code numElems <= maxChunkElems}. + * + * @param maxChunkElems the maximum number of elements in the chunk shape. The actual element count of the shape + * returned is likely to be different, and can be found with {@link Index#computeSize}. + * @return the shape of the largest possible contiguous chunk. + */ + public int[] computeChunkShape(long maxChunkElems) { + int[] chunkShape = new int[rank]; - for (int i = 0; i < size; i += nelems) { - origin[0] = i; - int left = size - i; - shape[0] = Math.min(nelems, left); + for (int iDim = 0; iDim < rank; ++iDim) { + chunkShape[iDim] = (int) (maxChunkElems / stride[iDim]); + chunkShape[iDim] = (chunkShape[iDim] == 0) ? 1 : chunkShape[iDim]; + chunkShape[iDim] = Math.min(chunkShape[iDim], shape[iDim] - current[iDim]); + } - Array data; - try { - data = oldVar.read(origin, shape); - if (oldVar.getDataType() == DataType.STRING) { - data = convertToChar(ncfile.findVariable(newName), data); + return chunkShape; } - if (data.getSize() > 0) {// zero when record dimension = 0 - ncfile.write(newName, origin, data); - if (debugWrite) System.out.println("write "+data.getSize()+" bytes"); + } + + /** + * Copies data from {@code oldVar} to {@code ncfile}. The writes are done in a series of chunks no larger than + * {@code maxChunkSize} bytes. + * + * @param ncfile the NetCDF file to write to. + * @param oldVar a variable from the original file to copy data from. + * @param maxChunkSize the size, in bytes, of the largest chunk to write. + * @throws IOException if an I/O error occurs. + */ + private static void copySome(NetcdfFileWriteable ncfile, Variable oldVar, long maxChunkSize) throws IOException { + String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getName()); + long maxChunkElems = maxChunkSize / oldVar.getElementSize(); + + ChunkingIndex index = new ChunkingIndex(oldVar.getShape()); + while (index.currentElement() < index.getSize()) { + try { + int[] chunkOrigin = index.getCurrentCounter(); + int[] chunkShape = index.computeChunkShape(maxChunkElems); + + Array data = oldVar.read(chunkOrigin, chunkShape); + + if (oldVar.getDataType() == DataType.STRING) { + data = convertToChar(ncfile.findVariable(newName), data); + } + + if (data.getSize() > 0) {// zero when record dimension = 0 + ncfile.write(newName, chunkOrigin, data); + if (debugWrite) { + System.out.println("write " + data.getSize() + " bytes"); + } + } + + index.setCurrentCounter(index.currentElement() + (int) Index.computeSize(chunkShape)); + } catch (InvalidRangeException e) { + e.printStackTrace(); + throw new IOException(e.getMessage()); + } } - - } catch (InvalidRangeException e) { - e.printStackTrace(); - throw new IOException(e.getMessage()); - } } - } private static Array convertToChar(Variable newVar, Array oldData) { ArrayChar newData = (ArrayChar) Array.factory(DataType.CHAR, newVar.getShape());