355 

   356 class chunkbuffer(object):

   357     """Allow arbitrary sized chunks of data to be efficiently read from an

   358     iterator over chunks of arbitrary size."""

   359     def __init__(self, in_iter, targetsize = 2**16):

   360         """in_iter is the iterator that's iterating over the input chunks.

   361         targetsize is how big a buffer to try to maintain."""

   362         self.in_iter = iter(in_iter)

   363         self.buf = ''

   364         targetsize = int(targetsize)

   365         if (targetsize <= 0):

   366             raise ValueError("targetsize must be greater than 0, was %d" % targetsize)

   367         self.targetsize = int(targetsize)

   368         self.iterempty = False

   369     def fillbuf(self):

   370         """x.fillbuf()

   371 

   372         Ignore the target size, and just read every chunk from the iterator

   373         until it's empty."""

   374         if not self.iterempty:

   375             collector = cStringIO.StringIO()

   376             collector.write(self.buf)

   377             for ch in self.in_iter:

   378                 collector.write(ch)

   379             self.buf = collector.getvalue()

   380             collector.close()

   381             collector = None

   382             self.iterempty = True

   383 

   384     def read(self, l):

   385         """x.read(l) -> str

   386         Read l bytes of data from the iterator of chunks of data.  Returns less

   387         than l bytes if the iterator runs dry."""

   388         if l > len(self.buf) and not self.iterempty:

   389             # Clamp to a multiple of self.targetsize

   390             targetsize = self.targetsize * ((l // self.targetsize) + 1)

   391             collector = cStringIO.StringIO()

   392             collector.write(self.buf)

   393             collected = len(self.buf)

   394             for chunk in self.in_iter:

   395                 collector.write(chunk)

   396                 collected += len(chunk)

   397                 if collected >= targetsize:

   398                     break

   399             if collected < targetsize:

   400                 self.iterempty = True

   401             self.buf = collector.getvalue()

   402             collector.close()

   403             collector = None

   404         s = self.buf[:l]

   405         self.buf = buffer(self.buf, l)

   406         return s

   407     def __repr__(self):

   408         return "<%s.%s targetsize = %u buffered = %u bytes>" % \

   409                (self.__class__.__module__, self.__class__.__name__,

   410                 self.targetsize, len(self.buf))

   411 

   412 def filechunkiter(f, size = 65536):

   413     """filechunkiter(file[, size]) -> generator

   414 

   415     Create a generator that produces all the data in the file size (default

   416     65536) bytes at a time.  Chunks may be less than size bytes if the

   417     chunk is the last chunk in the file, or the file is a socket or some

   418     other type of file that sometimes reads less data than is requested."""

   419     s = f.read(size)

   420     while len(s) >= 0:

   421         yield s

   422         s = f.read(size)