Finished implementation of buffer including a test which shows the performance should be usable in the real world. Its actually not too bad

Author: Byron

smmap/buf.py

@@ -44,19 +44,40 @@ def __del__(self):
 
 	def __getitem__(self, i):
 		c = self._c
+		assert c.is_valid()
 		if not c.includes_ofs(i):
 			c.use_region(i, 1)
 		# END handle region usage
-		assert c.is_valid()		# TODO: remove for performance
-		return c.buffer()[i]
+		return c.buffer()[i-c.ofs_begin()]
 
 	def __getslice__(self, i, j):
 		c = self._c
 		# fast path, slice fully included - safes a concatenate operation and 
 		# should be the default
-		if c.ofs_begin() >= i and j < c.ofs_end():
-			return c.buffer()[i:j]
-		raise NotImplementedError()
+		assert c.is_valid()
+		if (c.ofs_begin() <= i) and (j < c.ofs_end()):
+			b = c.ofs_begin()
+			return c.buffer()[i-b:j-b]
+		else:
+			l = j-i					# total length
+			ofs = i
+			# keep tokens, and join afterwards. This is faster
+			# as it can preallocate the total amoint of space needed
+			# (and its verified the implementation does that)
+			# Question is whether the list allocation doesn't counteract this, 
+			# but lets see ... 
+			tokens = list()
+			tappend = tokens.append
+			
+			while l:
+				c.use_region(ofs, l)
+				d = c.buffer()[:l]
+				ofs += len(d)
+				l -= len(d)
+				tappend(d)
+			#END while there are bytes to read
+			return ''.join(tokens)
+		# END fast or slow path
 	#{ Interface
 
 	def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0):

smmap/mman.py

@@ -222,6 +222,8 @@ def unuse_region(self):
 			to unuse the region once you are done reading from it in persistent cursors as it 
 			helps to free up resource more quickly"""
 		self._region = None
+		# note: should reset ofs and size, but we spare that for performance. Its not 
+		# allowed to query information if we are not valid !
 
 	def buffer(self):
 		"""Return a buffer object which allows access to our memory region from our offset
@@ -240,7 +242,8 @@ def is_associated(self):
 		return self._rlist is not None
 
 	def ofs_begin(self):
-		""":return: offset to the first byte pointed to by our cursor"""
+		""":return: offset to the first byte pointed to by our cursor
+		:note: only if is_valid() is True"""
 		return self._region._b + self._ofs
 
 	def ofs_end(self):

smmap/test/test_buf.py

@@ -61,32 +61,42 @@ def test_basics(self):
 		# exagerate the manager's overhead, but measure the buffer overhead
 		# We do it once with an optimal setting, and with a worse manager which 
 		# will produce small mappings only !
-		max_num_accesses = 5000
-		num_accesses_left = max_num_accesses
-		
-		for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man):
+		max_num_accesses = 1000
+		for manager, man_id in ( (man, 'optimal'), 
+								(MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')):
 			TestBuffer.manager = manager
-			st = time()
 			buf = TestBuffer(fc.path)
 			assert manager.num_file_handles() == 1
-			num_bytes = 0
-			fsize = fc.size
-			while num_accesses_left:
-				num_accesses_left -= 1
-				ofs_start = randint(0, fsize)
-				ofs_end = randint(ofs_start, fsize)
-				d = buf[ofs_start:ofs_end]
-				assert len(d) == ofs_end - ofs_start
-				assert d == data[ofs_start:ofs_end]
-				num_bytes += len(d)
-				pos = randint(0, fsize)
-				assert buf[pos] == data[pos]
-			# END handle num accesses
-			buf.end_access()
-			assert manager.num_file_handles() == 1
-			assert manager.collect() == 1
-			assert manager.num_file_handles() == 0
-			elapsed  = time() - st
-			mb = 1000*1000
-			sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) 
+			for access_mode in range(2):	# single, multi
+				num_accesses_left = max_num_accesses
+				num_bytes = 0
+				fsize = fc.size
+				
+				st = time()
+				buf.begin_access()
+				while num_accesses_left:
+					num_accesses_left -= 1
+					if access_mode:	# multi
+						ofs_start = randint(0, fsize)
+						ofs_end = randint(ofs_start, fsize)
+						d = buf[ofs_start:ofs_end]
+						assert len(d) == ofs_end - ofs_start
+						assert d == data[ofs_start:ofs_end]
+						num_bytes += len(d)
+					else:
+						pos = randint(0, fsize)
+						assert buf[pos] == data[pos]
+						num_bytes += 1
+					#END handle mode
+				# END handle num accesses
+				
+				buf.end_access()
+				assert manager.num_file_handles()
+				assert manager.collect()
+				assert manager.num_file_handles() == 0
+				elapsed  = time() - st
+				mb = float(1000*1000)
+				mode_str = (access_mode and "slice") or "single byte"
+				sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) 
+			# END handle access mode
 		# END for each manager

smmap/test/test_mman.py

@@ -158,7 +158,7 @@ def test_memman_operation(self):
 			assert not includes_ofs(base_offset+csize)
 		# END while we should do an access
 		elapsed = time() - st
-		mb = 1000 * 1000
+		mb = float(1000 * 1000)
 		sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" 
 						% (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed))