Implemented Window including test. Started Region implementation as well as test, but noticed that a critical feature, the mmap's offset, doesn't exist prior to python 2.6. Its total crap, so is python

Author: Byron

README.rst

@@ -15,12 +15,13 @@ For convenience, a stream class is provided which hides the usage of the memory
 ************
 LIMITATIONS
 ************
-The access is readonly by design.
+* The access is readonly by design.
+* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0.
 
 ************
 REQUIREMENTS
 ************
-* Python 2.4 or higher
+* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function.
 
 *******
 Install

smmap/mman.py

@@ -1,3 +1,111 @@
 """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files"""
 
-__all__ = []
+__all__ = ["MappedMemoryManager"]
+
+import os
+import mmap
+
+from mmap import PAGESIZE
+
+#{ Utilities
+
+def align_to_page(num, round_up):
+	"""Align the given integer number to the closest page offset, which usually is 4096 bytes.
+	:param round_up: if True, the next higher multiple of page size is used, otherwise
+		the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0)
+	:return: num rounded to closest page"""
+	res = (num / PAGESIZE) * PAGESIZE;
+	if round_up and (res != num):
+		res += PAGESIZE;
+	#END handle size
+	return res;
+
+#}END utilities
+
+class Window(object):
+	"""Utility type which is used to snap windows towards each other, and to adjust their size"""
+	__slots__ = (
+				'ofs',		# offset into the file in bytes
+				'size'				# size of the window in bytes
+				)
+
+	def __init__(self, offset, size):
+		self.ofs = offset
+		self.size = size
+
+	def __repr__(self):
+		return "Window(%i, %i)" % (self.ofs, self.size) 
+
+	@classmethod
+	def from_region(cls, region):
+		""":return: new window from a region"""
+		return cls(region.ofs_begin(), region.size())
+
+	def ofs_end(self):
+		return self.ofs + self.size
+
+	def align(self):
+		self.ofs = align_to_page(self.ofs, 0)
+		self.size = align_to_page(self.size, 1)
+
+	def extend_left_to(self, window, max_size):
+		"""Adjust the offset to start where the given window on our left ends if possible, 
+		but don't make yourself larger than max_size.
+		The resize will assure that the new window still contains the old window area"""
+		rofs = self.ofs - window.ofs_end()
+		nsize = rofs + self.size
+		rofs -= nsize - min(nsize, max_size)
+		self.ofs = self.ofs - rofs
+		self.size += rofs
+
+	def extend_right_to(self, window, max_size):
+		"""Adjust the size to make our window end where the right window begins, but don't
+		get larger than max_size"""
+		self.size = min(self.size + (window.ofs - self.ofs_end()), max_size)
+
+
+class Region(object):
+	"""Defines a mapped region of memory, aligned to pagesizes
+	:note: deallocates used region automatically on destruction"""
+	__slots__ = (
+					'_b'	, 	# beginning of mapping
+					'_mf',	# mapped memory chunk (as returned by mmap)
+					'_nc',	# number of clients using this region
+					'_uc'	# total amount of usages
+				)
+	
+	
+	def __init__(self, path, ofs, size):
+		"""Initialize a region, allocate the memory map
+		:param path: path to the file to map
+		:param ofs: **aligned** offset into the file to be mapped 
+		:param size: if size is larger then the file on disk, the whole file will be
+			allocated the the size automatically adjusted
+		:raise Exception: if no memory can be allocated"""
+		self._b = ofs
+		self._nc = 0
+		self._uc = 0
+		
+		fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0))
+		try:
+			self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs)
+		finally:
+			os.close(fd)
+		#END close file handle
+	
+
+class MappedMemoryManager(object):
+	"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily 
+	obtain additional regions assuring there is no overlap.
+	Once a certain memory limit is reached globally, or if there cannot be more open file handles 
+	which result from each mmap call, the least recently used, and currently unused mapped regions
+	are unloaded automatically.
+	
+	:note: currently not thread-safe !
+	:note: in the current implementation, we will automatically unload windows if we either cannot
+		create more memory maps (as the open file handles limit is hit) or if we have allocated more than 
+		a safe amount of memory already, which would possibly cause memory allocations to fail as our address
+		space is full."""
+		
+	__slots__ = tuple()
+	

smmap/test/lib.py

@@ -1,9 +1,50 @@
 """Provide base classes for the test system"""
 from unittest import TestCase
+import os
+import tempfile
 
-__all__ = ['TestBase']
+__all__ = ['TestBase', 'FileCreator']
 
 
+#{ Utilities
+
+class FileCreator(object):
+	"""A instance which creates a temporary file with a prefix and a given size
+	and provides this info to the user.
+	Once it gets deleted, it will remove the temporary file as well."""
+	__slots__ = ("_size", "_path")
+	
+	def __init__(self, size, prefix=''):
+		assert size, "Require size to be larger 0"
+		
+		self._path = tempfile.mktemp(prefix=prefix)
+		self._size = size
+		
+		fp = open(self._path, "wb")
+		fp.seek(size-1)
+		fp.write('1')
+		fp.close()
+		
+		assert os.path.getsize(self.path) == size
+
+	def __del__(self):
+		try:
+			os.remove(self.path)
+		except OSError:
+			pass
+		#END exception handling
+	
+
+	@property
+	def path(self):
+		return self._path
+		
+	@property
+	def size(self):
+		return self._size
+
+#} END utilities
+
 class TestBase(TestCase):
 	"""Foundation used by all tests"""
 

smmap/test/test_mman.py

@@ -1,7 +1,70 @@
-from lib import TestBase
+from lib import TestBase, FileCreator
 
 from smmap.mman import *
+from smmap.mman import Region
+from smmap.mman import Window
+
+import sys
+import mmap
 
 class TestMMan(TestBase):
+	
+	_window_test_size = 1000 * 1000 * 8 + 5195
+	
+	def test_window(self):
+		wl = Window(0, 1)	 	# left
+		wc = Window(1, 1)		# center
+		wc2 = Window(10, 5)		# another center
+		wr = Window(8000, 50)	# right
+		
+		assert wl.ofs_end() == 1
+		assert wc.ofs_end() == 2
+		assert wr.ofs_end() == 8050
+		
+		# extension does nothing if already in place
+		maxsize = 100
+		wc.extend_left_to(wl, maxsize)
+		assert wc.ofs == 1 and wc.size == 1
+		wl.extend_right_to(wc, maxsize)
+		wl.extend_right_to(wc, maxsize)
+		assert wl.ofs == 0 and wl.size == 1
+		
+		# an actual left extension
+		pofs_end = wc2.ofs_end()
+		wc2.extend_left_to(wc, maxsize)
+		assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() 
+		
+		
+		# respects maxsize
+		wc.extend_right_to(wr, maxsize)
+		assert wc.ofs == 1 and wc.size == maxsize
+		wc.extend_right_to(wr, maxsize)
+		assert wc.ofs == 1 and wc.size == maxsize
+		
+		# without maxsize
+		wc.extend_right_to(wr, sys.maxint)
+		assert wc.ofs_end() == wr.ofs and wc.ofs == 1
+		
+		# extend left
+		wr.extend_left_to(wc2, maxsize)
+		wr.extend_left_to(wc2, maxsize)
+		assert wr.size == maxsize
+		
+		wr.extend_left_to(wc2, sys.maxint)
+		assert wr.ofs == wc2.ofs_end()
+		
+		wc.align()
+		assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2
+		
+	
+	def test_region(self):
+		fc = FileCreator(self._window_test_size, "window_test")
+		rfull = Region(fc.path, 0, fc.size)
+		
+		
+		
+		Window.from_region # todo
+		pass
+		
 	def test_basics(self):
-		assert False
+		pass