more memory savings

lib/oelite/baker.py

@@ -422,6 +422,11 @@ def bake(self):
 
         rusage.end()
 
+        # We've done most of the expansion we'll ever do, so the dicts
+        # used for interning dependency sets and (value,
+        # dependency_set) tuples have served their purpose.
+        oelite.meta.dict.drop_caches()
+
         if count != total:
             print ""
             self.runq.print_metahashable_tasks()

lib/oelite/cookbook.py

@@ -32,7 +32,7 @@ def __init__(self, baker):
         self.db = sqlite.connect(":memory:")
         if not self.db:
             raise Exception("could not create in-memory sqlite db")
-        self.db.text_factory = str
+        self.db.text_factory = intern
         self.dbc = CursorWrapper(self.db.cursor(), profile=False)
         self.init_db()
         self.recipes = {}

lib/oelite/meta/dict.py

@@ -17,6 +17,31 @@ def deepcopy_str(x, memo):
 copy._deepcopy_dispatch[str] = deepcopy_str
 
 
+frozenset_cache = {}
+dep_tuple_cache = {}
+
+def intern_frozenset(it):
+    f = frozenset(it)
+    try:
+        return frozenset_cache[f]
+    except KeyError:
+        frozenset_cache[f] = f
+        return f
+def intern_dep_tuple(val, deps):
+    f = intern_frozenset(deps)
+    t = (val, f)
+    try:
+        return dep_tuple_cache[t]
+    except KeyError:
+        dep_tuple_cache[t] = t
+        return t
+
+def drop_caches():
+    global frozenset_cache
+    global dep_tuple_cache
+    frozenset_cache = {}
+    dep_tuple_cache = {}
+
 def unpickle(file):
     return DictMeta(meta=file)
 
@@ -217,10 +242,8 @@ def _get(self, var, expand=FULL_EXPANSION):
         override_dep = None
         if var in self.cplx and "__overrides" in self.cplx[var]:
             current_overrides, override_dep = self._get_overrides()
-            if override_dep:
-                override_dep.add("OVERRIDES")
-            else:
-                override_dep = set(["OVERRIDES"])
+            override_dep = set(override_dep)
+            override_dep.add("OVERRIDES")
             olist = self.cplx[var]["__overrides"]
             var_overrides = olist[self.OVERRIDE_TYPE['']] or {}
             append_overrides = olist[self.OVERRIDE_TYPE['>']] or {}
@@ -277,10 +300,10 @@ def _get(self, var, expand=FULL_EXPANSION):
 
         if override_dep:
             deps = deps.union(override_dep)
-        if not deps:
-            deps = None
-        self.expand_cache[var] = (val, deps)
-        return (val, deps)
+        t = intern_dep_tuple(val, deps)
+        self.expand_cache[var] = t
+        return t
+
 
     def _fill_expand_cache(self):
         if self.expand_cache_filled:

lib/oelite/meta/meta.py

@@ -33,28 +33,15 @@ def print_details(self):
         print "Expansion stack:\n%s"%(str(self.stack))
 
 
-class ExpansionStack:
-
-    def __init__(self):
-        self.stack = []
-        self.python = False
-        return
-
+class ExpansionStack(list):
     def push(self, var):
-        if var in self.stack:
-            raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self.stack))))
-        self.stack.append(var)
-        return
+        if var in self:
+            raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self))))
+        self.append(var)
 
-    def pop(self):
-        del self.stack[-1:]
-        return
-
-    def __len__(self):
-        return len(self.stack)
-
-    def __str__(self, prefix="  "):
-        return prefix + ("\n%s"%(prefix)).join(self.stack)
+    def __str__(self):
+        prefix = "  "
+        return prefix + ("\n%s"%(prefix)).join(self)
 
 
 pythonfunc_code_cache = {}
@@ -170,7 +157,6 @@ def get_boolean_flag(self, var, flag, **kwargs):
 
 
     def pythonfunc_init(self):
-        self.pythonfunc_cache = {}
         imports = (self.get("OE_IMPORTS", expand=False) or "")
         g = {}
         g["__builtins__"] = globals()["__builtins__"]
@@ -190,12 +176,9 @@ def get_pythonfunc_globals(self):
 
 
     def get_pythonfunc(self, var, name=None, tmpdir=None, set_os_environ=True):
-        #if function in self.pythonfunc_cache:
-        #    return self.pythonfunc_cache[function]
         function = oelite.function.PythonFunction(
             self, var, name=name, tmpdir=tmpdir,
             set_os_environ=set_os_environ)
-        #self.pythonfunc_cache[function] = function
         return function
 
 

lib/oelite/package.py

@@ -38,7 +38,7 @@ def get_provides(self):
             provides.add('%s:%s'%(package_type, self.name))
         else:
             provides.add(self.name)
-        return provides
+        return map(intern, provides)
 
     def get_recprovides(self, deptype, get_depends):
         depends = self.recipe.get('%s_%s'%(deptype, self.name))