diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py index acedd77f..472ecf7e 100644 --- a/lib/oelite/baker.py +++ b/lib/oelite/baker.py @@ -422,6 +422,11 @@ def bake(self): rusage.end() + # We've done most of the expansion we'll ever do, so the dicts + # used for interning dependency sets and (value, + # dependency_set) tuples have served their purpose. + oelite.meta.dict.drop_caches() + if count != total: print "" self.runq.print_metahashable_tasks() diff --git a/lib/oelite/cookbook.py b/lib/oelite/cookbook.py index 33f3d089..3e61f364 100644 --- a/lib/oelite/cookbook.py +++ b/lib/oelite/cookbook.py @@ -32,7 +32,7 @@ def __init__(self, baker): self.db = sqlite.connect(":memory:") if not self.db: raise Exception("could not create in-memory sqlite db") - self.db.text_factory = str + self.db.text_factory = intern self.dbc = CursorWrapper(self.db.cursor(), profile=False) self.init_db() self.recipes = {} diff --git a/lib/oelite/meta/dict.py b/lib/oelite/meta/dict.py index 1503eed4..cfae2d7d 100644 --- a/lib/oelite/meta/dict.py +++ b/lib/oelite/meta/dict.py @@ -17,6 +17,31 @@ def deepcopy_str(x, memo): copy._deepcopy_dispatch[str] = deepcopy_str +frozenset_cache = {} +dep_tuple_cache = {} + +def intern_frozenset(it): + f = frozenset(it) + try: + return frozenset_cache[f] + except KeyError: + frozenset_cache[f] = f + return f +def intern_dep_tuple(val, deps): + f = intern_frozenset(deps) + t = (val, f) + try: + return dep_tuple_cache[t] + except KeyError: + dep_tuple_cache[t] = t + return t + +def drop_caches(): + global frozenset_cache + global dep_tuple_cache + frozenset_cache = {} + dep_tuple_cache = {} + def unpickle(file): return DictMeta(meta=file) @@ -217,10 +242,8 @@ def _get(self, var, expand=FULL_EXPANSION): override_dep = None if var in self.cplx and "__overrides" in self.cplx[var]: current_overrides, override_dep = self._get_overrides() - if override_dep: - override_dep.add("OVERRIDES") - else: - override_dep = set(["OVERRIDES"]) + override_dep = set(override_dep) + override_dep.add("OVERRIDES") olist = self.cplx[var]["__overrides"] var_overrides = olist[self.OVERRIDE_TYPE['']] or {} append_overrides = olist[self.OVERRIDE_TYPE['>']] or {} @@ -277,10 +300,10 @@ def _get(self, var, expand=FULL_EXPANSION): if override_dep: deps = deps.union(override_dep) - if not deps: - deps = None - self.expand_cache[var] = (val, deps) - return (val, deps) + t = intern_dep_tuple(val, deps) + self.expand_cache[var] = t + return t + def _fill_expand_cache(self): if self.expand_cache_filled: diff --git a/lib/oelite/meta/meta.py b/lib/oelite/meta/meta.py index 05bc1670..dcca0920 100644 --- a/lib/oelite/meta/meta.py +++ b/lib/oelite/meta/meta.py @@ -33,28 +33,15 @@ def print_details(self): print "Expansion stack:\n%s"%(str(self.stack)) -class ExpansionStack: - - def __init__(self): - self.stack = [] - self.python = False - return - +class ExpansionStack(list): def push(self, var): - if var in self.stack: - raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self.stack)))) - self.stack.append(var) - return + if var in self: + raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self)))) + self.append(var) - def pop(self): - del self.stack[-1:] - return - - def __len__(self): - return len(self.stack) - - def __str__(self, prefix=" "): - return prefix + ("\n%s"%(prefix)).join(self.stack) + def __str__(self): + prefix = " " + return prefix + ("\n%s"%(prefix)).join(self) pythonfunc_code_cache = {} @@ -170,7 +157,6 @@ def get_boolean_flag(self, var, flag, **kwargs): def pythonfunc_init(self): - self.pythonfunc_cache = {} imports = (self.get("OE_IMPORTS", expand=False) or "") g = {} g["__builtins__"] = globals()["__builtins__"] @@ -190,12 +176,9 @@ def get_pythonfunc_globals(self): def get_pythonfunc(self, var, name=None, tmpdir=None, set_os_environ=True): - #if function in self.pythonfunc_cache: - # return self.pythonfunc_cache[function] function = oelite.function.PythonFunction( self, var, name=name, tmpdir=tmpdir, set_os_environ=set_os_environ) - #self.pythonfunc_cache[function] = function return function diff --git a/lib/oelite/package.py b/lib/oelite/package.py index c8f8cb46..9d350471 100644 --- a/lib/oelite/package.py +++ b/lib/oelite/package.py @@ -38,7 +38,7 @@ def get_provides(self): provides.add('%s:%s'%(package_type, self.name)) else: provides.add(self.name) - return provides + return map(intern, provides) def get_recprovides(self, deptype, get_depends): depends = self.recipe.get('%s_%s'%(deptype, self.name))