From fc5ca1ead478e48868d86e5a4defeb50986c77bb Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Thu, 9 Feb 2017 10:57:22 +0100 Subject: [PATCH 1/5] oelite/meta/dict.py: share (value, dependency set) tuples The values we store in the .expand_caches are immutable (if we make a change to one of the vars in the dependency set, the entry gets deleted). This means we may share those tuples among all the metadata instances, saving a significant amount of memory. First, this requires changing the second member to a frozenset, and that reveals a (harmless) bug in the override handling; one shouldn't mutate the dependency set one gets from a _get call (the effect was just to add "OVERRIDES" to the dependency set for the "OVERRIDES" variable itself). If we have other such bugs hiding somewhere, the change to using a frozenset should flush them out as well. Then it's just a matter of creating two dicts for holding the interned frozensets and tuples, using the same strategy as Python uses internally for interning strings. As a bonus, we get to drop the optimization where we used None instead of an empty set, since we now use the singleton empty frozenset. The saving is around 7% (40M for oe bake world), with no measurable effect on initialization time. I've done the usual dump of the metadata before and after to check that this does not affect hashes. The dict holding the interned tuples ends up growing to a few MB, so clear the caches after we've done all the hash computations. That may or may not reduce our memory footprint by those MB, depending on the details of Python's and glibc's memory allocation strategy. --- lib/oelite/baker.py | 5 +++++ lib/oelite/meta/dict.py | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py index acedd77f..472ecf7e 100644 --- a/lib/oelite/baker.py +++ b/lib/oelite/baker.py @@ -422,6 +422,11 @@ def bake(self): rusage.end() + # We've done most of the expansion we'll ever do, so the dicts + # used for interning dependency sets and (value, + # dependency_set) tuples have served their purpose. + oelite.meta.dict.drop_caches() + if count != total: print "" self.runq.print_metahashable_tasks() diff --git a/lib/oelite/meta/dict.py b/lib/oelite/meta/dict.py index 1503eed4..cfae2d7d 100644 --- a/lib/oelite/meta/dict.py +++ b/lib/oelite/meta/dict.py @@ -17,6 +17,31 @@ def deepcopy_str(x, memo): copy._deepcopy_dispatch[str] = deepcopy_str +frozenset_cache = {} +dep_tuple_cache = {} + +def intern_frozenset(it): + f = frozenset(it) + try: + return frozenset_cache[f] + except KeyError: + frozenset_cache[f] = f + return f +def intern_dep_tuple(val, deps): + f = intern_frozenset(deps) + t = (val, f) + try: + return dep_tuple_cache[t] + except KeyError: + dep_tuple_cache[t] = t + return t + +def drop_caches(): + global frozenset_cache + global dep_tuple_cache + frozenset_cache = {} + dep_tuple_cache = {} + def unpickle(file): return DictMeta(meta=file) @@ -217,10 +242,8 @@ def _get(self, var, expand=FULL_EXPANSION): override_dep = None if var in self.cplx and "__overrides" in self.cplx[var]: current_overrides, override_dep = self._get_overrides() - if override_dep: - override_dep.add("OVERRIDES") - else: - override_dep = set(["OVERRIDES"]) + override_dep = set(override_dep) + override_dep.add("OVERRIDES") olist = self.cplx[var]["__overrides"] var_overrides = olist[self.OVERRIDE_TYPE['']] or {} append_overrides = olist[self.OVERRIDE_TYPE['>']] or {} @@ -277,10 +300,10 @@ def _get(self, var, expand=FULL_EXPANSION): if override_dep: deps = deps.union(override_dep) - if not deps: - deps = None - self.expand_cache[var] = (val, deps) - return (val, deps) + t = intern_dep_tuple(val, deps) + self.expand_cache[var] = t + return t + def _fill_expand_cache(self): if self.expand_cache_filled: From 8e73d99db4c766b1f80bd4431e1930476c3c7c7d Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 10 Feb 2017 11:45:50 +0100 Subject: [PATCH 2/5] cookbook.py: use intern as text_factory I was wondering how we ended up creating ~5000 copies of the strings "machine" and "arm-926ejs-linux-gnueabi" (each thus costing about 3M of memory). It turns out these are the .type and .arch members of the OElitePackage instances, and the reason they each have their own copy is that the string objects are fresh out of the sqlite database. Let's ask pysqlite to intern every string it passes back to us. This saves a lot of memory, and is virtually free. --- lib/oelite/cookbook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oelite/cookbook.py b/lib/oelite/cookbook.py index 33f3d089..3e61f364 100644 --- a/lib/oelite/cookbook.py +++ b/lib/oelite/cookbook.py @@ -32,7 +32,7 @@ def __init__(self, baker): self.db = sqlite.connect(":memory:") if not self.db: raise Exception("could not create in-memory sqlite db") - self.db.text_factory = str + self.db.text_factory = intern self.dbc = CursorWrapper(self.db.cursor(), profile=False) self.init_db() self.recipes = {} From d0530e6e833e1eb931c93c43542fa9ba1b2a4037 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 10 Feb 2017 11:54:45 +0100 Subject: [PATCH 3/5] oelite/package.py: intern provides strings Every time we do the '(provides or "").split()', python obviously creates a new set of string objects. We stash the strings returned in the __provides flag for a lot of DEPENDS_* and RDEPENDS_* variables, which means we end up having thousands of copies of strings like "libc-6", "libpthread" etc. etc. Interning the strings before passing them back ends up saving more than 50000 string objects, or about 3M of memory. --- lib/oelite/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oelite/package.py b/lib/oelite/package.py index c8f8cb46..9d350471 100644 --- a/lib/oelite/package.py +++ b/lib/oelite/package.py @@ -38,7 +38,7 @@ def get_provides(self): provides.add('%s:%s'%(package_type, self.name)) else: provides.add(self.name) - return provides + return map(intern, provides) def get_recprovides(self, deptype, get_depends): depends = self.recipe.get('%s_%s'%(deptype, self.name)) From 81c80ac7e777b2121b199303b7dce25423969044 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 10 Feb 2017 12:32:16 +0100 Subject: [PATCH 4/5] oelite/meta/meta.py: removed unused pythonfunc_cache attribute We cache the compiled actual python functions in the global pythonfunc_code_cache; the PythonFunction wrappers are not and AFAICT cannot be cached (since they have mutable state preventing reuse). So delete the unused attribute, saving about 1M for an oe bake world (280 bytes * 4500 MetaData instances). --- lib/oelite/meta/meta.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/oelite/meta/meta.py b/lib/oelite/meta/meta.py index 05bc1670..90c097bf 100644 --- a/lib/oelite/meta/meta.py +++ b/lib/oelite/meta/meta.py @@ -170,7 +170,6 @@ def get_boolean_flag(self, var, flag, **kwargs): def pythonfunc_init(self): - self.pythonfunc_cache = {} imports = (self.get("OE_IMPORTS", expand=False) or "") g = {} g["__builtins__"] = globals()["__builtins__"] @@ -190,12 +189,9 @@ def get_pythonfunc_globals(self): def get_pythonfunc(self, var, name=None, tmpdir=None, set_os_environ=True): - #if function in self.pythonfunc_cache: - # return self.pythonfunc_cache[function] function = oelite.function.PythonFunction( self, var, name=name, tmpdir=tmpdir, set_os_environ=set_os_environ) - #self.pythonfunc_cache[function] = function return function From f66ccd1049f4792d6310679f0cf747b07fac9ee1 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 10 Feb 2017 14:09:49 +0100 Subject: [PATCH 5/5] oelite/meta/meta.py: make ExpansionStack smaller The memory footprint of an ExpansionStack instance is 72 bytes for the instance itself, 280 bytes for the __dict__ attribute, and then the memory for the actual list stored at the .stack attribute. We can get rid of the first two contributions by simply letting an ExpansionStack be a list with one extra method that does the sanity checking. Conveniently, the builtin list type even already has a .pop method which does the right thing. Also, __init__ and __len__ are already taken care of. Nobody calls __str__ directly allowing them to pass the optional argument and str(foo, "something else") would be a runtime error, so just make __str__ have the usual prototype. The .python attribute never seems to have been used for anything. Saving 352 bytes per instance adds up to about 1.5M for an oe bake world. --- lib/oelite/meta/meta.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/lib/oelite/meta/meta.py b/lib/oelite/meta/meta.py index 90c097bf..dcca0920 100644 --- a/lib/oelite/meta/meta.py +++ b/lib/oelite/meta/meta.py @@ -33,28 +33,15 @@ def print_details(self): print "Expansion stack:\n%s"%(str(self.stack)) -class ExpansionStack: - - def __init__(self): - self.stack = [] - self.python = False - return - +class ExpansionStack(list): def push(self, var): - if var in self.stack: - raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self.stack)))) - self.stack.append(var) - return + if var in self: + raise Exception("Circular expansion: %s"%("->".join(map(lambda x: "${%s}" % x, self)))) + self.append(var) - def pop(self): - del self.stack[-1:] - return - - def __len__(self): - return len(self.stack) - - def __str__(self, prefix=" "): - return prefix + ("\n%s"%(prefix)).join(self.stack) + def __str__(self): + prefix = " " + return prefix + ("\n%s"%(prefix)).join(self) pythonfunc_code_cache = {}