Skip to content

ENH: LineStats aggregation #380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Erotemic merged 5 commits into from
Aug 14, 2025
Merged

ENH: LineStats aggregation #380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6713fe7
Stat aggregation
TTsangSC Aug 10, 2025
07ef721
Tests
TTsangSC Aug 10, 2025
0e281ff
Speed up test
TTsangSC Aug 10, 2025
997af32
CHANGELOG entry
TTsangSC Aug 10, 2025
eef1d3b
Fix lint
TTsangSC Aug 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ Changes
* ``kernprof`` and ``python -m line_profiler`` CLI options
* ``GlobalProfiler`` configurations, and
* profiler output (e.g. ``LineProfiler.print_stats()``) formatting
* ENH: Added capability to combine profiling data both programmatically (``LineStats.__add__()``) and via the CLI (``python -m line_profiler``) (#380, originally proposed in #219)
Copy link
Collaborator

@Theelx Theelx Aug 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this may belong in a new area for 5.1.0 (not 5.0.1 because enhancements are minor version bumps according to SemVer)!

Copy link
Member

@Erotemic Erotemic Aug 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, when that happens, bump the version in the 3 required places (in 6.x I want to consolidate this into a single place: the __init__.py file).

Copy link
Collaborator Author

@TTsangSC TTsangSC Aug 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this PR I didn't feel comfortable presuming to bump the minor version – since I was under the assumption that there will be a 5.0.1 before that. But of course it makes better sense as far as semver goes, and if you already intended to do a minor bump I'm more than happy to oblige. The question now I guess will be whether we bump here and now, or stash this away in a dev/5.1.0 branch somewhere (which we don't currently have) until that happens.

Apropos, in principle the unification of the versions can already happen now without a major-version bump, since the days where kernprof could function independently of the line_profiler has been long gone. We can probably just define __version__ once somewhere and import it elsewhere for now, and just remove it where not explicitly needed in 6.0...


4.2.0
~~~~~
Expand Down
4 changes: 2 additions & 2 deletions line_profiler/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,14 +251,14 @@ def main():
# NOTE: This needs to be in sync with ../kernprof.py and line_profiler.py
__version__ = '5.0.1'

from .line_profiler import (LineProfiler,
from .line_profiler import (LineProfiler, LineStats,
load_ipython_extension, load_stats, main,
show_func, show_text,)


from .explicit_profiler import profile


__all__ = ['LineProfiler', 'line_profiler',
__all__ = ['LineProfiler', 'LineStats', 'line_profiler',
'load_ipython_extension', 'load_stats', 'main', 'show_func',
'show_text', '__version__', 'profile']
195 changes: 178 additions & 17 deletions line_profiler/line_profiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import functools
import inspect
import linecache
import operator
import os
import pickle
import sys
Expand All @@ -17,7 +18,8 @@
from datetime import datetime

try:
from ._line_profiler import LineProfiler as CLineProfiler
from ._line_profiler import (LineProfiler as CLineProfiler,
LineStats as CLineStats)
except ImportError as ex:
raise ImportError(
'The line_profiler._line_profiler c-extension is not importable. '
Expand Down Expand Up @@ -186,6 +188,169 @@ def __init__(self, func, profiler_id):
self.profiler_id = profiler_id


class LineStats(CLineStats):
def __repr__(self):
return '{}({}, {:.2G})'.format(
type(self).__name__, self.timings, self.unit)

def __eq__(self, other):
"""
Example:
>>> from copy import deepcopy
>>>
>>>
>>> stats1 = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 2, 1000), (12, 1, 500)]},
... 1E-6)
>>> stats2 = deepcopy(stats1)
>>> assert stats1 == stats2 is not stats1
>>> stats2.timings = 1E-7
>>> assert stats2 != stats1
>>> stats3 = deepcopy(stats1)
>>> assert stats1 == stats3 is not stats1
>>> stats3.timings['foo', 1, 'spam.py'][:] = [(2, 11, 330)]
>>> assert stats3 != stats1
"""
for attr in 'timings', 'unit':
getter = operator.attrgetter(attr)
try:
if getter(self) != getter(other):
return False
except (AttributeError, TypeError):
return NotImplemented
return True

def __add__(self, other):
"""
Example:
>>> stats1 = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 2, 1000), (12, 1, 500)]},
... 1E-6)
>>> stats2 = LineStats(
... {('bar', 10, 'spam.py'):
... [(11, 10, 20000), (12, 5, 1000)],
... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]},
... 1E-7)
>>> stats_sum = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 12, 3000), (12, 6, 600)],
... ('baz', 5, 'eggs.py'): [(5, 2, 500)]},
... 1E-6)
>>> assert stats1 + stats2 == stats2 + stats1 == stats_sum
"""
timings, unit = self._get_aggregated_timings([self, other])
return type(self)(timings, unit)

def __iadd__(self, other):
"""
Example:
>>> stats1 = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 2, 1000), (12, 1, 500)]},
... 1E-6)
>>> stats2 = LineStats(
... {('bar', 10, 'spam.py'):
... [(11, 10, 20000), (12, 5, 1000)],
... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]},
... 1E-7)
>>> stats_sum = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 12, 3000), (12, 6, 600)],
... ('baz', 5, 'eggs.py'): [(5, 2, 500)]},
... 1E-6)
>>> address = id(stats2)
>>> stats2 += stats1
>>> assert id(stats2) == address
>>> assert stats2 == stats_sum
"""
self.timings, self.unit = self._get_aggregated_timings([self, other])
return self

def print(self, stream=None, **kwargs):
show_text(self.timings, self.unit, stream=stream, **kwargs)

def to_file(self, filename):
""" Pickle the instance to the given filename.
"""
with open(filename, 'wb') as f:
pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)

@classmethod
def from_files(cls, file, /, *files):
"""
Utility function to load an instance from the given filenames.
"""
stats_objs = []
for file in [file, *files]:
with open(file, 'rb') as f:
stats_objs.append(pickle.load(f))
return cls.from_stats_objects(*stats_objs)

@classmethod
def from_stats_objects(cls, stats, /, *more_stats):
"""
Example:
>>> stats1 = LineStats(
... {('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 2, 1000), (12, 1, 500)]},
... 1E-6)
>>> stats2 = LineStats(
... {('bar', 10, 'spam.py'):
... [(11, 10, 20000), (12, 5, 1000)],
... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]},
... 1E-7)
>>> stats_combined = LineStats.from_stats_objects(
... stats1, stats2)
>>> assert stats_combined.unit == 1E-6
>>> assert stats_combined.timings == {
... ('foo', 1, 'spam.py'): [(2, 10, 300)],
... ('bar', 10, 'spam.py'):
... [(11, 12, 3000), (12, 6, 600)],
... ('baz', 5, 'eggs.py'): [(5, 2, 500)]}
"""
timings, unit = cls._get_aggregated_timings([stats, *more_stats])
return cls(timings, unit)

@staticmethod
def _get_aggregated_timings(stats_objs):
if not stats_objs:
raise ValueError(f'stats_objs = {stats_objs!r}: empty')
try:
stats, = stats_objs
except ValueError: # > 1 obj
# Add from small scaling factors to large to minimize
# rounding errors
stats_objs = sorted(stats_objs, key=operator.attrgetter('unit'))
unit = stats_objs[-1].unit
# type: dict[tuple[str, int, int], dict[int, tuple[int, float]]
timing_dict = {}
for stats in stats_objs:
factor = stats.unit / unit
for key, entries in stats.timings.items():
entry_dict = timing_dict.setdefault(key, {})
for lineno, nhits, time in entries:
prev_nhits, prev_time = entry_dict.get(lineno, (0, 0))
entry_dict[lineno] = (prev_nhits + nhits,
prev_time + factor * time)
timings = {
key: [(lineno, nhits, int(round(time, 0)))
for lineno, (nhits, time) in sorted(entry_dict.items())]
for key, entry_dict in timing_dict.items()}
else:
timings = {key: entries.copy()
for key, entries in stats.timings.items()}
unit = stats.unit
return timings, unit


class LineProfiler(CLineProfiler, ByCountProfilerMixin):
"""
A profiler that records the execution times of individual lines.
Expand Down Expand Up @@ -296,24 +461,24 @@ def _debug(self, msg):
msg = f'{self_repr}: {msg}'
logger.debug(msg)

def get_stats(self):
return LineStats.from_stats_objects(super().get_stats())

def dump_stats(self, filename):
""" Dump a representation of the data to a file as a pickled
:py:class:`~.LineStats` object from :py:meth:`~.get_stats()`.
"""
lstats = self.get_stats()
with open(filename, 'wb') as f:
pickle.dump(lstats, f, pickle.HIGHEST_PROTOCOL)
self.get_stats().to_file(filename)

def print_stats(self, stream=None, output_unit=None, stripzeros=False,
details=True, summarize=False, sort=False, rich=False, *,
config=None):
""" Show the gathered statistics.
"""
lstats = self.get_stats()
show_text(lstats.timings, lstats.unit, output_unit=output_unit,
stream=stream, stripzeros=stripzeros,
details=details, summarize=summarize, sort=sort, rich=rich,
config=config)
self.get_stats().print(
stream=stream, output_unit=output_unit,
stripzeros=stripzeros, details=details, summarize=summarize,
sort=sort, rich=rich, config=config)

def _add_namespace(
self, namespace, *,
Expand Down Expand Up @@ -799,12 +964,7 @@ def show_text(stats, unit, output_unit=None, stream=None, stripzeros=False,
stream.write(line + '\n')


def load_stats(filename):
""" Utility function to load a pickled :py:class:`~.LineStats`
object from a given filename.
"""
with open(filename, 'rb') as f:
return pickle.load(f)
load_stats = LineStats.from_files


def main():
Expand Down Expand Up @@ -846,7 +1006,8 @@ def main():
help='Print a summary of total function time. '
f'(Default: {default.conf_dict["summarize"]})')
add_argument(parser, 'profile_output',
help="'*.lprof' file created by `kernprof`")
nargs='+',
help="'*.lprof' file(s) created by `kernprof`")

args = parser.parse_args()
if args.config:
Expand All @@ -856,7 +1017,7 @@ def main():
if getattr(args, key, None) is None:
setattr(args, key, default)

lstats = load_stats(args.profile_output)
lstats = LineStats.from_files(*args.profile_output)
show_text(lstats.timings, lstats.unit,
output_unit=args.unit,
stripzeros=args.skip_zero,
Expand Down
52 changes: 47 additions & 5 deletions line_profiler/line_profiler.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
import io
import pathlib
from functools import cached_property, partial, partialmethod
from os import PathLike
from types import FunctionType, ModuleType
from typing import TYPE_CHECKING, overload, Callable, Literal, Mapping, TypeVar
from typing import (TYPE_CHECKING,
overload,
Callable, Mapping,
Literal, Self,
Protocol, TypeVar)
try:
from typing import ( # type: ignore[attr-defined] # noqa: F401
ParamSpec)
except ImportError:
from typing_extensions import ParamSpec # noqa: F401
from _typeshed import Incomplete
from ._line_profiler import LineProfiler as CLineProfiler
from ._line_profiler import (LineProfiler as CLineProfiler,
LineStats as CLineStats)
from .profiler_mixin import ByCountProfilerMixin, CLevelCallable
from .scoping_policy import ScopingPolicy, ScopingPolicyDict

Expand All @@ -33,6 +37,42 @@ def load_ipython_extension(ip) -> None:
...


class _StatsLike(Protocol):
timings: Mapping[tuple[str, int, str], # funcname, lineno, filename
list[tuple[int, int, int]]] # lineno, nhits, time
unit: float


class LineStats(CLineStats):
def to_file(self, filename: PathLike[str] | str) -> None:
...

def print(self, stream: Incomplete | None = None, **kwargs) -> None:
...

@classmethod
def from_files(cls, file: PathLike[str] | str, /,
*files: PathLike[str] | str) -> Self:
...

@classmethod
def from_stats_objects(cls, stats: _StatsLike, /,
*more_stats: _StatsLike) -> Self:
...

def __repr__(self) -> str:
...

def __eq__(self, other) -> bool:
...

def __add__(self, other: _StatsLike) -> Self:
...

def __iadd__(self, other: _StatsLike) -> Self:
...


class LineProfiler(CLineProfiler, ByCountProfilerMixin):
@overload
def __call__(self, # type: ignore[overload-overlap]
Expand Down Expand Up @@ -86,6 +126,9 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin):
name: str | None = None) -> Literal[0, 1]:
...

def get_stats(self) -> LineStats:
...

def dump_stats(self, filename) -> None:
...

Expand Down Expand Up @@ -148,8 +191,7 @@ def show_text(stats,
...


def load_stats(filename):
...
load_stats = LineStats.from_files


def main():
Expand Down
Loading
Loading