这个问题已经在这里回答过了:Python内存分析器

基本上你可以这样做(引自Guppy-PE):

>>> from guppy import hpy; h=hpy()
>>> h.heap()
Partition of a set of 48477 objects. Total size = 3265516 bytes.
Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
0  25773  53  1612820  49   1612820  49 str
1  11699  24   483960  15   2096780  64 tuple
2    174   0   241584   7   2338364  72 dict of module
3   3478   7   222592   7   2560956  78 types.CodeType
4   3296   7   184576   6   2745532  84 function
5    401   1   175112   5   2920644  89 dict of class
6    108   0    81888   3   3002532  92 dict (no owner)
7    114   0    79632   2   3082164  94 dict of type
8    117   0    51336   2   3133500  96 type
9    667   1    24012   1   3157512  97 __builtin__.wrapper_descriptor
<76 more rows. Type e.g. '_.more' to view.>
>>> h.iso(1,[],{})
Partition of a set of 3 objects. Total size = 176 bytes.
Index  Count   %     Size   % Cumulative  % Kind (class / dict of class)
0      1  33      136  77       136  77 dict (no owner)
1      1  33       28  16       164  93 list
2      1  33       12   7       176 100 int
>>> x=[]
>>> h.iso(x).sp
0: h.Root.i0_modules['__main__'].__dict__['x']
>>>

披露:

• 仅适用于Linux
• 报告当前进程作为一个整体使用的内存，而不是单独的功能

但它的优点在于简单:

import resource
def using(point=""):
usage=resource.getrusage(resource.RUSAGE_SELF)
return '''%s: usertime=%s systime=%s mem=%s mb
'''%(point,usage[0],usage[1],
usage[2]/1024.0 )

只要将using("Label")插入到你想要查看发生了什么的地方。例如

print(using("before"))
wrk = ["wasting mem"] * 1000000
print(using("after"))


>>> before: usertime=2.117053 systime=1.703466 mem=53.97265625 mb
>>> after: usertime=2.12023 systime=1.70708 mem=60.8828125 mb

如果你只想查看一个对象的内存使用情况，(其他问题的答案)

有一个名为Pympler的模块，它包含asizeof . bb0 模块。< / p >

使用方法如下:

from pympler import asizeof
asizeof.asizeof(my_object)

与sys.getsizeof不同，它适用于您自己创建的对象。

>>> asizeof.asizeof(tuple('bcd'))
200
>>> asizeof.asizeof({'foo': 'bar', 'baz': 'bar'})
400
>>> asizeof.asizeof({})
280
>>> asizeof.asizeof({'foo':'bar'})
360
>>> asizeof.asizeof('foo')
40
>>> asizeof.asizeof(Bar())
352
>>> asizeof.asizeof(Bar().__dict__)
280

>>> help(asizeof.asizeof)
Help on function asizeof in module pympler.asizeof:


asizeof(*objs, **opts)
Return the combined size in bytes of all objects passed as positional arguments.

Python 3.4包含了一个新模块:tracemalloc。它提供了关于哪些代码分配了最多内存的详细统计信息。下面的示例显示了分配内存的前三行。

from collections import Counter
import linecache
import os
import tracemalloc


def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)


print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print('    %s' % line)


other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))




tracemalloc.start()


counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
print('Top prefixes:', counts.most_common(3))


snapshot = tracemalloc.take_snapshot()
display_top(snapshot)

结果如下:

Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
Top 3 lines
#1: scratches/memory_test.py:37: 6527.1 KiB
words = list(words)
#2: scratches/memory_test.py:39: 247.7 KiB
prefix = word[:3]
#3: scratches/memory_test.py:40: 193.0 KiB
counts[prefix] += 1
4 other: 4.3 KiB
Total allocated size: 6972.1 KiB

什么时候内存泄漏不是泄漏?

当计算结束时内存仍然被占用时，这个示例非常好，但有时您的代码会分配大量内存，然后将其全部释放。从技术上讲，这不是内存泄漏，但它使用的内存比您认为的要多。当内存全部释放时，如何跟踪内存使用情况?如果这是您的代码，您可能可以添加一些调试代码，以便在运行时拍摄快照。如果没有，可以在主线程运行时启动后台线程监视内存使用情况。

下面是前面的示例，其中所有代码都被移动到count_prefixes()函数中。当该函数返回时，释放所有内存。我还添加了一些sleep()调用来模拟长时间运行的计算。

from collections import Counter
import linecache
import os
import tracemalloc
from time import sleep




def count_prefixes():
sleep(2)  # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3)  # Shut down time.
return most_common




def main():
tracemalloc.start()


most_common = count_prefixes()
print('Top prefixes:', most_common)


snapshot = tracemalloc.take_snapshot()
display_top(snapshot)




def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)


print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print('    %s' % line)


other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))




main()

当我运行这个版本时，内存使用量从6MB下降到4KB，因为函数在完成时释放了所有内存。

Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
Top 3 lines
#1: collections/__init__.py:537: 0.7 KiB
self.update(*args, **kwds)
#2: collections/__init__.py:555: 0.6 KiB
return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
#3: python3.6/heapq.py:569: 0.5 KiB
result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
10 other: 2.2 KiB
Total allocated size: 4.0 KiB

现在这里有一个受另一个答案启发的版本，它启动第二个线程来监视内存使用情况。

from collections import Counter
import linecache
import os
import tracemalloc
from datetime import datetime
from queue import Queue, Empty
from resource import getrusage, RUSAGE_SELF
from threading import Thread
from time import sleep


def memory_monitor(command_queue: Queue, poll_interval=1):
tracemalloc.start()
old_max = 0
snapshot = None
while True:
try:
command_queue.get(timeout=poll_interval)
if snapshot is not None:
print(datetime.now())
display_top(snapshot)


return
except Empty:
max_rss = getrusage(RUSAGE_SELF).ru_maxrss
if max_rss > old_max:
old_max = max_rss
snapshot = tracemalloc.take_snapshot()
print(datetime.now(), 'max RSS', max_rss)




def count_prefixes():
sleep(2)  # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3)  # Shut down time.
return most_common




def main():
queue = Queue()
poll_interval = 0.1
monitor_thread = Thread(target=memory_monitor, args=(queue, poll_interval))
monitor_thread.start()
try:
most_common = count_prefixes()
print('Top prefixes:', most_common)
finally:
queue.put('stop')
monitor_thread.join()




def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)


print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print('    %s' % line)


other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))




main()

resource模块让你检查当前内存使用情况，并保存峰值内存使用情况下的快照。该队列让主线程告诉内存监控器线程何时打印报告并关闭。当它运行时，它会显示list()调用所使用的内存:

2018-05-29 10:34:34.441334 max RSS 10188
2018-05-29 10:34:36.475707 max RSS 23588
2018-05-29 10:34:36.616524 max RSS 38104
2018-05-29 10:34:36.772978 max RSS 45924
2018-05-29 10:34:36.929688 max RSS 46824
2018-05-29 10:34:37.087554 max RSS 46852
Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
2018-05-29 10:34:56.281262
Top 3 lines
#1: scratches/scratch.py:36: 6527.0 KiB
words = list(words)
#2: scratches/scratch.py:38: 16.4 KiB
prefix = word[:3]
#3: scratches/scratch.py:39: 10.1 KiB
counts[prefix] += 1
19 other: 10.8 KiB
Total allocated size: 6564.3 KiB

如果你在Linux上，你可能会发现/proc/self/statm模块比resource模块更有用。

可能有帮助:
& lt; 看到额外的 > < / p >

pip install gprof2dot
sudo apt-get install graphviz


gprof2dot -f pstats profile_for_func1_001 | dot -Tpng -o profile.png


def profileit(name):
"""
@profileit("profile_for_func1_001")
"""
def inner(func):
def wrapper(*args, **kwargs):
prof = cProfile.Profile()
retval = prof.runcall(func, *args, **kwargs)
# Note use of name from outer scope
prof.dump_stats(name)
return retval
return wrapper
return inner


@profileit("profile_for_func1_001")
def func1(...)

下面是一个简单的函数装饰器，它可以跟踪函数调用之前，函数调用之后进程消耗了多少内存，以及有什么区别:

import time
import os
import psutil
 

 

def elapsed_since(start):
return time.strftime("%H:%M:%S", time.gmtime(time.time() - start))
 

 

def get_process_memory():
process = psutil.Process(os.getpid())
mem_info = process.memory_info()
return mem_info.rss
 

 

def profile(func):
def wrapper(*args, **kwargs):
mem_before = get_process_memory()
start = time.time()
result = func(*args, **kwargs)
elapsed_time = elapsed_since(start)
mem_after = get_process_memory()
print("{}: memory before: {:,}, after: {:,}, consumed: {:,}; exec time: {}".format(
func.__name__,
mem_before, mem_after, mem_after - mem_before,
elapsed_time))
return result
return wrapper

这是我的博客描述了所有的细节。(归档链接)

由于公认的答案和第二高的投票答案，在我看来，有一些问题，我想提供一个更多的答案，这是密切基于Ihor B。在回答中做了一些小而重要的修改。

这个解决方案允许你在要么上运行剖析，方法是用profile函数包装一个函数调用，然后用@profile装饰器装饰你的函数/方法来调用或。

当您希望在不破坏源代码的情况下分析某些第三方代码时，第一种技术非常有用，而当您不介意修改想要分析的函数/方法的源代码时，第二种技术稍微“干净”一些，效果更好。

我还修改了输出，以便获得RSS、VMS和共享内存。我不太关心“之前”和“之后”的值，但只关心delta，所以我删除了那些(如果你是比较Ihor B。的回答)。

分析代码

# profile.py
import time
import os
import psutil
import inspect




def elapsed_since(start):
#return time.strftime("%H:%M:%S", time.gmtime(time.time() - start))
elapsed = time.time() - start
if elapsed < 1:
return str(round(elapsed*1000,2)) + "ms"
if elapsed < 60:
return str(round(elapsed, 2)) + "s"
if elapsed < 3600:
return str(round(elapsed/60, 2)) + "min"
else:
return str(round(elapsed / 3600, 2)) + "hrs"




def get_process_memory():
process = psutil.Process(os.getpid())
mi = process.memory_info()
return mi.rss, mi.vms, mi.shared




def format_bytes(bytes):
if abs(bytes) < 1000:
return str(bytes)+"B"
elif abs(bytes) < 1e6:
return str(round(bytes/1e3,2)) + "kB"
elif abs(bytes) < 1e9:
return str(round(bytes / 1e6, 2)) + "MB"
else:
return str(round(bytes / 1e9, 2)) + "GB"




def profile(func, *args, **kwargs):
def wrapper(*args, **kwargs):
rss_before, vms_before, shared_before = get_process_memory()
start = time.time()
result = func(*args, **kwargs)
elapsed_time = elapsed_since(start)
rss_after, vms_after, shared_after = get_process_memory()
print("Profiling: {:>20}  RSS: {:>8} | VMS: {:>8} | SHR {"
":>8} | time: {:>8}"
.format("<" + func.__name__ + ">",
format_bytes(rss_after - rss_before),
format_bytes(vms_after - vms_before),
format_bytes(shared_after - shared_before),
elapsed_time))
return result
if inspect.isfunction(func):
return wrapper
elif inspect.ismethod(func):
return wrapper(*args,**kwargs)

示例用法，假设上面的代码保存为profile.py:

from profile import profile
from time import sleep
from sklearn import datasets # Just an example of 3rd party function call




# Method 1
run_profiling = profile(datasets.load_digits)
data = run_profiling()


# Method 2
@profile
def my_function():
# do some stuff
a_list = []
for i in range(1,100000):
a_list.append(i)
return a_list




res = my_function()

这将导致类似于下面的输出:

Profiling:        <load_digits>  RSS:   5.07MB | VMS:   4.91MB | SHR  73.73kB | time:  89.99ms
Profiling:        <my_function>  RSS:   1.06MB | VMS:   1.35MB | SHR       0B | time:   8.43ms

最后有几点重要的说明:

1. 请记住，这种分析方法只是近似的，因为机器上可能会发生许多其他事情。由于垃圾收集和其他因素，delta值甚至可能为零。
由于一些未知的原因，非常短的函数调用(例如1或2毫秒) 显示为零内存使用量。我怀疑这是某种限制 硬件/操作系统(在使用Linux的基本笔记本电脑上测试)的频率 更新内存统计信息。李< / > 为了保持示例简单，我没有使用任何函数参数，但它们应该像人们期望的那样工作，即。 profile(my_function, arg)配置my_function(arg)

一个简单的例子，使用memory_profile计算一个代码块/函数的内存使用情况，同时返回函数的结果:

import memory_profiler as mp


def fun(n):
tmp = []
for i in range(n):
tmp.extend(list(range(i*i)))
return "XXXXX"

在运行代码之前计算内存使用量，然后在代码期间计算最大使用量:

start_mem = mp.memory_usage(max_usage=True)
res = mp.memory_usage(proc=(fun, [100]), max_usage=True, retval=True)
print('start mem', start_mem)
print('max mem', res[0][0])
print('used mem', res[0][0]-start_mem)
print('fun output', res[1])

在运行函数时计算采样点的使用情况:

res = mp.memory_usage((fun, [100]), interval=.001, retval=True)
print('min mem', min(res[0]))
print('max mem', max(res[0]))
print('used mem', max(res[0])-min(res[0]))
print('fun output', res[1])

学分:@skeept