kenmirrors/asterisk
1
0
Fork 0
mirror of https://github.com/asterisk/asterisk.git synced 2025-09-20 20:44:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

refcounter.py: prevent use of excessive RAM with large refs logs

Browse Source 
When processing a 212MB refs file, refcounter.py used over 3GB of RAM.
This change greatly reduces memory usage in two ways:

* Saving object history in whole lines instead of separated values.
* Not saving normal/skewed/leaked object lists unless they are requested.

ASTERISK-23921 #close
Reported by: Corey Farrell
Review: https://reviewboard.asterisk.org/r/3668/
........

Merged revisions 417480 from http://svn.asterisk.org/svn/asterisk/branches/1.8
........

Merged revisions 417481 from http://svn.asterisk.org/svn/asterisk/branches/11
........

Merged revisions 417483 from http://svn.asterisk.org/svn/asterisk/branches/12


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@417485 65c4cc65-6c06-0410-ace0-fbb531ad65f3
This commit is contained in:
Corey Farrell
2014-06-27 19:18:32 +00:00
parent 15dcaeef82
commit 9a495107b8
1 changed files with 29 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
contrib/scripts/refcounter.py
View File

@@ -23,6 +23,7 @@ import os
from optparse import OptionParser from optparse import OptionParser
def parse_line(line): def parse_line(line):
"""Parse out a line into its constituent parts. """Parse out a line into its constituent parts.
@@ -46,11 +47,11 @@ def parse_line(line):
'function': tokens[5], 'function': tokens[5],
'state': tokens[6], 'state': tokens[6],
'tag': tokens[7], 'tag': tokens[7],
} }
return processed_line return processed_line
def process_file(filename): def process_file(options):
"""The routine that kicks off processing a ref file """The routine that kicks off processing a ref file
Keyword Arguments: Keyword Arguments:
@@ -67,6 +68,7 @@ def process_file(filename):
leaked_objects = [] leaked_objects = []
skewed_objects = [] skewed_objects = []
current_objects = {} current_objects = {}
filename = options.filepath
with open(filename, 'r') as ref_file: with open(filename, 'r') as ref_file:
for line in ref_file: for line in ref_file:
@@ -78,16 +80,25 @@ def process_file(filename):
if obj not in current_objects: if obj not in current_objects:
current_objects[obj] = [] current_objects[obj] = []
if 'constructor' not in parsed_line['state']: if options.skewed and 'constructor' not in parsed_line['state']:
skewed_objects.append(current_objects[obj]) skewed_objects.append((obj, current_objects[obj]))
current_objects[obj].append(parsed_line) current_objects[obj].append("[%s] %s:%s %s: %s %s - [%s]" % (
parsed_line['thread_id'],
parsed_line['file'],
parsed_line['line'],
parsed_line['function'],
parsed_line['delta'],
parsed_line['tag'],
parsed_line['state']))
if 'destructor' in parsed_line['state']: if 'destructor' in parsed_line['state']:
lifetime = current_objects.get(obj) if options.normal:
finished_objects.append(lifetime) finished_objects.append((obj, current_objects[obj]))
del current_objects[obj] del current_objects[obj]
leaked_objects = current_objects.values() if options.leaks:
for key, lines in current_objects.iteritems():
leaked_objects.append((key, lines))
return (finished_objects, leaked_objects, skewed_objects) return (finished_objects, leaked_objects, skewed_objects)
@@ -103,13 +114,9 @@ def print_objects(objects, prefix=""):
print "======== %s Objects ========" % prefix print "======== %s Objects ========" % prefix
print "\n" print "\n"
for obj in objects: for obj in objects:
print "==== %s Object %s history ====" % (prefix, obj[0]['addr']) print "==== %s Object %s history ====" % (prefix, obj[0])
for entry in obj: for line in obj[1]:
print "[%s] %s:%s %s: %s %s - [%s]" % (entry['thread_id'], print line
entry['file'], entry['line'],
entry['function'],
entry['delta'], entry['tag'],
entry['state'])
print "\n" print "\n"
@@ -131,15 +138,19 @@ def main(argv=None):
help="If specified, don't output leaked objects") help="If specified, don't output leaked objects")
parser.add_option("-n", "--suppress-normal", action="store_false", parser.add_option("-n", "--suppress-normal", action="store_false",
dest="normal", default=True, dest="normal", default=True,
help="If specified, don't output objects with a " \ help="If specified, don't output objects with a "
"complete lifetime") "complete lifetime")
parser.add_option("-s", "--suppress-skewed", action="store_false", parser.add_option("-s", "--suppress-skewed", action="store_false",
dest="skewed", default=True, dest="skewed", default=True,
help="If specified, don't output objects with a " \ help="If specified, don't output objects with a "
"skewed lifetime") "skewed lifetime")
(options, args) = parser.parse_args(argv) (options, args) = parser.parse_args(argv)
if not options.leaks and not options.skewed and not options.normal:
print >>sys.stderr, "All options disabled"
return -1
if not os.path.isfile(options.filepath): if not os.path.isfile(options.filepath):
print >>sys.stderr, "File not found: %s" % options.filepath print >>sys.stderr, "File not found: %s" % options.filepath
return -1 return -1
@@ -147,7 +158,7 @@ def main(argv=None):
try: try:
(finished_objects, (finished_objects,
leaked_objects, leaked_objects,
skewed_objects) = process_file(options.filepath) skewed_objects) = process_file(options)
if options.leaks and len(leaked_objects): if options.leaks and len(leaked_objects):
print_objects(leaked_objects, "Leaked") print_objects(leaked_objects, "Leaked")