testing: more robust leak test

Python sometimes does a strange memory increase after a few
iterations (even for pure Python test statements), so now we wait
until we got a fixed number of iterations without memory increase.
Simpler than before, and more stable.

diff --git a/tests/test_memory_leak.py b/tests/test_memory_leak.py
index 36860ce..8de7dff 100755 (executable)
--- a/tests/test_memory_leak.py
+++ b/tests/test_memory_leak.py
@@ -23,7 +23,6 @@ def check_garbage(msg = 'uncollectable garbage left over from previous tests'):
     garbage = []
     for obj in gc.garbage:
         # ignore garbage generated by numpy loadtxt command
-        # http://projects.scipy.org/numpy/ticket/1356
         if hasattr(obj, 'filename') and obj.filename == 'painting30sec.dat.gz':
             continue
         garbage.append(obj)
@@ -36,19 +35,30 @@ def leakTest_generic(func):
     doc = document.Document()
     #gc.set_debug(gc.DEBUG_LEAK)
 
-    m = []
-    N = 21
-    for i in range(N):
+    max_mem = 0
+    max_mem_stable = 0
+    no_leak = False
+    m1 = 0
+    for i in range(options.max_iterations):
         func(doc, i)
         if options.debug:
             if i == 3:
                 check_garbage()
                 helpers.record_memory_leak_status()
-            if i == 4:
+            if i == 4 or i == 5:
                 helpers.record_memory_leak_status(print_diff=True)
         m2 = mem()
-        m.append(m2)
-        print 'iteration %02d/%02d: %d pages used' % (i+1, N, m2)
+        print 'iteration %02d/%02d: %d pages used (%+d)' % (i+1, options.max_iterations, m2, m2-m1)
+        m1 = m2
+        if m2 > max_mem:
+            max_mem = m2
+            max_mem_stable = 0
+        else:
+            max_mem_stable += 1
+            if max_mem_stable == options.required:
+                print 'maximum was stable for', max_mem_stable, 'iterations'
+                no_leak = True
+                break
 
     #import objgraph
     #from lib import strokemap
@@ -58,18 +68,11 @@ def leakTest_generic(func):
     # note: if gc.DEBUG_LEAK is enabled above this is expected to fail
     check_garbage()
 
-    print m
-    # we also have oscillations for some tests
-    cmp_1st = m[N*1/3:N*2/3]
-    cmp_2nd = m[N*2/3:N*3/3]
-    diff = abs(max(cmp_1st) - max(cmp_2nd))
-    #if diff == 1:
-    #    print 'FIXME: known minor leak ignored'
-    #else:
-    if diff != 0:
-        print 'looks like a memory leak in ' + func.__name__
+    if no_leak:
+        print 'no leak found'
+    else:
+        print 'memory leak in ' + func.__name__
         sys.exit(LEAK_EXIT_CODE)
-    print 'no leak found'
 
 
 all_tests = {}
@@ -78,12 +81,25 @@ def leaktest(test_func):
     all_tests[test_func.__name__] = test_func
     return test_func
 
+#@leaktest
+def provoke_leak(doc, iteration):
+    # note: interestingly this leaky only shows in the later iterations
+    #       (and very small leaks might not be detected)
+    setattr(gc, 'my_test_leak_%d' % iteration, zeros(50000))
+
+@leaktest
+def noleak(doc, iteration):
+    setattr(gc, 'my_test_leak', zeros(50000))
+
+@leaktest
+def document_alloc(doc, iteration):
+    document.Document()
+
 @leaktest
 def surface_alloc(doc, iteration):
     tiledsurface.Surface()
 
-@leaktest
-def paint(doc, iteration):
+def paint(doc):
     events = painting30sec_events
     t_old = events[0][0]
     for i, (t, x, y, pressure) in enumerate(events):
@@ -114,20 +130,6 @@ def paint_save_clear(doc, iteration):
     doc.save('test_leak.ora')
     doc.clear()
 
-@leaktest
-def provoke_leak(doc, iteration):
-    # note: interestingly this leaky only shows in the later iterations
-    #       (and smaller leaks will not be detected)
-    setattr(gc, 'my_test_leak_%d' % iteration, zeros(50000))
-
-def leakTest_slow():
-
-    #leakTest_generic(provoke_leak)
-    leakTest_generic(paint_and_clear)
-    leakTest_generic(repeated_saving)
-    leakTest_generic(repeated_loading)
-    leakTest_generic(paint_save_clear)
-
 
 if __name__ == '__main__':
     from optparse import OptionParser
@@ -140,6 +142,10 @@ if __name__ == '__main__':
                       help='print leak analysis (slow)')
     parser.add_option('-e', '--exit', action='store_true', default=False,
                       help='exit at first error')
+    parser.add_option('-r', '--required', type='int', default=15,
+                      help='good iterations required (default: 15)')
+    parser.add_option('-m', '--max-iterations', type='int', default=100,
+                      help='maximum number of iterations (default: 100)')
     options, tests = parser.parse_args()
 
     if options.list:
@@ -147,6 +153,10 @@ if __name__ == '__main__':
             print name
         sys.exit(0)
 
+    if options.required >= options.max_iterations:
+        print 'requiring more good iterations than the iteration limit makes no sense'
+        sys.exit(1)
+
     if not tests:
         if options.all:
             tests = list(all_tests)