Ticket #1426: scroll-cythonized.patch

xpra/server/window/motion.pyx

@@ -1 +1 @@
 # coding=utf8
 # This file is part of Xpra.
-# Copyright (C) 2016 Antoine Martin <antoine@devloop.org.uk>
+# Copyright (C) 2016-2017 Antoine Martin <antoine@devloop.org.uk>
 # Xpra is released under the terms of the GNU GPL v2, or, at your option, any
 # later version. See the file COPYING for details.
 
@@ -9 +9 @@
 
 import os
 import time
+import collections
 
 from xpra.util import envbool
 from xpra.log import Logger
-logger = Logger("encoding")
+log = Logger("encoding", "scroll")
 
-import zlib
+
+cdef int DEBUG = envbool("XPRA_SCROLL_DEBUG", False)
 hashfn = None
 if envbool("XPRA_XXHASH", True):
     try:
@@ -22 +24 @@
         def hashfn(x):
             return xxhash.xxh64(x).intdigest()
     except ImportError as e:
-        logger.warn("Warning: xxhash python bindings not found")
+        log.warn("Warning: xxhash python bindings not found")
 else:
-    logger.warn("Warning: xxhash disabled")
+    log.warn("Warning: xxhash disabled")
 if hashfn is None:
-    logger.warn(" no scrolling detection")
+    log.warn(" no scrolling detection")
 
 
-cdef extern from "math.h":
-    double log(double x)
+from libc.stdint cimport int32_t, uint8_t, uint16_t, int16_t, uint32_t, int64_t
 
-from libc.stdint cimport int32_t, uint8_t, uint32_t, int64_t
-
-cdef extern from "stdlib.h":
-    int abs(int number)
-
 cdef extern from "string.h":
     void free(void * ptr) nogil
     void *memset(void * ptr, int value, size_t num) nogil
-    int memcmp(const void *a1, const void *a2, size_t size)
 
 cdef extern from "../../buffers/memalign.h":
     void *xmemalign(size_t size) nogil
@@ -76 +71 @@
     #assert v>=0, "invalid int to cast: %s" % v
     return v
 
-def calculate_distances(array1, array2, int min_score=0, int max_distance=1000):
-    #print("calculate_distances(..)")
-    assert len(array1)==len(array2)
-    cdef int l = len(array1)
-    cdef int i, y1, y2, miny, maxy, d
-    #we want fast array access,
-    #so cache both arrays in C arrays:
-    assert sizeof(int64_t)==64//8, "uint64_t is not 64-bit: %i!" % sizeof(int64_t)
-    cdef size_t asize = l*(sizeof(int64_t))
-    cdef int64_t *a1 = NULL
-    cdef int64_t *a2 = NULL
-    cdef int64_t a2v = 0
-    cdef int32_t *distances = NULL
-    #print("calculate_distances(%s, %s, %i, %i)" % (array1, array2, elen, min_score))
-    try:
-        a1 = <int64_t*> xmemalign(asize)
-        a2 = <int64_t*> xmemalign(asize)
-        assert a1!=NULL and a2!=NULL, "failed to allocate %i bytes of scroll array memory" % asize
-        for i in range(l):
-            a1[i] = castint64(array1[i])
-            a2[i] = castint64(array2[i])
+cdef inline da(int64_t *a, uint16_t l):
+    return [a[i] for i in range(l)]
+
+cdef inline dd(uint16_t *d, uint16_t l):
+    return [d[i] for i in range(l)]
+
+cdef inline ds(int16_t *d, uint16_t l):
+    return [d[i] for i in range(l)]
+
+
+assert sizeof(int64_t)==64//8, "uint64_t is not 64-bit: %i!" % sizeof(int64_t)
+
+
+cdef class ScrollDistances:
+
+    cdef object __weakref__
+    #for each distance, keep track of the hit count:
+    cdef uint16_t* distances
+    cdef uint16_t l
+    cdef int64_t *a1
+    cdef int64_t *a2
+
+    def init(self, array1, array2, uint16_t max_distance=1000):
+        assert len(array1)==len(array2)
+        assert len(array1)<2**15 and len(array1)>0, "invalid array length: %i" % len(array1)
+        self.l = len(array1)
+        self.distances = <uint16_t*> xmemalign(2*self.l*sizeof(uint16_t))
+        cdef size_t asize = self.l*(sizeof(int64_t))
+        self.a1 = <int64_t*> xmemalign(asize)
+        self.a2 = <int64_t*> xmemalign(asize)
+        assert self.distances!=NULL and self.a1!=NULL and self.a2!=NULL, "scroll memory allocation failed"
+        for i in range(self.l):
+            self.a1[i] = castint64(array1[i])
+            self.a2[i] = castint64(array2[i])
         #now compare all the values
-        distances = <int32_t*> xmemalign(2*l*sizeof(int32_t))
-        assert distances!=NULL
+        self.calculate(max_distance)
+
+    def __repr__(self):
+        return "ScrollDistances(%i)" % self.l
+
+    cdef calculate(self, uint16_t max_distance=1000):
+        cdef int64_t *a1 = self.a1
+        cdef int64_t *a2 = self.a2
+        cdef uint16_t l = self.l
+        cdef uint16_t y1, y2
+        cdef uint16_t miny=0, maxy=0
+        cdef int64_t a2v
         with nogil:
-            memset(<void*> distances, 0, 2*l*sizeof(int32_t))
+            memset(<void*> self.distances, 0, 2*self.l*sizeof(uint16_t))
             for y2 in range(l):
-                miny = max(0, y2-max_distance)
-                maxy = min(l, y2+max_distance)
+                #miny = max(0, y2-max_distance):
+                if y2>max_distance:
+                    miny = y2-max_distance
+                else:
+                    miny = 0
+                #maxy = min(l, y2+max_distance)
+                if y2+max_distance<l:
+                    maxy = y2+max_distance
+                else:
+                    maxy = l
                 a2v = a2[y2]
                 if a2v==0:
                     continue
@@ -111 +137 @@
                 for y1 in range(miny, maxy):
                     if a1[y1]==a2v:
                         #distance = y1-y2
-                        distances[l+y1-y2] += 1
-        r = {}
-        for i in range(2*l):
-            d = distances[i]
-            if abs(d)>=min_score:
-                r[i-l] = d
-        return r
-    finally:
-        if a1!=NULL:
-            free(a1)
-        if a2!=NULL:
-            free(a2)
-        if distances!=NULL:
-            free(distances)
+                        self.distances[l-(y1-y2)] += 1
+        if DEBUG:
+            log("ScrollDistance: l=%i, calculate(%s, %s, %i)=%s", self.l, da(self.a1, self.l), da(self.a2, self.l), max_distance, dd(self.distances, self.l*2))
 
-def match_distance(array1, array2, int distance):
-    assert len(array1)==len(array2)
-    l = len(array1)
-    if distance>=0:
-        return [i for i,v in enumerate(array1) if (i+distance)<l and array2[i+distance]==v]
-    distance = abs(distance)
-    return [i+distance for i,v in enumerate(array2) if (i+distance)<l and array1[i+distance]==v]
+    def get_best_scroll_values(self, uint16_t min_hits=2):
+        DEF MAX_MATCHES = 20
+        cdef uint16_t m_arr[MAX_MATCHES]    #number of hits
+        cdef int16_t s_arr[MAX_MATCHES]     #scroll distance
+        cdef int16_t i
+        cdef uint8_t j
+        memset(<void*> &m_arr, 0, MAX_MATCHES*sizeof(uint16_t))
+        memset(<void*> &s_arr, 0, MAX_MATCHES*sizeof(int16_t))
+        cdef int16_t low = 0
+        cdef int16_t matches
+        cdef uint16_t* distances = self.distances
+        cdef uint16_t l = self.l
+        with nogil:
+            for i in range(2*l):
+                matches = distances[i]
+                if matches>low and matches>min_hits:
+                    #add this candidate match to the arrays:
+                    for j in range(MAX_MATCHES):
+                        if m_arr[j]==low:
+                            break
+                    m_arr[j] = matches
+                    s_arr[j] = i-l
+                    #find the new lowest value:
+                    low = matches
+                    for j in range(MAX_MATCHES):
+                        if m_arr[j]<low:
+                            low = m_arr[j]
+                            if low==0:
+                                break
+        if DEBUG:
+            log("get_best_scroll_values: arrays: matches=%s, scroll=%s", dd(m_arr, MAX_MATCHES), ds(s_arr, MAX_MATCHES))
+        #first collect the list of distances sorted by highest number of matches:
+        #(there can be more than one distance value for each match count):
+        scroll_hits = {}
+        for i in range(MAX_MATCHES):
+            if m_arr[i]>min_hits:
+                scroll_hits.setdefault(m_arr[i], []).append(s_arr[i])
+        if DEBUG:
+            log("scroll hits=%s", scroll_hits)
+        #return a dict with the scroll distance as key,
+        #and the list of matching lines in a dictionary:
+        # {line-start : count, ..}
+        #this is destructive as we clear the checksums after use
+        scrolls = collections.OrderedDict()
+        cdef uint16_t m
+        #starting with the highest matches
+        for m in reversed(sorted(scroll_hits.keys())):
+            v = scroll_hits[m]
+            for scroll in v:
+                #find matching lines:
+                line_defs = self.match_distance(scroll)
+                if line_defs:
+                    scrolls[scroll] = line_defs
+        return scrolls
 
-def consecutive_lines(line_numbers):
-    #print("line_numbers_to_rectangles(%s)" % (line_numbers, ))
-    #aggregates consecutive lines:
-    #[1,2,3,4,8,9] -> [(1,3), (8,1)]
-    assert len(line_numbers)>0
-    if len(line_numbers)==1:
-        return [(line_numbers[0], 1)]
-    cdef int start = line_numbers[0]
-    cdef int last = start
-    cdef int line = 0
-    r = []
-    for line in line_numbers[1:]:
-        if line!=last+1:
-            #new rectangle
-            r.append((start, last-start+1))
-            start = line
-        last = line
-    if last!=line_numbers[0]:
-        r.append((start, last-start+1))
-    return r
+    def get_remaining_areas(self):
+        #all the lines which have not been zeroed out
+        #when we matched them with match_distance
+        cdef int64_t *a2 = self.a2
+        cdef uint16_t i, start = 0, count = 0
+        line_defs = collections.OrderedDict()
+        for i in range(self.l):
+            if a2[i]!=0:
+                if count==0:
+                    start = i
+                count += 1
+            elif count>0:
+                line_defs[start] = count
+                count = 0
+        if count>0:
+            line_defs[start] = count
+        return line_defs
+
+    def match_distance(self, int16_t distance):
+        """ find the lines that match the given scroll distance """
+        cdef int64_t *a1 = self.a1
+        cdef int64_t *a2 = self.a2
+        cdef char swap = 0
+        if distance<0:
+            #swap order:
+            swap = 1
+            a1 = self.a2
+            a2 = self.a1
+            distance = -distance
+        if DEBUG:
+            log("match_distance(%i) l=%i, a1=%s, a2=%s", distance, self.l, da(a1, self.l), da(a2, self.l))
+        assert distance<self.l, "invalid distance %i for size %i" % (distance, self.l)
+        cdef uint16_t i, start = 0, count = 0
+        line_defs = collections.OrderedDict()
+        for i in range(self.l-distance):
+            #if DEBUG:
+            #    log("%i: a1=%i / a2=%i", i, a1[i], a2[i+distance])
+            if a1[i]!=0 and a1[i]==a2[i+distance]:
+                #if DEBUG:
+                #    log("match at %i: %i", i, a1[i])
+                if count==0:
+                    #first match
+                    start = i
+                count += 1
+                #mark the target line as dealt with:
+                if swap:
+                    a1[i] = 0
+                else:
+                    a2[i+distance] = 0
+            elif count>0:
+                #we had a match
+                line_defs[start] = count
+                count = 0
+        if count>0:
+            #last few lines ended as a match:
+            line_defs[start] = count
+        if DEBUG:
+            log("match_distance(%i)=%s", distance, line_defs)
+        return line_defs
+
+
+    def get_best_match(self):
+        cdef int16_t max = 0
+        cdef int d = 0
+        cdef unsigned int i
+        for i in range(2*self.l):
+            if self.distances[i]>max:
+                max = self.distances[i]
+                d = i-self.l
+        return d, max
+
+    def __dealloc__(self):
+        cdef void* ptr = <void*> self.distances
+        if ptr:
+            self.distances = NULL
+            free(ptr)
+        ptr = <void*> self.a1
+        if ptr:
+            self.a1 = NULL
+            free(ptr)
+        ptr = <void*> self.a2
+        if ptr:
+            self.a2 = NULL
+            free(ptr)
+        
+
+def scroll_distances(array1, array2, unsigned int min_score=0, uint16_t max_distance=1000):
+    cdef ScrollDistances sd = ScrollDistances()
+    sd.init(array1, array2, max_distance)
+    return sd

xpra/server/window/window_video_source.py

@@ -14 +14 @@
 from xpra.codecs.codec_constants import TransientCodecException, RGB_FORMATS, PIXEL_SUBSAMPLING
 from xpra.server.window.window_source import WindowSource, STRICT_MODE, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY
 from xpra.server.window.region import rectangle, merge_all          #@UnresolvedImport
-from xpra.server.window.motion import match_distance, consecutive_lines, calculate_distances, CRC_Image #@UnresolvedImport
+from xpra.server.window.motion import scroll_distances, CRC_Image #@UnresolvedImport
 from xpra.server.window.video_subregion import VideoSubregion, VIDEO_SUBREGION
 from xpra.server.window.video_scoring import get_pipeline_score
 from xpra.codecs.loader import PREFERED_ENCODING_ORDER, EDGE_ENCODING_ORDER
@@ -1453 +1453 @@
         return packet
 
 
-    def encode_scrolling(self, image, distances, old_csums, csums, options):
+    def encode_scrolling(self, image, distances, options):
         start = time.time()
         try:
             del options["av-sync"]
@@ -1460 +1460 @@
         except:
             pass
         #tells make_data_packet not to invalidate the scroll data:
-        scrolllog("encode_scrolling(%s, {..}, [], [], %s) window-dimensions=%s", image, options, self.window_dimensions)
+        scrolllog("encode_scrolling(%s, %s, [], [], %s) window-dimensions=%s", image, distances, options, self.window_dimensions)
         x, y, w, h = image.get_geometry()[:4]
-        yscroll_values = []
-        max_scroll_regions = 50
-        #process distances with the highest score first:
-        for hits in reversed(sorted(distances.values())):
-            for scroll in (d for d,v in distances.items() if v==hits):
-                assert scroll<h
-                yscroll_values.append(scroll)
-            if len(yscroll_values)>=max_scroll_regions:
-                break
-        assert yscroll_values
-        #always add zero=no-change so we can drop those updates!
-        if 0 not in yscroll_values and 0 in distances:
-            #(but do this last so we don't end up cutting too many rectangles)
-            yscroll_values.append(0)
-        scrolllog(" will send scroll packets for yscroll=%s", yscroll_values)
-        #keep track of the lines we have handled already:
-        #(the same line may be available from multiple scroll directions)
-        handled = set()
+        raw_scroll = distances.get_best_scroll_values()
+        non_scroll = distances.get_remaining_areas()
+        scrolllog(" will send scroll data=%s, non-scroll=%s", raw_scroll, non_scroll)
+        flush = len(non_scroll)
+        assert raw_scroll, "failed to detect scroll values"
+        #convert to a screen rectangle list for the client:
         scrolls = []
-        max_scrolls = 1000
-        for s in yscroll_values:
-            #find all the lines that scroll by this much:
-            slines = match_distance(old_csums, csums, s)
-            assert slines, "no lines matching distance %i" % s
-            #remove any lines we have already handled:
-            lines = [v+s for v in slines if ((v+s) not in handled and v not in handled)]
-            if not lines:
-                continue
-            #and them to the handled set so we don't try to paint them again:
-            handled = handled.union(set(lines))
-            if s==0:
-                scrolllog(" %i lines have not changed", len(lines))
-            else:
-                #things have actually moved
-                #aggregate consecutive lines into rectangles:
-                cl = consecutive_lines(lines)
-                #scrolllog(" scroll groups for distance=%i : %s=%s", s, lines, cl)
-                scrolllog(" scroll groups for distance=%i : %s", s, cl)
-                for sy,sh in cl:
-                    #new rectangle
-                    scrolls.append((x, y+sy-s, w, sh, 0, s))
-                    if len(scrolls)>max_scrolls:
-                        break
-                if len(scrolls)>max_scrolls:
-                    break
-
-        non_scroll = []
-        remaining = set(range(h))-handled
-        if remaining:
-            damaged_lines = sorted(list(remaining))
-            non_scroll = consecutive_lines(damaged_lines)
-            scrolllog(" non scroll: %i packets: %s", len(non_scroll), non_scroll)
-        flush = len(non_scroll)
-        #send as scroll paints packets:
-        if scrolls:
-            client_options = options.copy()
-            if flush>0 and self.supports_flush:
-                client_options["flush"] = flush
-            coding = "scroll"
-            end = time.time()
-            packet = self.make_draw_packet(x, y, w, h, coding, LargeStructure(coding, scrolls), 0, client_options, options)
-            self.queue_damage_packet(packet)
-            compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %6s as %3i rectangles  (%5iKB)           , sequence %5i, client_options=%s",
-                 (end-start)*1000.0, w, h, x, y, self.wid, coding, len(scrolls), w*h*4/1024, self._damage_packet_sequence, client_options)
+        for scroll, line_defs in raw_scroll.items():
+            for line, count in line_defs.items():
+                scrolls.append((x, y+line, w, count, 0, scroll))
+        assert len(scrolls)>0
+        client_options = options.copy()
+        try:
+            del client_options["scroll"]
+        except:
+            pass
+        if flush>0 and self.supports_flush:
+            client_options["flush"] = flush
+        coding = "scroll"
+        end = time.time()
+        packet = self.make_draw_packet(x, y, w, h, coding, LargeStructure(coding, scrolls), 0, client_options, options)
+        self.queue_damage_packet(packet)
+        compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %6s as %3i rectangles  (%5iKB)           , sequence %5i, client_options=%s",
+             (end-start)*1000.0, w, h, x, y, self.wid, coding, len(scrolls), w*h*4/1024, self._damage_packet_sequence, client_options)
         #send the rest as rectangles:
         if non_scroll:
             nsstart = time.time()
@@ -1534 +1494 @@
             else:
                 #slower but can be lossless:
                 encoding = self.get_video_fallback_encoding(PREFERED_ENCODING_ORDER)
+            client_options = options.copy()
             if encoding:
                 encode_fn = self._encoders[encoding]
-                for sy, sh in non_scroll:
+                for sy, sh in non_scroll.items():
+                    #s = time.time()
                     sub = image.get_sub_image(0, sy, w, sh)
-                    flush -= 1
+                    #scrolllog("sub_image %i pixels: %.1fms", (w*sh), (time.time()-s)*1000)
                     ret = encode_fn(encoding, sub, options)
                     if not ret:
                         self.free_image_wrapper(sub)
@@ -1546 +1508 @@
                         return None
                     coding, data, client_options, outw, outh, outstride, _ = ret
                     assert data
-                    client_options = options.copy()
-                    if flush>0 and self.supports_flush:
+                    flush -= 1
+                    if self.supports_flush and flush>0:
                         client_options["flush"] = flush
                     packet = self.make_draw_packet(sub.get_x(), sub.get_y(), outw, outh, coding, data, outstride, client_options, options)
                     self.queue_damage_packet(packet)
@@ -1558 +1520 @@
                     compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %6s with ratio %5.1f%%  (%5iKB to %5iKB), sequence %5i, client_options=%s",
                          (end-nsstart)*1000.0, w, sh, 0, sy, self.wid, coding, 100.0*csize/psize, psize/1024, csize/1024, self._damage_packet_sequence, client_options)
                 scrolllog("non-scroll encoding using %s (quality=%i, speed=%i) took %ims for %i rectangles", encoding, self._current_quality, self._current_speed, (time.time()-nsstart)*1000, len(non_scroll))
+            else:
+                #we can't send the non-scroll areas, ouch!
+                flush = 0
         assert flush==0
         self.last_scroll_time = time.time()
         scrolllog("scroll encoding total time: %ims", (self.last_scroll_time-start)*1000)
@@ -1635 +1600 @@
                 if csums:
                     self.scroll_data = rectangle(x, y, w, h), csums
                     options["scroll"] = True
-                    scrolllog("updated scroll data, previously set: %s", bool(lsd))
+                    scrolllog("updated scroll data with %s, previously set: %s", self.scroll_data[0], (lsd or [None])[0])
                 if lsd and csums:
                     rect, lcsums = lsd
                     if rect.x!=x or rect.y!=y:
@@ -1646 +1611 @@
                     else:
                         #same size, try to find scrolling value
                         assert len(csums)==len(lcsums), "mismatch between checksums lists: %i vs %i items!" % (len(csums), len(lcsums))
-                        distances = calculate_distances(csums, lcsums, 2, 1000)
-                        if len(distances)==0:
+                        max_distance = min(1000, (100-SCROLL_MIN_PERCENT)*h//100)
+                        distances = scroll_distances(lcsums, csums, 2, max_distance)
+                        scroll, count = distances.get_best_match()
+                        if count==0:
                             scrolllog("no scroll distances found")
                         else:
-                            best = max(distances.values())
-                            scroll = distances.keys()[distances.values().index(best)]
                             end = time.time()
-                            best_pct = int(100*best/h)
-                            scrolllog("best scroll guess took %ims, matches %i%% of %i lines: %s", (end-start)*1000, best_pct, h, scroll)
+                            match_pct = int(100*count/h)
+                            scrolllog("best scroll guess took %ims, matches %i%% of %i lines: %s", (end-start)*1000, match_pct, h, scroll)
                             #if enough scrolling is detected, use scroll encoding for this frame:
-                            if best_pct>=SCROLL_MIN_PERCENT:
-                                return self.encode_scrolling(image, distances, lcsums, csums, options)
+                            if match_pct>=SCROLL_MIN_PERCENT:
+                                return self.encode_scrolling(image, distances, options)
             except Exception:
                 scrolllog.error("Error during scrolling detection!", exc_info=True)
 

unittests/unit/server/motion_test.py

@@ -1 +1 @@
 #!/usr/bin/env python
 # This file is part of Xpra.
-# Copyright (C) 2011-2014 Antoine Martin <antoine@devloop.org.uk>
+# Copyright (C) 2016-2017 Antoine Martin <antoine@devloop.org.uk>
 # Xpra is released under the terms of the GNU GPL v2, or, at your option, any
 # later version. See the file COPYING for details.
 
@@ -20 +20 @@
 
 class TestMotion(unittest.TestCase):
 
+        def calculate_distances(self, array1, array2, min_hits=2, max_distance=1):
+                sd = motion.scroll_distances(array1, array2, max_distance)
+                return sd.get_best_scroll_values(min_hits)
+
         def test_match_distance(self):
                 def t(a1, a2, distance, matches):
-                        lines = motion.match_distance(a1, a2, distance)
-                        assert len(lines)==matches, "expected %i matches for distance=%i but got %i for a1=%s, a2=%s, result=%s" % (matches, distance, len(lines), a1, a2, lines)
+                        sd = motion.scroll_distances(a1, a2)
+                        line_defs = sd.match_distance(distance)
+                        linecount = sum(line_defs.values())
+                        assert linecount==matches, "expected %i matches for distance=%i but got %i for a1=%s, a2=%s, result=%s" % (matches, distance, linecount, a1, a2, line_defs)
                 for N in (1, 10, 100):
-                        a = range(N)
+                        a = range(1, N+1)
                         t(a, a, 0, N)           #identity: all match
 
-                        a = [0]*N
+                        a = [1]*N
                         t(a, a, 0, N)
                         for M in range(N):
                                 t(a, a, M, N-M)
 
+                #from a1 to a2: shift by 2, get 2 hits
+                t([3, 4, 5, 6], [1, 2, 3, 4], 2, 2)
                 #from a2 to a1: shift by -2, get 2 hits
-                t([0, 1, 2, 3], [2, 3, 4, 5], -2, 2)
+                t([1, 2, 3, 4], [3, 4, 5, 6], -2, 2)
                 N = 100
-                a1 = range(N)
-                for M in (1, 2, 10, 100):
+                S = 1
+                a1 = range(S, S+N)
+                for M in (2, 3, 10, 100):
                         a2 = range(M, M+N)
-                        t(a1, a2, -M, N-abs(M))
-                        a2 = range(-M, N-M)
-                        t(a1, a2, M, N-abs(M))
-                for M in (-10, -20):
-                        a2 = range(M, N+M)
-                        t(a1, a2, -M, N-abs(M))
+                        t(a1, a2, S-M, S+N-M)
+                        t(a2, a1, M-S, S+N-M)
 
-        def test_consecutive_lines(self):
-                def f(v):
-                        try:
-                                motion.consecutive_lines(v)
-                        except:
-                                pass
-                        else:
-                                raise Exception("consecutive_lines should have failed for value %s" % v)
-                f(None)
-                f("")
-                f([])
-                f(1)
-                def t(v, e):
-                        r = motion.consecutive_lines(v)
-                        assert r==e, "expected %s but got %s for input=%s" % (e, r, v)
-                t([5, 10], [(5, 1), (10, 1)])
-                t([5], [(5, 1)])
-                t([1,2,3], [(1,3)])
-                t(range(100), [(0, 100),])
-                t([1,2,3,100,200,201], [(1,3), (100, 1), (200, 2)])
-
         def test_calculate_distances(self):
                 array1 = [crc32(str(x)) for x in (1234, "abc", 99999)]
-                array2 = array1
-                d = motion.calculate_distances(array1, array2, 1)
-                assert len(d)==1 and d[0]==3
+                array2 = array1[:]
+                d = self.calculate_distances(array1, array2, 1)
+                assert len(d)==1 and sum(d[0].values())==len(array1), "expected %i matches but got %s" % (len(array1), d[0])
 
-                array1 = range(0, 3)
-                array2 = range(1, 4)
-                d = motion.calculate_distances(array1, array2, 1)
-                assert len(d)==1, "expected 1 match but got: %s" % d
-                assert d.get(1)==2, "expected distance of 1 with 2 hits but got: %s" % d
+                array1 = range(1, 5)
+                array2 = range(2, 6)
+                d = self.calculate_distances(array1, array2)
+                assert len(d)==1, "expected 1 match but got: %s" % len(d)
+                common = set(array1).intersection(set(array2))
+                assert -1 in d, "expected distance of -1 but got: %s" % d.keys()
+                linecount = sum(d.get(-1).values())
+                assert linecount==len(common), "expected %i hits but got: %s" % (len(common), linecount)
                 def cdf(v1, v2):
                         try:
-                                motion.calculate_distances(v1, v2, 1)
+                                self.calculate_distances(v1, v2, 1)
                         except:
                                 return
                         raise Exception("calculate_distances should have failed for values: %s" % (v1, v2))
@@ -88 +75 @@
                 cdf([], None)
                 cdf(None, [])
                 cdf([1, 2], [1])
-                assert len(motion.calculate_distances([], [], 1))==0
 
                 #performance:
                 N = 4096
@@ -95 +81 @@
                 start = time.time()
                 array1 = range(N)
                 array2 = [N*2-x*2 for x in range(N)]
-                d = motion.calculate_distances(array1, array2, 1)
+                d = self.calculate_distances(array1, array2, 1)
                 end = time.time()
                 if SHOW_PERF:
                         print("calculate_distances %4i^2 in %5.1f ms" % (N, (end-start)*1000))
@@ -137 +123 @@
                                 print("\nCRC_Image %ix%i (%.1fMB) in %4.2f ms" % (W, H, len(buf2)//1024//1024, 1000.0*(end-start)))
                         assert len(ov2)==H
                         start = time.time()
-                        distances = motion.calculate_distances(ov1, ov2, min_score=1)
+                        distances = self.calculate_distances(ov1, ov2, min_hits=1)
                         end = time.time()
                         if SHOW_PERF:
                                 print("calculate_distances %4i^2 in %5.2f ms" % (H, 1000.0*(end-start)))
-                        linecount = distances.get(N, 0)
+                        line_defs = distances.get(-N, 0)
+                        linecount = sum(line_defs.values())
                         assert linecount>0, "could not find distance %i" % N
                         assert linecount == (H-N), "expected to match %i lines but got %i" % (H-N, linecount)
                 if False: