xpra icon
Bug tracker and wiki

Ticket #410: video-subregion-v4.patch

File video-subregion-v4.patch, 14.2 KB (added by Antoine Martin, 6 years ago)

remaining server side bits, still needs a proper way to ensure non-video regions don't use the video encoding (which does a re-init and wastes everything)

  • xpra/server/window_source.py

     
    436436            #in which case the dimensions may be zero (if so configured by the client)
    437437            return
    438438        now = time.time()
     439        if "auto_refresh" not in options:
     440            log.info("damage%s", (window, x, y, w, h, options))
     441            self.statistics.last_damage_events.append((x,y,w,h))
    439442        self.global_statistics.damage_events_count += 1
    440443        self.statistics.damage_events_count += 1
    441444        self.statistics.last_damage_event_time = now
     
    10011004        #actual network packet:
    10021005        packet = ["draw", wid, x, y, outw, outh, encoding, data, self._damage_packet_sequence, outstride, client_options]
    10031006        end = time.time()
    1004         log("%.1fms to compress %sx%s pixels using %s with ratio=%.1f%% (%sKB to %sKB), delta=%s",
     1007        log.info("%.1fms to compress %sx%s pixels using %s with ratio=%.1f%% (%sKB to %sKB), delta=%s",
    10051008                 (end-start)*1000.0, w, h, coding, 100.0*len(data)/isize, isize/1024, len(data)/1024, delta)
    10061009        self.global_statistics.packet_count += 1
    10071010        self.statistics.packet_count += 1
  • xpra/server/window_stats.py

     
    5454        self.encoding_totals = {}                       #for each encoding, how many frames we sent and how many pixels in total
    5555        self.encoding_pending = {}                      #damage regions waiting to be picked up by the encoding thread:
    5656                                                        #for each sequence no: (damage_time, w, h)
     57        self.last_damage_events = maxdeque(2*NRECS)
    5758        self.last_damage_event_time = None
    5859        self.damage_events_count = 0
    5960        self.packet_count = 0
  • xpra/server/window_video_source.py

     
    88import time
    99from threading import Lock
    1010
     11from xpra.util import AtomicInteger
    1112from xpra.net.protocol import Compressed
    1213from xpra.codecs.codec_constants import get_avutil_enum_from_colorspace, get_subsampling_divs, TransientCodecException, codec_spec
    1314from xpra.codecs.video_helper import getVideoHelper
    1415from xpra.server.window_source import WindowSource
    1516from xpra.server.background_worker import add_work_item
     17from xpra.gtk_common.region import rectangle
    1618from xpra.log import Logger
    1719
    1820log = Logger("video", "encoding")
     
    5658        self.uses_csc_atoms = self.encoding_options.get("csc_atoms", False)
    5759        self.supports_video_scaling = self.encoding_options.get("video_scaling", False)
    5860        self.supports_video_reinit = self.encoding_options.get("video_reinit", False)
     61        self.supports_video_subregion = self.encoding_options.get("video_subregion", False)
     62        self.video_subregion = None
    5963
    6064        if not self.encoding_client_options:
    6165            #old clients can only use 420P:
     
    120124        info[prefix+"client.uses_csc_atoms"] = self.uses_csc_atoms
    121125        info[prefix+"client.supports_video_scaling"] = self.supports_video_scaling
    122126        info[prefix+"client.supports_video_reinit"] = self.supports_video_reinit
     127        info[prefix+"client.supports_video_subregion"] = self.supports_video_subregion
     128        sr = self.video_subregion
     129        if sr:
     130            info[prefix+"video_subregion"] = sr.x, sr.y, sr.width, sr.height
    123131        info[prefix+"scaling"] = self.actual_scaling
    124132        csce = self._csc_encoder
    125133        if csce:
     
    191199        #client may restrict csc modes for specific windows
    192200        self.csc_modes = properties.get("encoding.csc_modes", self.csc_modes)
    193201        self.supports_video_scaling = properties.get("encoding.video_scaling", self.supports_video_scaling)
     202        self.supports_video_subregion = properties.get("encoding.video_subregion", self.supports_video_subregion)
    194203        self.uses_swscale = properties.get("encoding.uses_swscale", self.uses_swscale)
    195204        WindowSource.set_client_properties(self, properties)
    196         log("set_client_properties(%s) csc_modes=%s, video_scaling=%s, uses_swscale=%s", properties, self.csc_modes, self.supports_video_scaling, self.uses_swscale)
     205        log("set_client_properties(%s) csc_modes=%s, video_scaling=%s, video_subregion=%s, uses_swscale=%s", properties, self.csc_modes, self.supports_video_scaling, self.supports_video_subregion, self.uses_swscale)
    197206
    198207    def unmap(self):
    199208        WindowSource.cancel_damage(self)
     
    207216            #drop a frame which is being processed
    208217            self.cleanup_codecs()
    209218
     219
     220    def identify_video_subregion(self):
     221        if self.encoding not in self.video_encodings:
     222            log("identify video: not using a video mode! (%s)", self.encoding)
     223            self.video_subregion = None
     224            return
     225        lde = list(self.statistics.last_damage_events)
     226        dc = len(lde)
     227        if dc<20:
     228            log("identify video: not enough damage events yet (%s)", dc)
     229            self.video_subregion = None
     230            return
     231        ww, wh = self.window_dimensions
     232        #arbitrary minimum size for regions we will look at:
     233        #(we don't want video regions smaller than this)
     234        min_w = max(256, ww/4)
     235        min_h = max(192, wh/4)
     236        #count how many times we see each area / size:
     237        damage_count = {}
     238        c = 0
     239        lde = list(self.statistics.last_damage_events)
     240        damage_pixels = 0
     241        for x,y,w,h in lde:
     242            #ignore small regions:
     243            if w>=min_w and h>=min_h:
     244                damage_count.setdefault((x,y,w,h), AtomicInteger()).increase()
     245                damage_pixels += w*h
     246                c += 1
     247        if len(damage_count)==0:
     248            log.info("identify video: not much happening, keeping current region: %s", self.video_subregion)
     249            return
     250        #TODO: merge smaller regions into larger ones:
     251        #group by size:
     252        size_count = {}
     253        for region, count in damage_count.items():
     254            _, _, w, h = region
     255            size_count.setdefault((w,h), AtomicInteger(0)).increase(int(count))
     256
     257        #see if we can keep the region we have (if any):
     258        if self.video_subregion:
     259            #percentage of window area it occupies:
     260            vsw = self.video_subregion.width
     261            vsh = self.video_subregion.height
     262            vs_pct = 100*ww*wh/(vsw*vsh)
     263            #percentage of damage regions / damage pixels contained within:
     264            contained = [x for x in lde if self.video_subregion.contains(*x)]
     265            pixels_contained = sum([w*h for _,_,w,h in contained])
     266            dr_pct = 100*pixels_contained/damage_pixels
     267            if dr_pct>=vs_pct*2 or (vs_pct<65 and dr_pct>=80):
     268                log.info("keeping existing video region (%s%% of window area, %s%% of damage pixels): %s", vs_pct, dr_pct, self.video_subregion)
     269                return
     270
     271        log.info("identify video: damage_pixels=%s, damage count=%s", damage_pixels, damage_count)
     272        most_damaged = int(sorted(damage_count.values())[-1])
     273        if most_damaged>c*60/100:
     274            #use the region responsible for more than 60% of the large damage requests:
     275            matches = [k for k,v in damage_count.items() if v==most_damaged]
     276            assert len(matches)==1
     277            self.video_subregion = rectangle(*matches[0])
     278            log.info("identified video region (%s%% of damage requests): %s", 100*most_damaged/c, self.video_subregion)
     279            return
     280        #try by size alone:
     281        most_common_size = int(sorted(size_count.values())[-1])
     282        if most_common_size>=c*60/100:
     283            mcw, mch = [k for k,v in size_count.items() if v==most_common_size][0]
     284            #now this will match more than one area..
     285            #so find a recent one:
     286            for x,y,w,h in reversed(lde):
     287                if w==mcw and h==mch:
     288                    #recent and matching size, assume this is the one
     289                    self.video_subregion = rectangle(x, y, w, h)
     290                    log.info("identified video region by size (%sx%s), using recent match: %s", mcw, mch, self.video_subregion)
     291                    return
     292
     293        #try harder: try combining the top two regions
     294        #(flash player with firefox and youtube does stupid unnecessary repaints)
     295        if len(damage_count)>=2:
     296            top_two_counts = [int(x) for x in sorted(damage_count.values())[-2:]]
     297            if sum(top_two_counts)>=c*8/10:
     298                top_two = [k for k,v in damage_count.items() if v in top_two_counts]
     299                top1 = rectangle(top_two[0])
     300                top2 = rectangle(top_two[1])
     301                region = top1.clone()
     302                region.merge_rect(top2)
     303                #make sure this does not end up much bigger than needed:
     304                area = region.width*region.height
     305                if area<ww*wh*70/100 and (top1.width*top1.height+top2.width*top2.height)<area*140/100:
     306                    self.video_subregion = region
     307                    log.info("identified merged video region: %s + %s = %s", top1, top2, self.video_subregion)
     308
     309        log.info("failed to identified a video region")
     310        self.video_subregion = None
     311
     312
     313    def send_delayed_regions(self, damage_time, window, regions, coding, options):
     314        """
     315            Overriden here so we can try to intercept the video_subregion if one exists.
     316        """
     317        def get_non_video_encoding():
     318            pixel_count = sum([r.width*r.height for r in regions])
     319            ww, wh = window.get_dimensions()
     320            return self.get_best_encoding(True, window, pixel_count, ww, wh, coding)
     321        def send_nonvideo():
     322            coding = get_non_video_encoding()
     323            WindowSource.send_delayed_regions(self, damage_time, window, regions, coding, options)
     324
     325        vr = self.video_subregion
     326        if window.is_tray() or self.full_frames_only or vr is None or self.encoding not in self.video_encodings:
     327            send_nonvideo()
     328            return
     329
     330        actual_vr = None
     331        if vr in regions:
     332            #found the video region the easy way: exact match in list
     333            actual_vr = vr
     334        else:
     335            #find how many pixels are within the region (roughly):
     336            #find all unique regions that intersect with it:
     337            inter = [vr.intersection_rect(r) for r in regions]
     338            if len(inter)>0:
     339                #merge all regions into one:
     340                in_region = None
     341                for i in inter:
     342                    if i is None:
     343                        continue
     344                    if in_region is None:
     345                        in_region = i
     346                    else:
     347                        in_region.merge_rect(i)
     348                if in_region:
     349                    pixels_in_region = vr.width*vr.height
     350                    pixels_intersect = in_region.width*in_region.height
     351                    if pixels_intersect>=pixels_in_region*40/100:
     352                        #we have at least 40% of the video region
     353                        #that needs refreshing, do it:
     354                        actual_vr = vr
     355
     356            #still no luck?
     357            if actual_vr is None:
     358                #try to find one that has the same dimensions:
     359                same_d = [r for r in regions if r.width==vr.width and r.height==vr.height]
     360                if len(same_d)==1:
     361                    #probably right..
     362                    actual_vr = same_d[0]
     363                elif len(same_d)>1:
     364                    #find one that shares at least one coordinate:
     365                    same_c = [r for r in same_d if r.x==vr.x or r.y==vr.y]
     366                    if len(same_c)==1:
     367                        actual_vr = same_c[0]
     368
     369        if actual_vr is None:
     370            log("send_delayed_region: video region %s not found in: %s", vr, regions)
     371            send_nonvideo()
     372            return
     373
     374        #found the video region:
     375        #send this straight away using the video encoder:
     376        self.process_damage_region(damage_time, window, actual_vr.x, actual_vr.y, actual_vr.width, actual_vr.height, coding, options)
     377        #now substract this region from the rest:
     378        trimmed = []
     379        for r in regions:
     380            trimmed += r.substract_rect(actual_vr)
     381        #log.info("send_delayed_region: substracted %s from %s gives us %s", actual_vr, regions, trimmed)
     382        coding = get_non_video_encoding()
     383        WindowSource.send_delayed_regions(self, damage_time, window, trimmed, coding, options)
     384
    210385    def process_damage_region(self, damage_time, window, x, y, w, h, coding, options):
    211386        WindowSource.process_damage_region(self, damage_time, window, x, y, w, h, coding, options)
    212387        #now figure out if we need to send edges separately:
     
    239414            #video encoder cannot handle this size!
    240415            #(maybe this should be an 'assert' statement here?)
    241416            return None
     417        if self.video_subregion and (self.video_subregion.width!=ww or self.video_subregion.height!=wh):
     418            #we have a video region, and this is not it
     419            return None
    242420
    243421        def switch_to_lossless(reason):
    244422            coding = self.find_common_lossless_encoder(has_alpha, current_encoding, ww*wh)
     
    294472        """
    295473        log("reconfigure(%s) csc_encoder=%s, video_encoder=%s", force_reload, self._csc_encoder, self._video_encoder)
    296474        WindowSource.reconfigure(self, force_reload)
     475        if self.supports_video_subregion:
     476            self.identify_video_subregion()
    297477        if not self._video_encoder:
    298478            return
    299479        try:
     
    730910        """
    731911        log("video_encode%s", (encoding, image, options))
    732912        x, y, w, h = image.get_geometry()[:4]
    733         assert x==0 and y==0, "invalid position: %s,%s" % (x,y)
     913        assert self.supports_video_subregion or (x==0 and y==0), "invalid position: %s,%s" % (x,y)
    734914        src_format = image.get_pixel_format()
    735915        try:
    736916            self._lock.acquire()