xpra icon
Bug tracker and wiki

Ticket #410: video-subregion-v5.patch

File video-subregion-v5.patch, 18.3 KB (added by Antoine Martin, 6 years ago)

better patch if a little ugly: caller can provide method for deciding which encoding to use (so we can ensure we don't use video for anything but the current subregion)

  • xpra/server/window_source.py

     
    436436            #in which case the dimensions may be zero (if so configured by the client)
    437437            return
    438438        now = time.time()
     439        if "auto_refresh" not in options:
     440            log.info("damage%s", (window, x, y, w, h, options))
     441            self.statistics.last_damage_events.append((x,y,w,h))
    439442        self.global_statistics.damage_events_count += 1
    440443        self.statistics.damage_events_count += 1
    441444        self.statistics.last_damage_event_time = now
     
    604607        self.send_delayed_regions(*delayed)
    605608        return False
    606609
    607     def send_delayed_regions(self, damage_time, window, regions, coding, options):
     610    def send_delayed_regions(self, damage_time, window, regions, coding, options, get_region_encoding=None):
    608611        """ Called by 'send_delayed' when we expire a delayed region,
    609612            There may be many rectangles within this delayed region,
    610613            so figure out if we want to send them all or if we
     
    629632        bytes_threshold = ww*wh*self.max_bytes_percent/100
    630633        pixel_count = sum([rect.width*rect.height for rect in regions])
    631634        bytes_cost = pixel_count+self.small_packet_cost*len(regions)
     635        log.info("send_delayed_regions: bytes_cost=%s, bytes_threshold=%s, pixel_count=%s", bytes_cost, bytes_threshold, pixel_count)
    632636        if bytes_cost>=bytes_threshold:
    633637            #too many bytes
    634638            send_full_window_update()
     
    645649                #replace with just one region:
    646650                regions = [merged]
    647651
    648         log("send_delayed_regions: %s regions with %s pixels", len(regions), pixel_count)
    649         actual_encoding = self.get_best_encoding(True, window, pixel_count, ww, wh, coding)
    650         if self.must_encode_full_frame(window, actual_encoding):
     652        log.info("send_delayed_regions: %s regions with %s pixels (coding=%s)", len(regions), pixel_count, coding)
     653        if coding and self.must_encode_full_frame(window, coding):
    651654            #use full screen dimensions:
    652             self.process_damage_region(damage_time, window, 0, 0, ww, wh, actual_encoding, options)
     655            self.process_damage_region(damage_time, window, 0, 0, ww, wh, coding, options)
    653656            return
    654657
     658        if get_region_encoding is None:
     659            get_region_encoding = self.get_best_encoding
    655660        #we're processing a number of regions with a non video encoding:
    656661        for region in regions:
     662            actual_encoding = get_region_encoding(True, window, pixel_count, ww, wh, coding)
    657663            self.process_damage_region(damage_time, window, region.x, region.y, region.width, region.height, actual_encoding, options)
    658664
    659665
     
    10011007        #actual network packet:
    10021008        packet = ["draw", wid, x, y, outw, outh, encoding, data, self._damage_packet_sequence, outstride, client_options]
    10031009        end = time.time()
    1004         log("%.1fms to compress %sx%s pixels using %s with ratio=%.1f%% (%sKB to %sKB), delta=%s",
    1005                  (end-start)*1000.0, w, h, coding, 100.0*len(data)/isize, isize/1024, len(data)/1024, delta)
     1010        #calculate rations based on actual pixel data size, not input size which may include a large rowstride:
     1011        psize = w*h*4
     1012        log.info("%.1fms to compress %sx%s pixels using %s with ratio=%.1f%% (%sKB to %sKB), delta=%s",
     1013                 (end-start)*1000.0, w, h, coding, 100.0*len(data)/psize, psize/1024, len(data)/1024, delta)
    10061014        self.global_statistics.packet_count += 1
    10071015        self.statistics.packet_count += 1
    10081016        self._damage_packet_sequence += 1
  • xpra/server/window_stats.py

     
    5454        self.encoding_totals = {}                       #for each encoding, how many frames we sent and how many pixels in total
    5555        self.encoding_pending = {}                      #damage regions waiting to be picked up by the encoding thread:
    5656                                                        #for each sequence no: (damage_time, w, h)
     57        self.last_damage_events = maxdeque(2*NRECS)
    5758        self.last_damage_event_time = None
    5859        self.damage_events_count = 0
    5960        self.packet_count = 0
  • xpra/server/window_video_source.py

     
    88import time
    99from threading import Lock
    1010
     11from xpra.util import AtomicInteger
    1112from xpra.net.protocol import Compressed
    1213from xpra.codecs.codec_constants import get_avutil_enum_from_colorspace, get_subsampling_divs, TransientCodecException, codec_spec
    1314from xpra.codecs.video_helper import getVideoHelper
    14 from xpra.server.window_source import WindowSource
     15from xpra.server.window_source import WindowSource, AUTO_SWITCH_TO_RGB, MAX_PIXELS_PREFER_RGB
    1516from xpra.server.background_worker import add_work_item
     17from xpra.gtk_common.region import rectangle
     18from xpra.codecs.loader import PREFERED_ENCODING_ORDER
    1619from xpra.log import Logger
    1720
    1821log = Logger("video", "encoding")
     
    5659        self.uses_csc_atoms = self.encoding_options.get("csc_atoms", False)
    5760        self.supports_video_scaling = self.encoding_options.get("video_scaling", False)
    5861        self.supports_video_reinit = self.encoding_options.get("video_reinit", False)
     62        self.supports_video_subregion = self.encoding_options.get("video_subregion", False)
     63        self.video_subregion = None
    5964
    6065        if not self.encoding_client_options:
    6166            #old clients can only use 420P:
     
    120125        info[prefix+"client.uses_csc_atoms"] = self.uses_csc_atoms
    121126        info[prefix+"client.supports_video_scaling"] = self.supports_video_scaling
    122127        info[prefix+"client.supports_video_reinit"] = self.supports_video_reinit
     128        info[prefix+"client.supports_video_subregion"] = self.supports_video_subregion
     129        sr = self.video_subregion
     130        if sr:
     131            info[prefix+"video_subregion"] = sr.x, sr.y, sr.width, sr.height
    123132        info[prefix+"scaling"] = self.actual_scaling
    124133        csce = self._csc_encoder
    125134        if csce:
     
    191200        #client may restrict csc modes for specific windows
    192201        self.csc_modes = properties.get("encoding.csc_modes", self.csc_modes)
    193202        self.supports_video_scaling = properties.get("encoding.video_scaling", self.supports_video_scaling)
     203        self.supports_video_subregion = properties.get("encoding.video_subregion", self.supports_video_subregion)
    194204        self.uses_swscale = properties.get("encoding.uses_swscale", self.uses_swscale)
    195205        WindowSource.set_client_properties(self, properties)
    196         log("set_client_properties(%s) csc_modes=%s, video_scaling=%s, uses_swscale=%s", properties, self.csc_modes, self.supports_video_scaling, self.uses_swscale)
     206        log("set_client_properties(%s) csc_modes=%s, video_scaling=%s, video_subregion=%s, uses_swscale=%s", properties, self.csc_modes, self.supports_video_scaling, self.supports_video_subregion, self.uses_swscale)
    197207
    198208    def unmap(self):
    199209        WindowSource.cancel_damage(self)
     
    207217            #drop a frame which is being processed
    208218            self.cleanup_codecs()
    209219
     220
     221    def identify_video_subregion(self):
     222        if self.encoding not in self.video_encodings:
     223            log("identify video: not using a video mode! (%s)", self.encoding)
     224            self.video_subregion = None
     225            return
     226        lde = list(self.statistics.last_damage_events)
     227        dc = len(lde)
     228        if dc<20:
     229            log("identify video: not enough damage events yet (%s)", dc)
     230            self.video_subregion = None
     231            return
     232        ww, wh = self.window_dimensions
     233        #arbitrary minimum size for regions we will look at:
     234        #(we don't want video regions smaller than this)
     235        min_w = max(256, ww/4)
     236        min_h = max(192, wh/4)
     237        #count how many times we see each area / size:
     238        damage_count = {}
     239        c = 0
     240        lde = list(self.statistics.last_damage_events)
     241        damage_pixels = 0
     242        for x,y,w,h in lde:
     243            #ignore small regions:
     244            if w>=min_w and h>=min_h:
     245                damage_count.setdefault((x,y,w,h), AtomicInteger()).increase()
     246                damage_pixels += w*h
     247                c += 1
     248        if len(damage_count)==0:
     249            log.info("identify video: not much happening, keeping current region: %s", self.video_subregion)
     250            return
     251        #TODO: merge smaller regions into larger ones:
     252        #group by size:
     253        size_count = {}
     254        for region, count in damage_count.items():
     255            _, _, w, h = region
     256            size_count.setdefault((w,h), AtomicInteger(0)).increase(int(count))
     257
     258        #see if we can keep the region we have (if any):
     259        if self.video_subregion:
     260            #percentage of window area it occupies:
     261            vsw = self.video_subregion.width
     262            vsh = self.video_subregion.height
     263            vs_pct = 100*ww*wh/(vsw*vsh)
     264            #percentage of damage regions / damage pixels contained within:
     265            contained = [x for x in lde if self.video_subregion.contains(*x)]
     266            pixels_contained = sum([w*h for _,_,w,h in contained])
     267            dr_pct = 100*pixels_contained/damage_pixels
     268            if dr_pct>=vs_pct*2 or (vs_pct<65 and dr_pct>=80):
     269                log.info("keeping existing video region (%s%% of window area, %s%% of damage pixels): %s", vs_pct, dr_pct, self.video_subregion)
     270                return
     271
     272        log.info("identify video: damage_pixels=%s, damage count=%s", damage_pixels, damage_count)
     273        most_damaged = int(sorted(damage_count.values())[-1])
     274        if most_damaged>c*50/100:
     275            #use the region responsible for more than 50% of the large damage requests:
     276            matches = [k for k,v in damage_count.items() if v==most_damaged]
     277            assert len(matches)==1
     278            self.video_subregion = rectangle(*matches[0])
     279            log.info("identified video region (%s%% of damage requests): %s", 100*most_damaged/c, self.video_subregion)
     280            return
     281        #try by size alone:
     282        most_common_size = int(sorted(size_count.values())[-1])
     283        if most_common_size>=c*60/100:
     284            mcw, mch = [k for k,v in size_count.items() if v==most_common_size][0]
     285            #now this will match more than one area..
     286            #so find a recent one:
     287            for x,y,w,h in reversed(lde):
     288                if w==mcw and h==mch:
     289                    #recent and matching size, assume this is the one
     290                    self.video_subregion = rectangle(x, y, w, h)
     291                    log.info("identified video region by size (%sx%s), using recent match: %s", mcw, mch, self.video_subregion)
     292                    return
     293
     294        #try harder: try combining the top two regions
     295        #(flash player with firefox and youtube does stupid unnecessary repaints)
     296        if len(damage_count)>=2:
     297            top_two_counts = [int(x) for x in sorted(damage_count.values())[-2:]]
     298            if sum(top_two_counts)>=c*8/10:
     299                top_two = [k for k,v in damage_count.items() if v in top_two_counts]
     300                top1 = rectangle(*top_two[0])
     301                top2 = rectangle(*top_two[1])
     302                region = top1.clone()
     303                region.merge_rect(top2)
     304                #make sure this does not end up much bigger than needed:
     305                area = region.width*region.height
     306                if area<ww*wh*70/100 and (top1.width*top1.height+top2.width*top2.height)<area*140/100:
     307                    self.video_subregion = region
     308                    log.info("identified merged video region: %s + %s = %s", top1, top2, self.video_subregion)
     309
     310        log.info("failed to identify a video region")
     311        self.video_subregion = None
     312
     313
     314    def send_delayed_regions(self, damage_time, window, regions, coding, options):
     315        """
     316            Overriden here so we can try to intercept the video_subregion if one exists.
     317        """
     318        #overrides the default method for finding the encoding of a region
     319        #so we can ensure we don't use the video encoder when we don't want to:
     320        def get_non_video_encoding(batching, window, pixel_count, ww, wh, current_encoding):
     321            if window.has_alpha() and self.supports_transparency:
     322                return self.get_transparent_encoding(current_encoding)
     323            if window.is_tray():
     324                return self.find_common_lossless_encoder(window.has_alpha(), coding, ww*wh)           
     325            pixel_count = sum([r.width*r.height for r in regions])
     326            if AUTO_SWITCH_TO_RGB and not batching and pixel_count<MAX_PIXELS_PREFER_RGB:
     327                return self.pick_encoding(["rgb24"])
     328            non_video_encodings = [x for x in PREFERED_ENCODING_ORDER if x not in self.video_encodings]
     329            return self.pick_encoding(non_video_encodings, "rgb24")
     330
     331        def send_nonvideo():
     332            WindowSource.send_delayed_regions(self, damage_time, window, regions, coding, options, get_region_encoding=get_non_video_encoding)
     333
     334        vr = self.video_subregion
     335        if window.is_tray() or self.full_frames_only or vr is None or self.encoding not in self.video_encodings:
     336            send_nonvideo()
     337            return
     338
     339        actual_vr = None
     340        if vr in regions:
     341            #found the video region the easy way: exact match in list
     342            actual_vr = vr
     343        else:
     344            #find how many pixels are within the region (roughly):
     345            #find all unique regions that intersect with it:
     346            inter = [vr.intersection_rect(r) for r in regions]
     347            if len(inter)>0:
     348                #merge all regions into one:
     349                in_region = None
     350                for i in inter:
     351                    if i is None:
     352                        continue
     353                    if in_region is None:
     354                        in_region = i
     355                    else:
     356                        in_region.merge_rect(i)
     357                if in_region:
     358                    pixels_in_region = vr.width*vr.height
     359                    pixels_intersect = in_region.width*in_region.height
     360                    if pixels_intersect>=pixels_in_region*40/100:
     361                        #we have at least 40% of the video region
     362                        #that needs refreshing, do it:
     363                        actual_vr = vr
     364
     365            #still no luck?
     366            if actual_vr is None:
     367                #try to find one that has the same dimensions:
     368                same_d = [r for r in regions if r.width==vr.width and r.height==vr.height]
     369                if len(same_d)==1:
     370                    #probably right..
     371                    actual_vr = same_d[0]
     372                elif len(same_d)>1:
     373                    #find one that shares at least one coordinate:
     374                    same_c = [r for r in same_d if r.x==vr.x or r.y==vr.y]
     375                    if len(same_c)==1:
     376                        actual_vr = same_c[0]
     377
     378        if actual_vr is None:
     379            log("send_delayed_regions: video region %s not found in: %s", vr, regions)
     380            send_nonvideo()
     381            return
     382
     383        #found the video region:
     384        #send this straight away using the video encoder:
     385        log.info("send_delayed_regions: sending video region %s", actual_vr)
     386        self.process_damage_region(damage_time, window, actual_vr.x, actual_vr.y, actual_vr.width, actual_vr.height, coding, options)
     387        #now substract this region from the rest:
     388        trimmed = []
     389        for r in regions:
     390            trimmed += r.substract_rect(actual_vr)
     391        if len(trimmed)==0:
     392            log.info("send_delayed_regions: nothing left after removing video region!")
     393            return
     394        log.info("send_delayed_regions: substracted %s from %s gives us %s", actual_vr, regions, trimmed)
     395        WindowSource.send_delayed_regions(self, damage_time, window, trimmed, None, options, get_region_encoding=get_non_video_encoding)
     396
    210397    def process_damage_region(self, damage_time, window, x, y, w, h, coding, options):
    211398        WindowSource.process_damage_region(self, damage_time, window, x, y, w, h, coding, options)
    212399        #now figure out if we need to send edges separately:
     
    239426            #video encoder cannot handle this size!
    240427            #(maybe this should be an 'assert' statement here?)
    241428            return None
     429        if self.video_subregion and (self.video_subregion.width!=ww or self.video_subregion.height!=wh):
     430            #we have a video region, and this is not it, so don't use video:
     431            return None
    242432
    243433        def switch_to_lossless(reason):
    244434            coding = self.find_common_lossless_encoder(has_alpha, current_encoding, ww*wh)
     
    294484        """
    295485        log("reconfigure(%s) csc_encoder=%s, video_encoder=%s", force_reload, self._csc_encoder, self._video_encoder)
    296486        WindowSource.reconfigure(self, force_reload)
     487        if self.supports_video_subregion:
     488            self.identify_video_subregion()
    297489        if not self._video_encoder:
    298490            return
    299491        try:
     
    730922        """
    731923        log("video_encode%s", (encoding, image, options))
    732924        x, y, w, h = image.get_geometry()[:4]
    733         assert x==0 and y==0, "invalid position: %s,%s" % (x,y)
     925        assert self.supports_video_subregion or (x==0 and y==0), "invalid position: %s,%s" % (x,y)
    734926        src_format = image.get_pixel_format()
    735927        try:
    736928            self._lock.acquire()
  • tests/xpra/test_apps/test_videoregions.py

     
    5555        def redraw(self):
    5656                self.step += 1
    5757                self.darea.queue_draw()
    58                 self.label.set_text("Step = %s" % self.step)
    59                 if self.step%4==0:
    60                         self.button.set_label("Hello %s" % (self.step/4))
    61                 if self.step%20==0:
     58                sm = 4
     59                if self.step%sm==0:
     60                        self.label.set_text("Step = %s" % (self.step/sm))
     61                hm = 8
     62                if self.step%hm==0:
     63                        self.button.set_label("Hello %s" % (self.step/hm))
     64                fr = 20
     65                if self.step%fr==0:
    6266                        self.redraw()
    6367                return True
    6468