xpra icon
Bug tracker and wiki

This bug tracker and wiki are being discontinued
please use https://github.com/Xpra-org/xpra instead.



Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/xpra/server/window/window_video_source.py

    r13238 r28374  
    1 # coding=utf8
     1# -*- coding: utf-8 -*-
    22# This file is part of Xpra.
    3 # Copyright (C) 2013-2016 Antoine Martin <antoine@devloop.org.uk>
     3# Copyright (C) 2013-2019 Antoine Martin <antoine@xpra.org>
    44# Xpra is released under the terms of the GNU GPL v2, or, at your option, any
    55# later version. See the file COPYING for details.
     
    99import operator
    1010import threading
     11from math import sqrt
     12from functools import reduce
    1113
    1214from xpra.net.compression import Compressed, LargeStructure
    1315from xpra.codecs.codec_constants import TransientCodecException, RGB_FORMATS, PIXEL_SUBSAMPLING
    14 from xpra.server.window.window_source import WindowSource, STRICT_MODE, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY
    15 from xpra.server.window.region import merge_all            #@UnresolvedImport
    16 from xpra.server.window.motion import match_distance, consecutive_lines, calculate_distances, CRC_Image #@UnresolvedImport
    17 from xpra.server.window.video_subregion import VideoSubregion
     16from xpra.server.window.window_source import (
     17    WindowSource, DelayedRegions,
     18    STRICT_MODE, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY, MAX_RGB,
     19    )
     20from xpra.rectangle import rectangle, merge_all          #@UnresolvedImport
     21from xpra.server.window.motion import ScrollData                    #@UnresolvedImport
     22from xpra.server.window.video_subregion import VideoSubregion, VIDEO_SUBREGION
    1823from xpra.server.window.video_scoring import get_pipeline_score
    19 from xpra.codecs.loader import PREFERED_ENCODING_ORDER, EDGE_ENCODING_ORDER
    20 from xpra.util import parse_scaling_value, engs
     24from xpra.codecs.codec_constants import PREFERRED_ENCODING_ORDER, EDGE_ENCODING_ORDER
     25from xpra.codecs.loader import has_codec
     26from xpra.util import parse_scaling_value, engs, envint, envbool, csv, roundup, print_nested_dict, first_time, typedict
     27from xpra.os_util import monotonic_time, bytestostr
    2128from xpra.log import Logger
    2229
     
    2936avsynclog = Logger("av-sync")
    3037scrolllog = Logger("scroll")
    31 
    32 
    33 def envint(name, d):
    34     try:
    35         return int(os.environ.get(name, d))
    36     except:
    37         return d
    38 
     38compresslog = Logger("compress")
     39refreshlog = Logger("refresh")
     40regionrefreshlog = Logger("regionrefresh")
     41
     42
     43TEXT_USE_VIDEO = envbool("XPRA_TEXT_USE_VIDEO", False)
    3944MAX_NONVIDEO_PIXELS = envint("XPRA_MAX_NONVIDEO_PIXELS", 1024*4)
     45MIN_VIDEO_FPS = envint("XPRA_MIN_VIDEO_FPS", 10)
     46MIN_VIDEO_EVENTS = envint("XPRA_MIN_VIDEO_EVENTS", 20)
     47ENCODE_QUEUE_MIN_GAP = envint("XPRA_ENCODE_QUEUE_MIN_GAP", 5)
     48
     49VIDEO_TIMEOUT = envint("XPRA_VIDEO_TIMEOUT", 10)
     50VIDEO_NODETECT_TIMEOUT = envint("XPRA_VIDEO_NODETECT_TIMEOUT", 10*60)
    4051
    4152FORCE_CSC_MODE = os.environ.get("XPRA_FORCE_CSC_MODE", "")   #ie: "YUV444P"
     
    4354    log.warn("ignoring invalid CSC mode specified: %s", FORCE_CSC_MODE)
    4455    FORCE_CSC_MODE = ""
    45 FORCE_CSC = bool(FORCE_CSC_MODE) or  os.environ.get("XPRA_FORCE_CSC", "0")=="1"
    46 SCALING = os.environ.get("XPRA_SCALING", "1")=="1"
     56FORCE_CSC = bool(FORCE_CSC_MODE) or envbool("XPRA_FORCE_CSC", False)
     57SCALING = envbool("XPRA_SCALING", True)
    4758SCALING_HARDCODED = parse_scaling_value(os.environ.get("XPRA_SCALING_HARDCODED", ""))
    48 
    49 VIDEO_SUBREGION = envint("XPRA_VIDEO_SUBREGION", 1)==1
     59SCALING_PPS_TARGET = envint("XPRA_SCALING_PPS_TARGET", 25*1920*1080)
     60SCALING_MIN_PPS = envint("XPRA_SCALING_MIN_PPS", 25*320*240)
     61SCALING_OPTIONS = (1, 10), (1, 5), (1, 4), (1, 3), (1, 2), (2, 3), (1, 1)
     62def parse_scaling_options_str(scaling_options_str):
     63    if not scaling_options_str:
     64        return SCALING_OPTIONS
     65    #parse 1/10,1/5,1/4,1/3,1/2,2/3,1/1
     66    #or even: 1:10, 1:5, ...
     67    vs_options = []
     68    for option in scaling_options_str.split(","):
     69        try:
     70            if option.find("%")>0:
     71                v = float(option[:option.find("%")])*100
     72                vs_options.append(v.as_integer_ratio())
     73            elif option.find("/")<0:
     74                v = float(option)
     75                vs_options.append(v.as_integer_ratio())
     76            else:
     77                num, den = option.strip().split("/")
     78                vs_options.append((int(num), int(den)))
     79        except ValueError:
     80            scalinglog.warn("Warning: invalid scaling string '%s'", option.strip())
     81    if vs_options:
     82        return tuple(vs_options)
     83    return SCALING_OPTIONS
     84SCALING_OPTIONS = parse_scaling_options_str(os.environ.get("XPRA_SCALING_OPTIONS"))
     85scalinglog("scaling options: SCALING=%s, HARDCODED=%s, PPS_TARGET=%i, MIN_PPS=%i, OPTIONS=%s",
     86           SCALING, SCALING_HARDCODED, SCALING_PPS_TARGET, SCALING_MIN_PPS, SCALING_OPTIONS)
     87
     88DEBUG_VIDEO_CLEAN = envbool("XPRA_DEBUG_VIDEO_CLEAN", False)
     89
    5090FORCE_AV_DELAY = envint("XPRA_FORCE_AV_DELAY", 0)
    51 B_FRAMES = envint("XPRA_B_FRAMES", 1)==1
    52 VIDEO_SKIP_EDGE = envint("XPRA_VIDEO_SKIP_EDGE", 0)==1
    53 SCROLL_ENCODING = envint("XPRA_SCROLL_ENCODING", 0)==1
    54 SCROLL_MIN_PERCENT = max(1, min(100, envint("XPRA_SCROLL_MIN_PERCENT", 40)))
    55 
    56 FAST_ORDER = ["jpeg", "rgb32", "rgb24", "png"] + PREFERED_ENCODING_ORDER
     91B_FRAMES = envbool("XPRA_B_FRAMES", True)
     92VIDEO_SKIP_EDGE = envbool("XPRA_VIDEO_SKIP_EDGE", False)
     93SCROLL_MIN_PERCENT = max(1, min(100, envint("XPRA_SCROLL_MIN_PERCENT", 50)))
     94MIN_SCROLL_IMAGE_SIZE = envint("XPRA_MIN_SCROLL_IMAGE_SIZE", 128)
     95
     96SAVE_VIDEO_PATH = os.environ.get("XPRA_SAVE_VIDEO_PATH", "")
     97SAVE_VIDEO_STREAMS = envbool("XPRA_SAVE_VIDEO_STREAMS", False)
     98SAVE_VIDEO_FRAMES = os.environ.get("XPRA_SAVE_VIDEO_FRAMES")
     99if SAVE_VIDEO_FRAMES not in ("png", "jpeg", None):
     100    log.warn("Warning: invalid value for 'XPRA_SAVE_VIDEO_FRAMES'")
     101    log.warn(" only 'png' or 'jpeg' are allowed")
     102    SAVE_VIDEO_FRAMES = None
     103
     104FAST_ORDER = tuple(["jpeg", "rgb32", "rgb24", "webp", "png"] + list(PREFERRED_ENCODING_ORDER))
    57105
    58106
     
    64112    def __init__(self, *args):
    65113        #this will call init_vars():
    66         WindowSource.__init__(self, *args)
    67         self.scroll_encoding = SCROLL_ENCODING
    68         self.supports_scrolling = self.scroll_encoding and self.encoding_options.boolget("scrolling")
    69         self.supports_video_scaling = self.encoding_options.boolget("video_scaling", False)
    70         self.supports_video_b_frames = self.encoding_options.strlistget("video_b_frames", [])
    71         self.supports_video_subregion = VIDEO_SUBREGION
     114        self.supports_scrolling = False
     115        self.video_subregion = None
     116        super().__init__(*args)
     117        self.supports_eos = self.encoding_options.boolget("eos")
     118        self.supports_scrolling = "scroll" in self.common_encodings or (
     119            #for older clients, we check an encoding option:
     120            "scroll" in self.server_core_encodings and self.encoding_options.boolget("scrolling") and not STRICT_MODE)
     121        self.scroll_min_percent = self.encoding_options.intget("scrolling.min-percent", SCROLL_MIN_PERCENT)
     122        self.supports_video_b_frames = self.encoding_options.strtupleget("video_b_frames", ())
     123        self.video_max_size = self.encoding_options.inttupleget("video_max_size", (8192, 8192), 2, 2)
     124        self.video_subregion = VideoSubregion(self.timeout_add, self.source_remove, self.refresh_subregion, self.auto_refresh_delay)
     125        self.video_stream_file = None
    72126
    73127    def init_encoders(self):
    74128        WindowSource.init_encoders(self)
    75129        #for 0.12 onwards: per encoding lists:
    76         self.full_csc_modes = {}
    77         self.parse_csc_modes(self.encoding_options.dictget("full_csc_modes", default_value=None))
    78130
    79131        self.video_encodings = self.video_helper.get_encodings()
    80132        for x in self.video_encodings:
    81133            if x in self.server_core_encodings:
    82                 self._encoders[x] = self.video_encode
     134                self.add_encoder(x, self.video_encode)
     135        self.add_encoder("auto", self.video_encode)
     136        if has_codec("csc_libyuv"):
     137            #need libyuv to be able to handle 'grayscale' video:
     138            #(to convert ARGB to grayscale)
     139            self.add_encoder("grayscale", self.video_encode)
    83140        #these are used for non-video areas, ensure "jpeg" is used if available
    84141        #as we may be dealing with large areas still, and we want speed:
    85142        nv_common = (set(self.server_core_encodings) & set(self.core_encodings)) - set(self.video_encodings)
    86         self.non_video_encodings = [x for x in PREFERED_ENCODING_ORDER if x in nv_common]
    87 
     143        self.non_video_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER
     144                                         if x in nv_common)
     145        self.common_video_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER
     146                                            if x in self.video_encodings and x in self.core_encodings)
     147        if "scroll" in self.server_core_encodings:
     148            self.add_encoder("scroll", self.scroll_encode)
    88149        #those two instances should only ever be modified or accessed from the encode thread:
    89150        self._csc_encoder = None
     
    96157    def init_vars(self):
    97158        WindowSource.init_vars(self)
    98         self.video_subregion = VideoSubregion(self.timeout_add, self.source_remove, self.refresh_subregion, self.auto_refresh_delay)
    99         self.video_subregion.set_enabled(False)     #disabled until we parse the client props
    100 
    101159        #these constraints get updated with real values
    102160        #when we construct the video pipeline:
     
    110168
    111169        self.last_pipeline_params = None
    112         self.last_pipeline_scores = []
     170        self.last_pipeline_scores = ()
    113171        self.last_pipeline_time = 0
    114172
    115         self.supports_video_scaling = False
    116         self.supports_video_subregion = False
    117 
    118         self.full_csc_modes = {}                            #for 0.12 onwards: per encoding lists
    119         self.video_encodings = []
    120         self.non_video_encodings = []
     173        self.video_encodings = ()
     174        self.common_video_encodings = ()
     175        self.non_video_encodings = ()
    121176        self.edge_encoding = None
    122177        self.start_video_frame = 0
     178        self.video_encoder_timer = None
    123179        self.b_frame_flush_timer = None
     180        self.b_frame_flush_data = None
     181        self.encode_from_queue_timer = None
     182        self.encode_from_queue_due = 0
    124183        self.scroll_data = None
    125 
    126     def set_auto_refresh_delay(self, d):
    127         WindowSource.set_auto_refresh_delay(self, d)
     184        self.last_scroll_time = 0
     185
     186    def do_set_auto_refresh_delay(self, min_delay, delay):
     187        super().do_set_auto_refresh_delay(min_delay, delay)
    128188        r = self.video_subregion
    129189        if r:
    130             r.set_auto_refresh_delay(d)
    131 
    132     def calculate_batch_delay(self, has_focus, other_is_fullscreen, other_is_maximized):
    133         WindowSource.calculate_batch_delay(self, has_focus, other_is_fullscreen, other_is_maximized)
    134         vsr = self.video_subregion
    135         bc = self.batch_config
    136         if not bc.locked and vsr:
    137             #we have a video subregion, update its refresh delay
    138             r = vsr.rectangle
    139             if r:
    140                 ww, wh = self.window_dimensions
    141                 pct = (100*r.width*r.height)/(ww*wh)
    142                 d = 2 * int(max(100, self.auto_refresh_delay * max(50, pct) / 50, bc.delay*4))
    143                 vsr.set_auto_refresh_delay(d)
     190            r.set_auto_refresh_delay(self.base_auto_refresh_delay)
    144191
    145192    def update_av_sync_frame_delay(self):
     
    147194        ve = self._video_encoder
    148195        if ve:
     196            #how many frames are buffered in the encoder, if any:
    149197            d = ve.get_info().get("delayed", 0)
    150             self.av_sync_frame_delay += 40 * d
    151             avsynclog("update_av_sync_frame_delay() video encoder=%s, delayed frames=%i, frame delay=%i", ve, d, self.av_sync_frame_delay)
     198            if d>0:
     199                #clamp the batch delay to a reasonable range:
     200                frame_delay = min(100, max(10, self.batch_config.delay))
     201                self.av_sync_frame_delay += frame_delay * d
     202            avsynclog("update_av_sync_frame_delay() video encoder=%s, delayed frames=%i, frame delay=%i",
     203                      ve, d, self.av_sync_frame_delay)
    152204        self.may_update_av_sync_delay()
    153205
    154206
    155     def get_client_info(self):
    156         info = {
    157             "supports_video_scaling"    : self.supports_video_scaling,
    158             "supports_video_subregion"  : self.supports_video_subregion,
    159             }
    160         for enc, csc_modes in (self.full_csc_modes or {}).items():
    161             info["csc_modes.%s" % enc] = csc_modes
    162         return info
    163 
    164     def get_property_info(self):
     207    def get_property_info(self) -> dict:
    165208        i = WindowSource.get_property_info(self)
    166         i.update({
    167                 "scaling.control"       : self.scaling_control,
    168                 "scaling"               : self.scaling or (1, 1),
    169                 })
     209        if self.scaling_control is None:
     210            i["scaling.control"] = "auto"
     211        else:
     212            i["scaling.control"] = self.scaling_control
     213        i["scaling"] = self.scaling or (1, 1)
    170214        return i
    171215
    172     def get_info(self):
     216    def get_info(self) -> dict:
    173217        info = WindowSource.get_info(self)
    174218        sr = self.video_subregion
    175219        if sr:
    176             info["video_subregion"] = sr.get_info()
     220            sri = sr.get_info()
     221            sri["video-mode"] = self.subregion_is_video()
     222            info["video_subregion"] = sri
    177223        info["scaling"] = self.actual_scaling
     224        info["video-max-size"] = self.video_max_size
    178225        def addcinfo(prefix, x):
    179226            if not x:
     
    183230                i[""] = x.get_type()
    184231                info[prefix] = i
    185             except:
     232            except Exception:
    186233                log.error("Error collecting codec information from %s", x, exc_info=True)
    187234        addcinfo("csc", self._csc_encoder)
     
    189236        info.setdefault("encodings", {}).update({
    190237                                                 "non-video"    : self.non_video_encodings,
     238                                                 "video"        : self.common_video_encodings,
    191239                                                 "edge"         : self.edge_encoding or "",
     240                                                 "eos"          : self.supports_eos,
    192241                                                 })
    193242        einfo = {
    194243                 "pipeline_param" : self.get_pipeline_info(),
    195                  "scrolling"      : self.supports_scrolling,
     244                 "scrolling"      : {
     245                     "enabled"      : self.supports_scrolling,
     246                     "min-percent"  : self.scroll_min_percent,
     247                     }
    196248                 }
    197249        if self._last_pipeline_check>0:
    198             einfo["pipeline_last_check"] = int(1000*(time.time()-self._last_pipeline_check))
     250            einfo["pipeline_last_check"] = int(1000*(monotonic_time()-self._last_pipeline_check))
    199251        lps = self.last_pipeline_scores
    200252        if lps:
     
    205257        return info
    206258
    207     def get_pipeline_info(self):
     259    def get_pipeline_info(self) -> dict:
    208260        lp = self.last_pipeline_params
    209261        if not lp:
     
    220272            try:
    221273                return x.codec_type
    222             except:
     274            except AttributeError:
    223275                return repr(x)
    224276        pi  = {
     
    245297
    246298
     299    def suspend(self):
     300        WindowSource.suspend(self)
     301        #we'll create a new video pipeline when resumed:
     302        self.cleanup_codecs()
     303
     304
    247305    def cleanup(self):
    248306        WindowSource.cleanup(self)
     
    256314        """
    257315        self.cancel_video_encoder_flush()
    258         if self._csc_encoder:
    259             self.csc_encoder_clean()
    260         if self._video_encoder:
    261             self.video_encoder_clean()
    262 
    263     def csc_encoder_clean(self):
    264         """ Calls self._csc_encoder.clean() from the encode thread """
     316        self.video_context_clean()
     317
     318    def video_context_clean(self):
     319        """ Calls clean() from the encode thread """
    265320        csce = self._csc_encoder
     321        ve = self._video_encoder
     322        if csce or ve:
     323            if DEBUG_VIDEO_CLEAN:
     324                log.warn("video_context_clean() for wid %i: %s and %s", self.wid, csce, ve)
     325                import traceback
     326                traceback.print_stack()
     327            self._csc_encoder = None
     328            self._video_encoder = None
     329            def clean():
     330                if DEBUG_VIDEO_CLEAN:
     331                    log.warn("video_context_clean() done")
     332                self.csc_clean(csce)
     333                self.ve_clean(ve)
     334            self.call_in_encode_thread(False, clean)
     335
     336    def csc_clean(self, csce):
    266337        if csce:
    267             self._csc_encoder = None
    268             self.call_in_encode_thread(False, csce.clean)
    269 
    270     def video_encoder_clean(self):
    271         """ Calls self._video_encoder.clean() from the encode thread """
    272         ve = self._video_encoder
     338            csce.clean()
     339
     340    def ve_clean(self, ve):
     341        self.cancel_video_encoder_timer()
    273342        if ve:
    274             self._video_encoder = None
    275             self.call_in_encode_thread(False, ve.clean)
    276 
    277 
    278     def parse_csc_modes(self, full_csc_modes):
    279         #only override if values are specified:
    280         csclog("parse_csc_modes(%s) current value=%s", full_csc_modes, self.full_csc_modes)
    281         if full_csc_modes is not None and type(full_csc_modes)==dict:
    282             self.full_csc_modes = full_csc_modes
     343            ve.clean()
     344            #only send eos if this video encoder is still current,
     345            #(otherwise, sending the new stream will have taken care of it already,
     346            # and sending eos then would close the new stream, not the old one!)
     347            if self.supports_eos and self._video_encoder==ve:
     348                log("sending eos for wid %i", self.wid)
     349                self.queue_packet(("eos", self.wid))
     350            if SAVE_VIDEO_STREAMS:
     351                self.close_video_stream_file()
     352
     353    def close_video_stream_file(self):
     354        vsf = self.video_stream_file
     355        if vsf:
     356            self.video_stream_file = None
     357            try:
     358                vsf.close()
     359            except OSError:
     360                log.error("Error closing video stream file", exc_info=True)
     361
     362    def ui_cleanup(self):
     363        WindowSource.ui_cleanup(self)
     364        self.video_subregion = None
    283365
    284366
     
    287369            #ensure we re-init the codecs asap:
    288370            self.cleanup_codecs()
    289         WindowSource.set_new_encoding(self, encoding, strict)
    290 
    291     def update_encoding_selection(self, encoding=None, exclude=[]):
     371        super().set_new_encoding(encoding, strict)
     372
     373    def update_encoding_selection(self, encoding=None, exclude=None, init=False):
    292374        #override so we don't use encodings that don't have valid csc modes:
    293         log("update_encoding_selection(%s, %s)", encoding, exclude)
     375        log("wvs.update_encoding_selection(%s, %s, %s) full_csc_modes=%s", encoding, exclude, init, self.full_csc_modes)
     376        if exclude is None:
     377            exclude = []
    294378        for x in self.video_encodings:
    295379            if x not in self.core_encodings:
     380                log("video encoding %s not in core encodings", x)
    296381                exclude.append(x)
    297382                continue
    298             csc_modes = self.full_csc_modes.get(x)
    299             csclog("full_csc_modes[%s]=%s", x, csc_modes)
     383            csc_modes = self.full_csc_modes.strtupleget(x)
    300384            if not csc_modes or x not in self.core_encodings:
    301385                exclude.append(x)
    302                 csclog.warn("client does not support any csc modes with %s", x)
    303         WindowSource.update_encoding_selection(self, encoding, exclude)
     386                msg_args = ("Warning: client does not support any csc modes with %s on window %i", x, self.wid)
     387                if not init and first_time("no-csc-%s-%i" % (x, self.wid)):
     388                    log.warn(*msg_args)
     389                else:
     390                    log(*msg_args)
     391        self.common_video_encodings = [x for x in PREFERRED_ENCODING_ORDER if x in self.video_encodings and x in self.core_encodings]
     392        log("update_encoding_options: common_video_encodings=%s, csc_encoder=%s, video_encoder=%s",
     393            self.common_video_encodings, self._csc_encoder, self._video_encoder)
     394        super().update_encoding_selection(encoding, exclude, init)
    304395
    305396    def do_set_client_properties(self, properties):
    306397        #client may restrict csc modes for specific windows
    307         self.parse_csc_modes(properties.dictget("encoding.full_csc_modes", default_value=None))
    308         self.supports_scrolling = self.scroll_encoding and properties.boolget("encoding.scrolling", self.supports_scrolling)
    309         self.supports_video_scaling = properties.boolget("encoding.video_scaling", self.supports_video_scaling)
    310         self.supports_video_subregion = properties.boolget("encoding.video_subregion", self.supports_video_subregion)
    311         self.scaling_control = max(0, min(100, properties.intget("scaling.control", self.scaling_control)))
    312         WindowSource.do_set_client_properties(self, properties)
     398        self.supports_scrolling = "scroll" in self.common_encodings or (
     399            #for older clients, we check an encoding option:
     400            "scroll" in self.server_core_encodings and properties.boolget("scrolling", self.supports_scrolling) and not STRICT_MODE)
     401        self.scroll_min_percent = properties.intget("scrolling.min-percent", self.scroll_min_percent)
     402        self.video_subregion.supported = properties.boolget("encoding.video_subregion", VIDEO_SUBREGION) and VIDEO_SUBREGION
     403        if properties.get("scaling.control") is not None:
     404            self.scaling_control = max(0, min(100, properties.intget("scaling.control", 0)))
     405        super().do_set_client_properties(properties)
    313406        #encodings may have changed, so redo this:
    314407        nv_common = (set(self.server_core_encodings) & set(self.core_encodings)) - set(self.video_encodings)
    315         self.non_video_encodings = [x for x in PREFERED_ENCODING_ORDER if x in nv_common]
    316         try:
    317             self.edge_encoding = [x for x in EDGE_ENCODING_ORDER if x in self.non_video_encodings][0]
    318         except:
    319             self.edge_encoding = None
    320         log("do_set_client_properties(%s) full_csc_modes=%s, video_scaling=%s, video_subregion=%s, non_video_encodings=%s, edge_encoding=%s, scaling_control=%s",
    321             properties, self.full_csc_modes, self.supports_video_scaling, self.supports_video_subregion, self.non_video_encodings, self.edge_encoding, self.scaling_control)
     408        self.non_video_encodings = [x for x in PREFERRED_ENCODING_ORDER if x in nv_common]
     409        if not VIDEO_SKIP_EDGE:
     410            try:
     411                self.edge_encoding = [x for x in EDGE_ENCODING_ORDER if x in self.non_video_encodings][0]
     412            except IndexError:
     413                self.edge_encoding = None
     414        log("do_set_client_properties(%s) full_csc_modes=%s, video_subregion=%s, non_video_encodings=%s, edge_encoding=%s, scaling_control=%s",
     415            properties, self.full_csc_modes, self.video_subregion.supported, self.non_video_encodings, self.edge_encoding, self.scaling_control)
    322416
    323417    def get_best_encoding_impl_default(self):
    324         return self.get_best_encoding_video
    325 
    326 
    327     def get_best_encoding_video(self, pixel_count, ww, wh, speed, quality, current_encoding):
     418        if self.encoding!="grayscale" or has_codec("csc_libyuv"):
     419            if self.common_video_encodings or self.supports_scrolling:
     420                return self.get_best_encoding_video
     421        return super().get_best_encoding_impl_default()
     422
     423
     424    def get_best_encoding_video(self, ww, wh, speed, quality, current_encoding):
    328425        """
    329426            decide whether we send a full window update using the video encoder,
    330427            or if a separate small region(s) is a better choice
    331428        """
    332         def nonvideo(q=quality):
     429        pixel_count = ww*wh
     430        def nonvideo(q=quality, info=""):
    333431            s = max(0, min(100, speed))
    334432            q = max(0, min(100, q))
    335             return self.get_best_nonvideo_encoding(pixel_count, ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings)
     433            log("nonvideo(%i, %s)", q, info)
     434            return self.get_best_nonvideo_encoding(ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings)
    336435
    337436        def lossless(reason):
    338             log("get_best_encoding_video(..) temporarily switching to lossless mode for %8i pixels: %s", pixel_count, reason)
     437            log("get_best_encoding_video(..) temporarily switching to lossless mode for %8i pixels: %s",
     438                pixel_count, reason)
    339439            s = max(0, min(100, speed))
    340440            q = 100
    341             return self.get_best_nonvideo_encoding(pixel_count, ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings)
    342 
    343         if len(self.non_video_encodings)==0:
     441            return self.get_best_nonvideo_encoding(ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings)
     442
     443        #log("get_best_encoding_video%s non_video_encodings=%s, common_video_encodings=%s, supports_scrolling=%s",
     444        #    (pixel_count, ww, wh, speed, quality, current_encoding), self.non_video_encodings, self.common_video_encodings, self.supports_scrolling)
     445
     446        if not self.non_video_encodings:
    344447            return current_encoding
    345 
    346         sr = self.video_subregion.rectangle
    347 
    348         rgbmax = self._rgb_auto_threshold
    349         if sr:
    350             rgbmax = min(rgbmax, sr.width*sr.height//2)
    351         if pixel_count<=rgbmax:
    352             return lossless("low pixel count")
    353 
    354         if current_encoding not in self.video_encodings:
    355             #not doing video, bail out:
    356             return nonvideo()
     448        if not self.common_video_encodings and not self.supports_scrolling:
     449            return nonvideo(info="no common video encodings")
     450        if self.is_tray:
     451            return nonvideo(100, "system tray")
     452        text_hint = self.content_type=="text"
     453        if text_hint and not TEXT_USE_VIDEO:
     454            return nonvideo(info="text content-type")
    357455
    358456        #ensure the dimensions we use for decision making are the ones actually used:
    359457        cww = ww & self.width_mask
    360458        cwh = wh & self.height_mask
    361 
    362         if cww*cwh<=MAX_NONVIDEO_PIXELS:
    363             #window is too small!
    364             return nonvideo()
     459        video_hint = self.content_type=="video"
     460
     461        rgbmax = self._rgb_auto_threshold
     462        videomin = cww*cwh // (1+video_hint*2)
     463        sr = self.video_subregion.rectangle
     464        if sr:
     465            videomin = min(videomin, sr.width * sr.height)
     466            rgbmax = min(rgbmax, sr.width*sr.height//2)
     467        elif not text_hint:
     468            videomin = min(640*480, cww*cwh)
     469        if pixel_count<=rgbmax or cww<8 or cwh<8:
     470            return lossless("low pixel count")
     471
     472        if current_encoding not in ("auto", "grayscale") and current_encoding not in self.common_video_encodings:
     473            return nonvideo(info="%s not a supported video encoding" % current_encoding)
     474
     475        if cww*cwh<=MAX_NONVIDEO_PIXELS or cww<16 or cwh<16:
     476            return nonvideo(quality+30, "window is too small")
    365477
    366478        if cww<self.min_w or cww>self.max_w or cwh<self.min_h or cwh>self.max_h:
    367             #video encoder cannot handle this size!
    368             #(maybe this should be an 'assert' statement here?)
    369             return nonvideo()
    370 
    371         now = time.time()
     479            return nonvideo(info="size out of range for video encoder")
     480
     481        now = monotonic_time()
     482        if now-self.statistics.last_packet_time>1:
     483            return nonvideo(quality+30, "no recent updates")
    372484        if now-self.statistics.last_resized<0.350:
    373             #window has just been resized, may still resize
    374             return nonvideo(q=quality-30)
    375 
    376         lde = list(self.statistics.last_damage_events)
    377         lim = now-2
    378         pixels_last_2secs = sum(w*h for when,_,_,w,h in lde if when>lim)
    379         if pixels_last_2secs<5*min(640*480, cww*cwh):
    380             #less than 5 480p frames / full window updates in last 2 seconds
    381             return nonvideo()
     485            return nonvideo(quality-30, "resized recently")
    382486
    383487        if self._current_quality!=quality or self._current_speed!=speed:
    384             #quality or speed override, best not to force video encoder re-init
    385             return nonvideo()
     488            return nonvideo(info="quality or speed overriden")
    386489
    387490        if sr and ((sr.width&self.width_mask)!=cww or (sr.height&self.height_mask)!=cwh):
    388491            #we have a video region, and this is not it, so don't use video
    389492            #raise the quality as the areas around video tend to not be graphics
    390             return nonvideo(q=quality+30)
    391 
    392         #calculate the threshold for using video vs small regions:
    393         factors = (max(1, (speed-75)/5.0),                      #speed multiplier
    394                    1 + int(self.is_OR or self.is_tray)*2,       #OR windows tend to be static
    395                    max(1, 10-self._sequence),                   #gradual discount the first 9 frames, as the window may be temporary
    396                    1.0 / (int(bool(self._video_encoder)) + 1)   #if we have a video encoder already, make it more likely we'll use it:
    397                    )
    398         max_nvp = int(reduce(operator.mul, factors, MAX_NONVIDEO_PIXELS))
    399         if pixel_count<=max_nvp:
    400             #below threshold
    401             return nonvideo()
    402 
    403         if cww<self.min_w or cww>self.max_w or cwh<self.min_h or cwh>self.max_h:
    404             #failsafe:
    405             return nonvideo()
     493            return nonvideo(quality+30, "not the video region")
     494
     495        if not video_hint and not self.is_shadow:
     496            if now-self.global_statistics.last_congestion_time>5:
     497                lde = tuple(self.statistics.last_damage_events)
     498                lim = now-4
     499                pixels_last_4secs = sum(w*h for when,_,_,w,h in lde if when>lim)
     500                if pixels_last_4secs<((3+text_hint*6)*videomin):
     501                    return nonvideo(quality+30, "not enough frames")
     502                lim = now-1
     503                pixels_last_sec = sum(w*h for when,_,_,w,h in lde if when>lim)
     504                if pixels_last_sec<pixels_last_4secs//8:
     505                    #framerate is dropping?
     506                    return nonvideo(quality+30, "framerate lowered")
     507
     508            #calculate the threshold for using video vs small regions:
     509            factors = (max(1, (speed-75)/5.0),                      #speed multiplier
     510                       1 + int(self.is_OR or self.is_tray)*2,       #OR windows tend to be static
     511                       max(1, 10-self._sequence),                   #gradual discount the first 9 frames, as the window may be temporary
     512                       1.0 / (int(bool(self._video_encoder)) + 1),  #if we have a video encoder already, make it more likely we'll use it:
     513                       )
     514            max_nvp = int(reduce(operator.mul, factors, MAX_NONVIDEO_PIXELS))
     515            if pixel_count<=max_nvp:
     516                #below threshold
     517                return nonvideo(quality+30, "not enough pixels")
    406518        return current_encoding
    407519
    408     def get_best_nonvideo_encoding(self, pixel_count, ww, wh, speed, quality, current_encoding, options=[]):
     520    def get_best_nonvideo_encoding(self, ww, wh, speed, quality, current_encoding=None, options=()):
     521        if self.encoding=="grayscale":
     522            return self.encoding_is_grayscale(ww, wh, speed, quality, current_encoding)
    409523        #if we're here, then the window has no alpha (or the client cannot handle alpha)
    410524        #and we can ignore the current encoding
    411525        options = options or self.non_video_encodings
    412         if pixel_count<self._rgb_auto_threshold:
     526        depth = self.image_depth
     527        if depth==8 and "png/P" in options:
     528            return "png/P"
     529        if self._mmap_size>0 and self.encoding!="grayscale":
     530            return "mmap"
     531        pixel_count = ww*wh
     532        if pixel_count<self._rgb_auto_threshold or self.is_tray or ww<=2 or wh<=2:
    413533            #high speed and high quality, rgb is still good
     534            if self.is_tray and "rgb32" in options:
     535                return "rgb32"
    414536            if "rgb24" in options:
    415537                return "rgb24"
     
    418540        #use sliding scale for lossless threshold
    419541        #(high speed favours switching to lossy sooner)
    420         #take into account how many pixels need to be encoder:
     542        #take into account how many pixels need to be encoded:
    421543        #more pixels means we switch to lossless more easily
    422544        lossless_q = min(100, self._lossless_threshold_base + self._lossless_threshold_pixel_boost * pixel_count / (ww*wh))
    423         if quality<lossless_q:
    424             #lossy options:
    425             if "jpeg" in options:
    426                 #assume that we have "turbojpeg",
    427                 #which beats everything in terms of efficiency for lossy compression:
    428                 return "jpeg"
    429             #avoid large areas (too slow), especially at low speed and high quality:
    430             if "webp" in options and pixel_count>16384:
    431                 max_webp = 1024*1024 * (200-quality)/100 * speed/100
    432                 if speed>30 and pixel_count<max_webp:
    433                     return "webp"
    434         else:
    435             #lossless options:
    436             #webp: don't enable it for "true" lossless (q>99) unless speed is high enough
    437             #because webp forces speed=100 for true lossless mode
    438             #also avoid very small and very large areas (both slow)
    439             if "webp" in options and (quality<100 or speed>=50) and pixel_count>16384:
    440                 max_webp = 1024*1024 * (200-quality)/100 * speed/100
    441                 if pixel_count<max_webp:
    442                     return "webp"
    443             if speed>75:
    444                 if "rgb24" in options:
    445                     return "rgb24"
    446                 if "rgb32" in options:
    447                     return "rgb32"
    448             if "png" in options:
    449                 return "png"
     545        if quality<lossless_q and depth>16 and "jpeg" in options and ww>=8 and wh>=8:
     546            #assume that we have "turbojpeg",
     547            #which beats everything in terms of efficiency for lossy compression:
     548            return "jpeg"
     549        if "webp" in options and pixel_count>=16384 and ww>=2 and wh>=2 and depth in (24, 32):
     550            return "webp"
     551        #lossless options:
     552        if speed==100 or (speed>=95 and pixel_count<MAX_RGB) or depth>24:
     553            if depth>24 and "rgb32" in options:
     554                return "rgb32"
     555            if "rgb24" in options:
     556                return "rgb24"
     557            if "rgb32" in options:
     558                return "rgb32"
     559        if "png" in options:
     560            return "png"
    450561        #we failed to find a good match, default to the first of the options..
    451562        if options:
     
    454565
    455566
    456     def unmap(self):
    457         WindowSource.cancel_damage(self)
    458         self.cleanup_codecs()
     567    def do_damage(self, ww, wh, x, y, w, h, options):
     568        vs = self.video_subregion
     569        if vs:
     570            r = vs.rectangle
     571            if r and r.intersects(x, y, w, h):
     572                #the damage will take care of scheduling it again
     573                vs.cancel_refresh_timer()
     574        super().do_damage(ww, wh, x, y, w, h, options)
     575
    459576
    460577    def cancel_damage(self):
    461         self.video_subregion.cancel_refresh_timer()
    462         self.scroll_data = None
     578        self.cancel_encode_from_queue()
     579        self.free_encode_queue_images()
     580        vsr = self.video_subregion
     581        if vsr:
     582            vsr.cancel_refresh_timer()
     583        self.free_scroll_data()
     584        self.last_scroll_time = 0
    463585        WindowSource.cancel_damage(self)
    464586        #we must clean the video encoder to ensure
     
    467589
    468590
    469     def full_quality_refresh(self, damage_options={}):
    470         #override so we can:
    471         if self.video_subregion.detection:
    472             #reset the video region on full quality refresh
    473             self.video_subregion.reset()
    474         else:
    475             #keep the region, but cancel the refresh:
    476             self.video_subregion.cancel_refresh_timer()
    477         self.scroll_data = None
    478         #refresh the whole window in one go:
    479         damage_options["novideo"] = True
    480         WindowSource.full_quality_refresh(self, damage_options)
     591    def full_quality_refresh(self, damage_options):
     592        vs = self.video_subregion
     593        if vs and vs.rectangle:
     594            if vs.detection:
     595                #reset the video region on full quality refresh
     596                vs.reset()
     597            else:
     598                #keep the region, but cancel the refresh:
     599                vs.cancel_refresh_timer()
     600        self.free_scroll_data()
     601        self.last_scroll_time = 0
     602        if self.non_video_encodings:
     603            #refresh the whole window in one go:
     604            damage_options["novideo"] = True
     605        super().full_quality_refresh(damage_options)
     606
     607    def timer_full_refresh(self):
     608        self.free_scroll_data()
     609        self.last_scroll_time = 0
     610        super().timer_full_refresh()
     611
     612
     613    def quality_changed(self, window, *args):
     614        super().quality_changed(window, args)
     615        self.video_context_clean()
     616        return True
     617
     618    def speed_changed(self, window, *args):
     619        super().speed_changed(window, args)
     620        self.video_context_clean()
     621        return True
    481622
    482623
     
    484625        #force batching when using video region
    485626        #because the video region code is in the send_delayed path
    486         return self.video_subregion.rectangle is not None or WindowSource.must_batch(self, delay)
     627        return self.video_subregion.rectangle is not None or super().must_batch(delay)
    487628
    488629
    489630    def get_speed(self, encoding):
    490         s = WindowSource.get_speed(self, encoding)
     631        s = super().get_speed(encoding)
    491632        #give a boost if we have a video region and this is not video:
    492633        if self.video_subregion.rectangle and encoding not in self.video_encodings:
     
    495636
    496637    def get_quality(self, encoding):
    497         q = WindowSource.get_quality(self, encoding)
     638        q = super().get_quality(encoding)
    498639        #give a boost if we have a video region and this is not video:
    499640        if self.video_subregion.rectangle and encoding not in self.video_encodings:
    500641            q += 40
    501         return q
     642        return min(100, q)
    502643
    503644
     
    505646        #maybe the stream is now corrupted..
    506647        self.cleanup_codecs()
    507         WindowSource.client_decode_error(self, error, message)
     648        super().client_decode_error(error, message)
    508649
    509650
     
    514655    def refresh_subregion(self, regions):
    515656        #callback from video subregion to trigger a refresh of some areas
    516         sublog("refresh_subregion(%s)", regions)
     657        regionrefreshlog("refresh_subregion(%s)", regions)
    517658        if not regions or not self.can_refresh():
    518             return
    519         now = time.time()
     659            return False
     660        now = monotonic_time()
     661        if now-self.global_statistics.last_congestion_time<5:
     662            return False
     663        self.flush_video_encoder_now()
    520664        encoding = self.auto_refresh_encodings[0]
    521665        options = self.get_refresh_options()
    522         WindowSource.do_send_delayed_regions(self, now, regions, encoding, options, get_best_encoding=self.get_refresh_subregion_encoding)
    523 
    524     def get_refresh_subregion_encoding(self, *args):
     666        super().do_send_delayed_regions(now, regions, encoding, options, get_best_encoding=self.get_refresh_subregion_encoding)
     667        return True
     668
     669    def get_refresh_subregion_encoding(self, *_args):
    525670        ww, wh = self.window_dimensions
    526671        w, h = ww, wh
     
    529674        if vr:
    530675            w, h = vr.width, vr.height
    531         return self.get_best_nonvideo_encoding(ww*wh, w, h, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY, self.auto_refresh_encodings[0], self.auto_refresh_encodings)
     676        return self.get_best_nonvideo_encoding(w, h, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY, self.auto_refresh_encodings[0], self.auto_refresh_encodings)
    532677
    533678    def remove_refresh_region(self, region):
    534679        #override so we can update the subregion timers / regions tracking:
    535         WindowSource.remove_refresh_region(self, region)
     680        super().remove_refresh_region(region)
    536681        self.video_subregion.remove_refresh_region(region)
    537682
     
    541686        #don't refresh the video region as part of normal refresh,
    542687        #use subregion refresh for that
     688        sarr = super().add_refresh_region
    543689        vr = self.video_subregion.rectangle
    544690        if vr is None:
    545691            #no video region, normal code path:
    546             return WindowSource.add_refresh_region(self, region)
     692            return sarr(region)
    547693        if vr.contains_rect(region):
    548694            #all of it is in the video region:
     
    552698        if ir is None:
    553699            #region is outside video region, normal code path:
    554             return WindowSource.add_refresh_region(self, region)
     700            return sarr(region)
    555701        #add intersection (rectangle in video region) to video refresh:
    556702        self.video_subregion.add_video_refresh(ir)
    557703        #add any rectangles not in the video region
    558704        #(if any: keep track if we actually added anything)
    559         pixels_modified = 0
    560         for r in region.substract_rect(vr):
    561             pixels_modified += WindowSource.add_refresh_region(self, r)
    562         return pixels_modified
     705        return sum(sarr(r) for r in region.substract_rect(vr))
     706
     707    def matches_video_subregion(self, width, height):
     708        vr = self.video_subregion.rectangle
     709        if not vr:
     710            return None
     711        mw = abs(width - vr.width) & self.width_mask
     712        mh = abs(height - vr.height) & self.height_mask
     713        if mw!=0 or mh!=0:
     714            return None
     715        return vr
     716
     717    def subregion_is_video(self):
     718        vs = self.video_subregion
     719        if not vs:
     720            return False
     721        vr = vs.rectangle
     722        if not vr:
     723            return False
     724        events_count = self.statistics.damage_events_count - vs.set_at
     725        min_video_events = MIN_VIDEO_EVENTS
     726        min_video_fps = MIN_VIDEO_FPS
     727        if self.content_type=="video":
     728            min_video_events //= 2
     729            min_video_fps //= 2
     730        if events_count<min_video_events:
     731            return False
     732        if vs.fps<min_video_fps:
     733            return False
     734        return True
    563735
    564736
     
    567739            Overriden here so we can try to intercept the video_subregion if one exists.
    568740        """
     741        vr = self.video_subregion.rectangle
    569742        #overrides the default method for finding the encoding of a region
    570743        #so we can ensure we don't use the video encoder when we don't want to:
    571744        def send_nonvideo(regions=regions, encoding=coding, exclude_region=None, get_best_encoding=self.get_best_nonvideo_encoding):
     745            if self.b_frame_flush_timer and exclude_region is None:
     746                #a b-frame is already due, don't clobber it!
     747                exclude_region = vr
    572748            WindowSource.do_send_delayed_regions(self, damage_time, regions, encoding, options, exclude_region=exclude_region, get_best_encoding=get_best_encoding)
    573749
    574750        if self.is_tray:
    575751            sublog("BUG? video for tray - don't use video region!")
    576             return send_nonvideo(encoding=None)
    577 
    578         if coding not in self.video_encodings:
    579             sublog("not a video encoding")
     752            send_nonvideo(encoding=None)
     753            return
     754
     755        if coding not in ("auto", "grayscale") and coding not in self.video_encodings:
     756            sublog("not a video encoding: %s" % coding)
    580757            #keep current encoding selection function
    581             return send_nonvideo(get_best_encoding=self.get_best_encoding)
     758            send_nonvideo(get_best_encoding=self.get_best_encoding)
     759            return
    582760
    583761        if options.get("novideo"):
    584762            sublog("video disabled in options")
    585             return send_nonvideo(encoding=None)
    586 
    587         vr = self.video_subregion.rectangle
    588         if not vr or not self.video_subregion.enabled:
     763            send_nonvideo(encoding=None)
     764            return
     765
     766        if not vr:
    589767            sublog("no video region, we may use the video encoder for something else")
    590             WindowSource.do_send_delayed_regions(self, damage_time, regions, coding, options)
     768            super().do_send_delayed_regions(damage_time, regions, coding, options)
    591769            return
    592770        assert not self.full_frames_only
     
    599777            #find how many pixels are within the region (roughly):
    600778            #find all unique regions that intersect with it:
    601             inter = [x for x in (vr.intersection_rect(r) for r in regions) if x is not None]
    602             if len(inter)>0:
     779            inter = tuple(x for x in (vr.intersection_rect(r) for r in regions) if x is not None)
     780            if inter:
    603781                #merge all regions into one:
    604782                in_region = merge_all(inter)
     
    613791            if actual_vr is None:
    614792                #try to find one that has the same dimensions:
    615                 same_d = [r for r in regions if r.width==vr.width and r.height==vr.height]
     793                same_d = tuple(r for r in regions if r.width==vr.width and r.height==vr.height)
    616794                if len(same_d)==1:
    617795                    #probably right..
     
    619797                elif len(same_d)>1:
    620798                    #find one that shares at least one coordinate:
    621                     same_c = [r for r in same_d if r.x==vr.x or r.y==vr.y]
     799                    same_c = tuple(r for r in same_d if r.x==vr.x or r.y==vr.y)
    622800                    if len(same_c)==1:
    623801                        actual_vr = same_c[0]
    624802
    625803        if actual_vr is None:
    626             sublog("send_delayed_regions: video region %s not found in: %s", vr, regions)
     804            sublog("do_send_delayed_regions: video region %s not found in: %s", vr, regions)
    627805        else:
    628806            #found the video region:
     807            #sanity check in case the window got resized since:
     808            ww, wh = self.window.get_dimensions()
     809            if actual_vr.x+actual_vr.width>ww or actual_vr.y+actual_vr.height>wh:
     810                sublog("video region partially outside the window")
     811                send_nonvideo(encoding=None)
     812                return
    629813            #send this using the video encoder:
    630814            video_options = options.copy()
     
    637821                trimmed += r.substract_rect(actual_vr)
    638822            if not trimmed:
    639                 sublog("send_delayed_regions: nothing left after removing video region %s", actual_vr)
     823                sublog("do_send_delayed_regions: nothing left after removing video region %s", actual_vr)
    640824                return
    641             sublog("send_delayed_regions: substracted %s from %s gives us %s", actual_vr, regions, trimmed)
     825            sublog("do_send_delayed_regions: subtracted %s from %s gives us %s", actual_vr, regions, trimmed)
    642826            regions = trimmed
    643827
     
    646830        dr = self._damage_delayed
    647831        if dr:
    648             regions = dr[1] + regions
    649             damage_time = min(damage_time, dr[0])
     832            regions = dr.regions + regions
     833            damage_time = min(damage_time, dr.damage_time)
    650834            self._damage_delayed = None
    651835            self.cancel_expire_timer()
     
    657841        else:
    658842            #non-video is delayed at least 50ms, 4 times the batch delay, but no more than non_max_wait:
    659             elapsed = int(1000.0*(time.time()-damage_time))
    660             delay = max(self.batch_config.delay*4, 50)
     843            elapsed = int(1000.0*(monotonic_time()-damage_time))
     844            delay = max(self.batch_config.delay*4, self.batch_config.expire_delay)
    661845            delay = min(delay, self.video_subregion.non_max_wait-elapsed)
     846            delay = int(delay)
    662847        if delay<=25:
    663             send_nonvideo(regions=regions, encoding=None)
     848            send_nonvideo(regions=regions, encoding=None, exclude_region=actual_vr)
    664849        else:
    665             self._damage_delayed = damage_time, regions, coding, options or {}
    666             sublog("send_delayed_regions: delaying non video regions %s some more by %ims", regions, delay)
    667             self.expire_timer = self.timeout_add(int(delay), self.expire_delayed_region, delay)
     850            self._damage_delayed = DelayedRegions(damage_time, regions, coding, options=options)
     851            sublog("do_send_delayed_regions: delaying non video regions %s some more by %ims", regions, delay)
     852            self.expire_timer = self.timeout_add(delay, self.expire_delayed_region)
    668853
    669854    def must_encode_full_frame(self, encoding):
     
    686871        assert coding is not None
    687872        if w==0 or h==0:
     873            log("process_damage_region: dropped, zero dimensions")
    688874            return
    689875        if not self.window.is_managed():
    690             log("the window %s is not composited!?", self.window)
     876            log("process_damage_region: the window %s is not managed", self.window)
    691877            return
    692878        self._sequence += 1
    693879        sequence = self._sequence
    694880        if self.is_cancelled(sequence):
    695             log("get_window_pixmap: dropping damage request with sequence=%s", sequence)
     881            log("process_damage_region: dropping damage request with sequence=%s", sequence)
    696882            return
    697883
    698         rgb_request_time = time.time()
    699         image = self.window.get_image(x, y, w, h, logger=log)
     884        rgb_request_time = monotonic_time()
     885        image = self.window.get_image(x, y, w, h)
    700886        if image is None:
    701             log("get_window_pixmap: no pixel data for window %s, wid=%s", self.window, self.wid)
     887            log("process_damage_region: no pixel data for window %s, wid=%s", self.window, self.wid)
    702888            return
    703889        if self.is_cancelled(sequence):
     890            log("process_damage_region: dropping damage request with sequence=%s", sequence)
    704891            image.free()
    705892            return
    706893        self.pixel_format = image.get_pixel_format()
    707 
    708         must_freeze = options.get("av-sync", False) or coding in self.video_encodings
     894        self.image_depth = image.get_depth()
     895        #image may have been clipped to the new window size during resize:
     896        w = image.get_width()
     897        h = image.get_height()
     898        if self.send_window_size:
     899            options["window-size"] = self.window_dimensions
     900
     901        av_delay = self.get_frame_encode_delay(options)
     902        #TODO: encode delay can be derived rather than hard-coded
     903        encode_delay = 50
     904        av_delay = max(0, av_delay - encode_delay)
     905        #freeze if:
     906        # * we want av-sync
     907        # * the video encoder needs a thread safe image
     908        #   (the xshm backing may change from underneath us if we don't freeze it)
     909        video_mode = coding in self.video_encodings or coding=="auto"
     910        must_freeze = av_delay>0 or (video_mode and not image.is_thread_safe())
     911        log("process_damage_region: av_delay=%s, must_freeze=%s, size=%s, encoding=%s",
     912            av_delay, must_freeze, (w, h), coding)
    709913        if must_freeze:
    710             newstride = image.get_width()*4
    711             if not image.restride(newstride):
    712                 avsynclog("Warning: failed to freeze image pixels for:")
    713                 avsynclog(" %s", image)
    714         av_delay = self.get_frame_encode_delay(options)
     914            image.freeze()
    715915        def call_encode(ew, eh, eimage, encoding, eflush):
    716916            self._sequence += 1
    717917            sequence = self._sequence
    718918            if self.is_cancelled(sequence):
    719                 log("get_window_pixmap: dropping damage request with sequence=%s", sequence)
     919                log("call_encode: dropping damage request with sequence=%s", sequence)
    720920                return
    721             now = time.time()
    722             log("process_damage_region: wid=%i, adding pixel data to encode queue (%4ix%-4i - %5s), elapsed time: %.1f ms, request time: %.1f ms, frame delay=%ims",
    723                     self.wid, ew, eh, encoding, 1000*(now-damage_time), 1000*(now-rgb_request_time), av_delay)
     921            now = monotonic_time()
     922            log("process_damage_region: wid=%i, sequence=%i, adding pixel data to encode queue (%4ix%-4i - %5s), elapsed time: %3.1f ms, request time: %3.1f ms, frame delay=%3ims",
     923                    self.wid, sequence, ew, eh, encoding, 1000*(now-damage_time), 1000*(now-rgb_request_time), av_delay)
    724924            item = (ew, eh, damage_time, now, eimage, encoding, sequence, options, eflush)
    725             if av_delay<0:
     925            if av_delay<=0:
    726926                self.call_in_encode_thread(True, self.make_data_packet_cb, *item)
    727927            else:
    728928                self.encode_queue.append(item)
    729                 self.timeout_add(av_delay, self.call_in_encode_thread, True, self.encode_from_queue)
     929                self.schedule_encode_from_queue(av_delay)
    730930        #now figure out if we need to send edges separately:
    731         if coding in self.video_encodings and self.edge_encoding and not VIDEO_SKIP_EDGE:
     931        if video_mode and self.edge_encoding:
    732932            dw = w - (w & self.width_mask)
    733933            dh = h - (h & self.height_mask)
    734             if dw>0:
     934            if dw>0 and h>0:
    735935                sub = image.get_sub_image(w-dw, 0, dw, h)
    736936                call_encode(dw, h, sub, self.edge_encoding, flush+1+int(dh>0))
    737937                w = w & self.width_mask
    738             if dh>0:
     938            if dh>0 and w>0:
    739939                sub = image.get_sub_image(0, h-dh, w, dh)
    740940                call_encode(dw, h, sub, self.edge_encoding, flush+1)
    741941                h = h & self.height_mask
    742942        #the main area:
    743         call_encode(w, h, image, coding, flush)
     943        if w>0 and h>0:
     944            call_encode(w, h, image, coding, flush)
    744945
    745946    def get_frame_encode_delay(self, options):
     
    747948            return FORCE_AV_DELAY
    748949        if options.get("av-sync", False):
    749             return -1
     950            return 0
     951        if self.content_type in ("text", "picture"):
     952            return 0
    750953        l = len(self.encode_queue)
    751954        if l>=self.encode_queue_max_size:
     
    754957        return self.av_sync_delay
    755958
     959    def cancel_encode_from_queue(self):
     960        #free all items in the encode queue:
     961        self.encode_from_queue_due = 0
     962        eqt = self.encode_from_queue_timer
     963        avsynclog("cancel_encode_from_queue() timer=%s for wid=%i", eqt, self.wid)
     964        if eqt:
     965            self.encode_from_queue_timer = None
     966            self.source_remove(eqt)
     967
     968    def free_encode_queue_images(self):
     969        eq = self.encode_queue
     970        avsynclog("free_encode_queue_images() freeing %i images for wid=%i", len(eq), self.wid)
     971        if not eq:
     972            return
     973        self.encode_queue = []
     974        for item in eq:
     975            try:
     976                self.free_image_wrapper(item[4])
     977            except Exception:
     978                log.error("Error: cannot free image wrapper %s", item[4], exc_info=True)
     979
     980    def schedule_encode_from_queue(self, av_delay):
     981        #must be called from the UI thread for synchronization
     982        #we ensure that the timer will fire no later than av_delay
     983        #re-scheduling it if it was due later than that
     984        due = monotonic_time()+av_delay/1000.0
     985        if self.encode_from_queue_due==0 or due<self.encode_from_queue_due:
     986            self.cancel_encode_from_queue()
     987            self.encode_from_queue_due = due
     988            self.encode_from_queue_timer = self.timeout_add(av_delay, self.timer_encode_from_queue)
     989
     990    def timer_encode_from_queue(self):
     991        self.encode_from_queue_timer = None
     992        self.encode_from_queue_due = 0
     993        self.call_in_encode_thread(True, self.encode_from_queue)
     994
    756995    def encode_from_queue(self):
    757996        #note: we use a queue here to ensure we preserve the order
    758997        #(so we encode frames in the same order they were grabbed)
    759998        eq = self.encode_queue
    760         avsynclog("encode_from_queue: %s items", len(eq))
     999        avsynclog("encode_from_queue: %s items for wid=%i", len(eq), self.wid)
    7611000        if not eq:
    7621001            return      #nothing to encode, must have been picked off already
    7631002        self.update_av_sync_delay()
    7641003        #find the first item which is due
    765         #in seconds, same as time.time():
    766         av_delay = self.av_sync_delay/1000.0
     1004        #in seconds, same as monotonic_time():
    7671005        if len(self.encode_queue)>=self.encode_queue_max_size:
    7681006            av_delay = 0        #we must free some space!
    769         now = time.time()
     1007        elif FORCE_AV_DELAY>0:
     1008            av_delay = FORCE_AV_DELAY/1000.0
     1009        else:
     1010            av_delay = self.av_sync_delay/1000.0
     1011        now = monotonic_time()
    7701012        still_due = []
    771         pop = None
     1013        remove = []
    7721014        index = 0
    7731015        item = None
     1016        sequence = None
     1017        done_packet = False     #only one packet per iteration
    7741018        try:
    7751019            for index,item in enumerate(eq):
     
    7781022                if self.is_cancelled(sequence):
    7791023                    self.free_image_wrapper(item[4])
     1024                    remove.append(index)
    7801025                    continue
    7811026                ts = item[3]
    7821027                due = ts + av_delay
    783                 if due<now and pop is None:
     1028                if due<=now and not done_packet:
    7841029                    #found an item which is due
    785                     pop = index
    786                     avsynclog("encode_from_queue: processing item %s/%s (overdue by %ims)", index+1, len(self.encode_queue), int(1000*(now-due)))
     1030                    remove.append(index)
     1031                    avsynclog("encode_from_queue: processing item %s/%s (overdue by %ims)",
     1032                              index+1, len(self.encode_queue), int(1000*(now-due)))
    7871033                    self.make_data_packet_cb(*item)
     1034                    done_packet = True
    7881035                else:
    789                     #we only process only one item per call
     1036                    #we only process only one item per call (see "done_packet")
    7901037                    #and just keep track of extra ones:
    791                     still_due.append(due)
     1038                    still_due.append(int(1000*(due-now)))
    7921039        except Exception:
    793             avsynclog.error("error processing encode queue at index %i", index)
    794             avsynclog.error("item=%s", item, exc_info=True)
    795         if pop is not None:
    796             eq.pop(pop)
    797             return
    798         #README: encode_from_queue is scheduled to run every time we add an item
    799         #to the encode_queue, but since the av_delay can change it is possible
    800         #for us to not pop() any items from the list sometimes, and therefore we must ensure
    801         #we run this method again later when the items are actually due,
    802         #so we need to calculate when that is:
    803         if len(still_due)==0:
     1040            if not self.is_cancelled(sequence):
     1041                avsynclog.error("error processing encode queue at index %i", index)
     1042                avsynclog.error("item=%s", item, exc_info=True)
     1043        #remove the items we've dealt with:
     1044        #(in reverse order since we pop them from the queue)
     1045        if remove:
     1046            for x in reversed(remove):
     1047                eq.pop(x)
     1048        #if there are still some items left in the queue, re-schedule:
     1049        if not still_due:
    8041050            avsynclog("encode_from_queue: nothing due")
    8051051            return
    806         first_due = int(max(0, min(still_due)-time.time())*1000)
    807         avsynclog("encode_from_queue: first due in %ims, due list=%s", first_due, still_due)
    808         self.timeout_add(first_due, self.call_in_encode_thread, True, self.encode_from_queue)
    809 
     1052        first_due = max(ENCODE_QUEUE_MIN_GAP, min(still_due))
     1053        avsynclog("encode_from_queue: first due in %ims, due list=%s (av-sync delay=%i, actual=%i, for wid=%i)",
     1054                  first_due, still_due, self.av_sync_delay, av_delay, self.wid)
     1055        self.idle_add(self.schedule_encode_from_queue, first_due)
     1056
     1057    def _more_lossless(self):
     1058        return self.subregion_is_video()
    8101059
    8111060    def update_encoding_options(self, force_reload=False):
     
    8211070            Can be called from any thread.
    8221071        """
    823         WindowSource.update_encoding_options(self, force_reload)
    824         log("update_encoding_options(%s) supports_video_subregion=%s, csc_encoder=%s, video_encoder=%s", force_reload, self.supports_video_subregion, self._csc_encoder, self._video_encoder)
    825         self.video_subregion.set_enabled(self.supports_video_subregion)
    826         if self.supports_video_subregion:
    827             if self.encoding in self.video_encodings and not self.full_frames_only and not STRICT_MODE and len(self.non_video_encodings)>0 and not (self._mmap and self._mmap_size>0):
     1072        super().update_encoding_options(force_reload)
     1073        vs = self.video_subregion
     1074        if vs:
     1075            if (self.encoding not in ("auto", "grayscale") and self.encoding not in self.common_video_encodings) or \
     1076                self.full_frames_only or STRICT_MODE or not self.non_video_encodings or not self.common_video_encodings or \
     1077                self.content_type=="text" or \
     1078                self._mmap_size>0:
     1079                #cannot use video subregions
     1080                #FIXME: small race if a refresh timer is due when we change encoding - meh
     1081                vs.reset()
     1082            else:
     1083                old = vs.rectangle
    8281084                ww, wh = self.window_dimensions
    829                 self.video_subregion.identify_video_subregion(ww, wh, self.statistics.damage_events_count, self.statistics.last_damage_events, self.statistics.last_resized)
    830             else:
    831                 #FIXME: small race if a refresh timer is due when we change encoding - meh
    832                 self.video_subregion.reset()
    833 
    834             if self.video_subregion.rectangle:
    835                 #when we have a video region, lower the lossless threshold
    836                 #especially for small regions
    837                 self._lossless_threshold_base = min(80, 10+self._current_speed//5)
    838                 self._lossless_threshold_pixel_boost = 90-self._current_speed//5
    839 
     1085                vs.identify_video_subregion(ww, wh,
     1086                                            self.statistics.damage_events_count,
     1087                                            self.statistics.last_damage_events,
     1088                                            self.statistics.last_resized,
     1089                                            self.children)
     1090                newrect = vs.rectangle
     1091                if ((newrect is None) ^ (old is None)) or newrect!=old:
     1092                    if old is None and newrect and newrect.get_geometry()==(0, 0, ww, wh):
     1093                        #not actually changed!
     1094                        #the region is the whole window
     1095                        pass
     1096                    elif newrect is None and old and old.get_geometry()==(0, 0, ww, wh):
     1097                        #not actually changed!
     1098                        #the region is the whole window
     1099                        pass
     1100                    else:
     1101                        videolog("video subregion was %s, now %s (window size: %i,%i)", old, newrect, ww, wh)
     1102                        self.cleanup_codecs()
     1103                if newrect:
     1104                    #remove this from regular refresh:
     1105                    if old is None or old!=newrect:
     1106                        refreshlog("identified new video region: %s", newrect)
     1107                        #figure out if the new region had pending regular refreshes:
     1108                        subregion_needs_refresh = any(newrect.intersects_rect(x) for x in self.refresh_regions)
     1109                        if old:
     1110                            #we don't bother substracting new and old (too complicated)
     1111                            refreshlog("scheduling refresh of old region: %s", old)
     1112                            #this may also schedule a refresh:
     1113                            super().add_refresh_region(old)
     1114                        super().remove_refresh_region(newrect)
     1115                        if not self.refresh_regions:
     1116                            self.cancel_refresh_timer()
     1117                        if subregion_needs_refresh:
     1118                            vs.add_video_refresh(newrect)
     1119                    else:
     1120                        refreshlog("video region unchanged: %s - no change in refresh", newrect)
     1121                elif old:
     1122                    #add old region to regular refresh:
     1123                    refreshlog("video region cleared, scheduling refresh of old region: %s", old)
     1124                    self.add_refresh_region(old)
     1125                    vs.cancel_refresh_timer()
    8401126        if force_reload:
    8411127            self.cleanup_codecs()
     
    8501136            Can be called from any thread.
    8511137        """
    852         elapsed = time.time()-self._last_pipeline_check
    853         if not force_reload and elapsed<0.75:
    854             scorelog("cannot score: only %ims since last check", 1000*elapsed)
     1138        if self._mmap_size>0:
     1139            scorelog("cannot score: mmap enabled")
     1140            return
     1141        if self.content_type=="text" and self.non_video_encodings:
     1142            scorelog("no pipelines for 'text' content-type")
     1143            return
     1144        elapsed = monotonic_time()-self._last_pipeline_check
     1145        max_elapsed = 0.75
     1146        if self.is_idle:
     1147            max_elapsed = 60
     1148        if not force_reload and elapsed<max_elapsed:
     1149            scorelog("cannot score: only %ims since last check (idle=%s)", 1000*elapsed, self.is_idle)
    8551150            #already checked not long ago
    8561151            return
     
    8661161            self.cleanup_codecs()
    8671162        #do some sanity checks to see if there is any point in finding a suitable video encoding pipeline:
    868         encoding = self.encoding
    869         if encoding not in self.video_encodings:
    870             return checknovideo("non-video encoding: %s", encoding)
    871         if self._sequence<2 or self._damage_cancelled>=float("inf"):
     1163        if self._sequence<2 or self.is_cancelled():
    8721164            #too early, or too late!
    8731165            return checknovideo("sequence=%s (cancelled=%s)", self._sequence, self._damage_cancelled)
     1166        #which video encodings to evaluate:
     1167        if self.encoding in ("auto", "grayscale"):
     1168            eval_encodings = self.common_video_encodings
     1169        else:
     1170            if self.encoding not in self.common_video_encodings:
     1171                return checknovideo("non-video / unsupported encoding: %s", self.encoding)
     1172            eval_encodings = [self.encoding]
    8741173        ww, wh = self.window_dimensions
    8751174        w = ww & self.width_mask
     
    8821181                h = r.height & self.width_mask
    8831182        if w<self.min_w or w>self.max_w or h<self.min_h or h>self.max_h:
    884             return checknovideo("out of bounds: %sx%s (min %sx%s, max %sx%s)", w, h, self.min_w, self.min_h, self.max_w, self.max_h)
    885         if time.time()-self.statistics.last_resized<0.500:
    886             return checknovideo("resized just %.1f seconds ago", time.time()-self.statistics.last_resized)
     1183            return checknovideo("out of bounds: %sx%s (min %sx%s, max %sx%s)",
     1184                                w, h, self.min_w, self.min_h, self.max_w, self.max_h)
     1185        #if monotonic_time()-self.statistics.last_resized<0.500:
     1186        #    return checknovideo("resized just %.1f seconds ago", monotonic_time()-self.statistics.last_resized)
    8871187
    8881188        #must copy reference to those objects because of threading races:
     
    8901190        csce = self._csc_encoder
    8911191        if ve is not None and ve.is_closed():
    892             scorelog("cannot score: video encoder is closed or closing")
     1192            scorelog("cannot score: video encoder %s is closed or closing", ve)
    8931193            return
    8941194        if csce is not None and csce.is_closed():
    895             scorelog("cannot score: csc is closed or closing")
     1195            scorelog("cannot score: csc %s is closed or closing", csce)
    8961196            return
    8971197
    898         scores = self.get_video_pipeline_options(encoding, w, h, self.pixel_format, force_reload)
    899         if len(scores)==0:
     1198        scores = self.get_video_pipeline_options(eval_encodings, w, h, self.pixel_format, force_reload)
     1199        if not scores:
    9001200            scorelog("check_pipeline_score(%s) no pipeline options found!", force_reload)
    9011201            return
     
    9031203        scorelog("check_pipeline_score(%s) best=%s", force_reload, scores[0])
    9041204        _, _, _, csc_width, csc_height, csc_spec, enc_in_format, _, enc_width, enc_height, encoder_spec = scores[0]
     1205        clean = False
    9051206        if csce:
    9061207            if csc_spec is None:
    907                 scorelog("check_pipeline_score(%s) csc is no longer needed: %s", force_reload, scores[0])
    908                 self.csc_encoder_clean()
     1208                scorelog("check_pipeline_score(%s) csc is no longer needed: %s",
     1209                         force_reload, scores[0])
     1210                clean = True
    9091211            elif csce.get_dst_format()!=enc_in_format:
    910                 scorelog("check_pipeline_score(%s) change of csc output format from %s to %s", force_reload, csce.get_dst_format(), enc_in_format)
    911                 self.csc_encoder_clean()
     1212                scorelog("check_pipeline_score(%s) change of csc output format from %s to %s",
     1213                         force_reload, csce.get_dst_format(), enc_in_format)
     1214                clean = True
    9121215            elif csce.get_src_width()!=csc_width or csce.get_src_height()!=csc_height:
    913                 scorelog("check_pipeline_score(%s) change of csc input dimensions from %ix%i to %ix%i", force_reload, csce.get_src_width(), csce.get_src_height(), csc_width, csc_height)
    914                 self.csc_encoder_clean()
    915         if ve is None:
     1216                scorelog("check_pipeline_score(%s) change of csc input dimensions from %ix%i to %ix%i",
     1217                         force_reload, csce.get_src_width(), csce.get_src_height(), csc_width, csc_height)
     1218                clean = True
     1219            elif csce.get_dst_width()!=enc_width or csce.get_dst_height()!=enc_height:
     1220                scorelog("check_pipeline_score(%s) change of csc ouput dimensions from %ix%i to %ix%i",
     1221                         force_reload, csce.get_dst_width(), csce.get_dst_height(), enc_width, enc_height)
     1222                clean = True
     1223        if ve is None or clean:
    9161224            pass    #nothing to check or clean
    9171225        elif ve.get_src_format()!=enc_in_format:
    918             scorelog("check_pipeline_score(%s) change of video input format from %s to %s", force_reload, ve.get_src_format(), enc_in_format)
    919             self.video_encoder_clean()
     1226            scorelog("check_pipeline_score(%s) change of video input format from %s to %s",
     1227                     force_reload, ve.get_src_format(), enc_in_format)
     1228            clean = True
    9201229        elif ve.get_width()!=enc_width or ve.get_height()!=enc_height:
    921             scorelog("check_pipeline_score(%s) change of video input dimensions from %ix%i to %ix%i", force_reload, ve.get_width(), ve.get_height(), enc_width, enc_height)
    922             self.video_encoder_clean()
    923         elif type(ve)!=encoder_spec.codec_class:
    924             scorelog("check_pipeline_score(%s) found a better video encoder class than %s: %s", force_reload, type(ve), scores[0])
    925             self.video_encoder_clean()
    926         self._last_pipeline_check = time.time()
    927 
    928 
    929     def get_video_pipeline_options(self, encoding, width, height, src_format, force_refresh=False):
     1230            scorelog("check_pipeline_score(%s) change of video input dimensions from %ix%i to %ix%i",
     1231                     force_reload, ve.get_width(), ve.get_height(), enc_width, enc_height)
     1232            clean = True
     1233        elif not isinstance(ve, encoder_spec.codec_class):
     1234            scorelog("check_pipeline_score(%s) found a better video encoder class than %s: %s",
     1235                     force_reload, type(ve), scores[0])
     1236            clean = True
     1237        if clean:
     1238            self.video_context_clean()
     1239        self._last_pipeline_check = monotonic_time()
     1240
     1241
     1242    def get_video_pipeline_options(self, encodings, width, height, src_format, force_refresh=False):
    9301243        """
    9311244            Given a picture format (width, height and src pixel format),
    9321245            we find all the pipeline options that will allow us to compress
    933             it using the given encoding.
     1246            it using the given encodings.
    9341247            First, we try with direct encoders (for those that support the
    9351248            source pixel format natively), then we try all the combinations
     
    9421255            Can be called from any thread.
    9431256        """
    944         if not force_refresh and (time.time()-self.last_pipeline_time<1) and self.last_pipeline_params and self.last_pipeline_params==(encoding, width, height, src_format):
     1257        if not force_refresh and (monotonic_time()-self.last_pipeline_time<1) and self.last_pipeline_params and self.last_pipeline_params==(encodings, width, height, src_format):
    9451258            #keep existing scores
    946             scorelog("get_video_pipeline_options%s using cached values from %ims ago", (encoding, width, height, src_format, force_refresh), 1000.0*(time.time()-self.last_pipeline_time))
     1259            scorelog("get_video_pipeline_options%s using cached values from %ims ago",
     1260                     (encodings, width, height, src_format, force_refresh), 1000.0*(monotonic_time()-self.last_pipeline_time))
    9471261            return self.last_pipeline_scores
     1262        scorelog("get_video_pipeline_options%s last params=%s, full_csc_modes=%s",
     1263                 (encodings, width, height, src_format, force_refresh), self.last_pipeline_params, self.full_csc_modes)
    9481264
    9491265        vh = self.video_helper
    9501266        if vh is None:
    951             return []       #closing down
    952 
    953         #these are the CSC modes the client can handle for this encoding:
    954         #we must check that the output csc mode for each encoder is one of those
    955         supported_csc_modes = self.full_csc_modes.get(encoding)
    956         if not supported_csc_modes:
    957             scorelog("get_video_pipeline_options: no supported csc modes for %s", encoding)
    958             return []
    959         encoder_specs = vh.get_encoder_specs(encoding)
    960         if not encoder_specs:
    961             scorelog("get_video_pipeline_options: no encoder specs for %s", encoding)
    962             return []
     1267            return ()       #closing down
     1268
    9631269        target_q = int(self._current_quality)
    9641270        min_q = self._fixed_min_quality
     
    9661272        min_s = self._fixed_min_speed
    9671273        #tune quality target for (non-)video region:
    968         vr = self.video_subregion.rectangle
    969         if self.video_subregion.enabled and vr:
    970             mw = abs(width - vr.width) & self.width_mask
    971             mh = abs(height - vr.height) & self.height_mask
    972             if mw==0 and mh==0:
    973                 if target_q<100:
    974                     #dealing with the video region at less than 100% quality,
    975                     #lower quality a bit more since this is definitely video:
    976                     target_q = max(min_q, int(target_q*0.75))
    977                     scorelog("lowering quality for video encoding of video region")
     1274        vr = self.matches_video_subregion(width, height)
     1275        if vr and target_q<100:
     1276            if self.subregion_is_video():
     1277                #lower quality a bit more:
     1278                fps = self.video_subregion.fps
     1279                f = min(90, 2*fps)
     1280                target_q = max(min_q, int(target_q*(100-f)//100))
     1281                scorelog("lowering quality target %i by %i%% for video %s (fps=%i)", target_q, f, vr, fps)
    9781282            else:
    979                 if target_q<100:
    980                     #not the video region, raise quality a bit:
    981                     target_q = min(100, int(target_q*1.33))
    982                     scorelog("raising quality for video encoding of non-video region")
    983         scorelog("get_video_pipeline_options%s speed: %s (min %s), quality: %s (min %s)", (encoding, width, height, src_format), target_s, min_s, target_q, min_q)
     1283                #not the video region, or not really video content, raise quality a bit:
     1284                target_q = int(sqrt(target_q/100.0)*100)
     1285                scorelog("raising quality for video encoding of non-video region")
     1286        scorelog("get_video_pipeline_options%s speed: %s (min %s), quality: %s (min %s)",
     1287                 (encodings, width, height, src_format), target_s, min_s, target_q, min_q)
     1288        vmw, vmh = self.video_max_size
     1289        ffps = self.get_video_fps(width, height)
    9841290        scores = []
    985         def add_scores(info, csc_spec, enc_in_format):
    986             scorelog("add_scores(%s, %s, %s)", info, csc_spec, enc_in_format)
    987             #find encoders that take 'enc_in_format' as input:
    988             colorspace_specs = encoder_specs.get(enc_in_format)
    989             if not colorspace_specs:
    990                 scorelog("add_scores: no matching colorspace specs for %s", enc_in_format)
    991                 return
    992             #log("%s encoding from %s: %s", info, pixel_format, colorspace_specs)
    993             for encoder_spec in colorspace_specs:
    994                 #ensure that the output of the encoder can be processed by the client:
    995                 matches = set(encoder_spec.output_colorspaces) & set(supported_csc_modes)
    996                 if not matches:
    997                     scorelog("add_scores: no matches for %s (%s and %s)", encoder_spec, encoder_spec.output_colorspaces, supported_csc_modes)
    998                     continue
    999                 scaling = self.calculate_scaling(width, height, encoder_spec.max_w, encoder_spec.max_h)
    1000                 score_data = get_pipeline_score(enc_in_format, csc_spec, encoder_spec, width, height, scaling,
    1001                                                 target_q, min_q, target_s, min_s,
    1002                                                 self._csc_encoder, self._video_encoder)
    1003                 if score_data:
    1004                     scores.append(score_data)
    1005         if not FORCE_CSC or src_format==FORCE_CSC_MODE:
    1006             add_scores("direct (no csc)", None, src_format)
    1007 
    1008         #now add those that require a csc step:
    1009         csc_specs = vh.get_csc_specs(src_format)
    1010         if csc_specs:
    1011             #log("%s can also be converted to %s using %s", pixel_format, [x[0] for x in csc_specs], set(x[1] for x in csc_specs))
    1012             #we have csc module(s) that can get us from pixel_format to out_csc:
    1013             for out_csc, l in csc_specs.items():
    1014                 actual_csc = self.csc_equiv(out_csc)
    1015                 if not bool(FORCE_CSC_MODE) or FORCE_CSC_MODE==out_csc:
    1016                     for csc_spec in l:
    1017                         add_scores("via %s (%s)" % (out_csc, actual_csc), csc_spec, out_csc)
     1291        for encoding in encodings:
     1292            #these are the CSC modes the client can handle for this encoding:
     1293            #we must check that the output csc mode for each encoder is one of those
     1294            supported_csc_modes = self.full_csc_modes.strtupleget(encoding)
     1295            if not supported_csc_modes:
     1296                scorelog(" no supported csc modes for %s", encoding)
     1297                continue
     1298            encoder_specs = vh.get_encoder_specs(encoding)
     1299            if not encoder_specs:
     1300                scorelog(" no encoder specs for %s", encoding)
     1301                continue
     1302            #if not specified as an encoding option,
     1303            #discount encodings further down the list of preferred encodings:
     1304            #(ie: prefer h264 to vp9)
     1305            try:
     1306                encoding_score_delta = len(PREFERRED_ENCODING_ORDER)//2-PREFERRED_ENCODING_ORDER.index(encoding)
     1307            except ValueError:
     1308                encoding_score_delta = 0
     1309            encoding_score_delta = self.encoding_options.get("%s.score-delta" % encoding, encoding_score_delta)
     1310            def add_scores(info, csc_spec, enc_in_format):
     1311                #find encoders that take 'enc_in_format' as input:
     1312                colorspace_specs = encoder_specs.get(enc_in_format)
     1313                if not colorspace_specs:
     1314                    scorelog(" no matching colorspace specs for %s - %s", enc_in_format, info)
     1315                    return
     1316                #log("%s encoding from %s: %s", info, pixel_format, colorspace_specs)
     1317                for encoder_spec in colorspace_specs:
     1318                    #ensure that the output of the encoder can be processed by the client:
     1319                    matches = tuple(x for x in encoder_spec.output_colorspaces if x in supported_csc_modes)
     1320                    if not matches or self.is_cancelled():
     1321                        scorelog(" no matches for %s (%s and %s) - %s",
     1322                                 encoder_spec, encoder_spec.output_colorspaces, supported_csc_modes, info)
     1323                        continue
     1324                    max_w = min(encoder_spec.max_w, vmw)
     1325                    max_h = min(encoder_spec.max_h, vmh)
     1326                    if (csc_spec and csc_spec.can_scale) or encoder_spec.can_scale:
     1327                        scaling = self.calculate_scaling(width, height, max_w, max_h)
     1328                    else:
     1329                        scaling = (1, 1)
     1330                    score_delta = encoding_score_delta
     1331                    if self.is_shadow and enc_in_format in ("NV12", "YUV420P", "YUV422P") and scaling==(1, 1):
     1332                        #avoid subsampling with shadow servers:
     1333                        score_delta -= 40
     1334                    vs = self.video_subregion
     1335                    detection = bool(vs) and vs.detection
     1336                    score_data = get_pipeline_score(enc_in_format, csc_spec, encoder_spec, width, height, scaling,
     1337                                                    target_q, min_q, target_s, min_s,
     1338                                                    self._csc_encoder, self._video_encoder,
     1339                                                    score_delta, ffps, detection)
     1340                    if score_data:
     1341                        scores.append(score_data)
     1342                    else:
     1343                        scorelog(" no score data for %s",
     1344                                 (enc_in_format, csc_spec, encoder_spec, width, height, scaling, ".."))
     1345            if not FORCE_CSC or src_format==FORCE_CSC_MODE:
     1346                add_scores("direct (no csc)", None, src_format)
     1347
     1348            #now add those that require a csc step:
     1349            csc_specs = vh.get_csc_specs(src_format)
     1350            if csc_specs:
     1351                #log("%s can also be converted to %s using %s",
     1352                #    pixel_format, [x[0] for x in csc_specs], set(x[1] for x in csc_specs))
     1353                #we have csc module(s) that can get us from pixel_format to out_csc:
     1354                for out_csc, l in csc_specs.items():
     1355                    if not bool(FORCE_CSC_MODE) or FORCE_CSC_MODE==out_csc:
     1356                        for csc_spec in l:
     1357                            add_scores("via %s" % out_csc, csc_spec, out_csc)
    10181358        s = sorted(scores, key=lambda x : -x[0])
    1019         scorelog("get_video_pipeline_options%s scores=%s", (encoding, width, height, src_format), s)
    1020         self.last_pipeline_params = (encoding, width, height, src_format)
    1021         self.last_pipeline_scores = s
    1022         self.last_pipeline_time = time.time()
     1359        scorelog("get_video_pipeline_options%s scores=%s", (encodings, width, height, src_format), s)
     1360        if self.is_cancelled():
     1361            self.last_pipeline_params = None
     1362            self.last_pipeline_scores = ()
     1363        else:
     1364            self.last_pipeline_params = (encodings, width, height, src_format)
     1365            self.last_pipeline_scores = s
     1366        self.last_pipeline_time = monotonic_time()
    10231367        return s
    10241368
    1025     def csc_equiv(self, csc_mode):
    1026         #in some places, we want to check against the subsampling used
    1027         #and not the colorspace itself.
    1028         #and NV12 uses the same subsampling as YUV420P...
    1029         return {"NV12" : "YUV420P",
    1030                 "BGRX" : "YUV444P"}.get(csc_mode, csc_mode)
    1031 
     1369
     1370    def get_video_fps(self, width, height):
     1371        mvsub = self.matches_video_subregion(width, height)
     1372        vs = self.video_subregion
     1373        if vs and mvsub:
     1374            #matches the video subregion,
     1375            #for which we have the fps already:
     1376            return self.video_subregion.fps
     1377        return self.do_get_video_fps(width, height)
     1378
     1379    def do_get_video_fps(self, width, height):
     1380        now = monotonic_time()
     1381        #calculate full frames per second (measured in pixels vs window size):
     1382        stime = now-5           #only look at the last 5 seconds max
     1383        lde = tuple((t,w,h) for t,_,_,w,h in tuple(self.statistics.last_damage_events) if t>stime)
     1384        if len(lde)>=10:
     1385            #the first event's first element is the oldest event time:
     1386            otime = lde[0][0]
     1387            if now>otime:
     1388                pixels = sum(w*h for _,w,h in lde)
     1389                return int(pixels/(width*height)/(now - otime))
     1390        return 0
    10321391
    10331392    def calculate_scaling(self, width, height, max_w=4096, max_h=4096):
     1393        if width==0 or height==0:
     1394            return (1, 1)
    10341395        q = self._current_quality
    10351396        s = self._current_speed
    1036         actual_scaling = self.scaling
    1037         def get_min_required_scaling():
    1038             if width<=max_w and height<=max_h:
    1039                 return (1, 1)       #no problem
     1397        now = monotonic_time()
     1398        crs = self.client_render_size
     1399        def get_min_required_scaling(default_value=(1, 1)):
     1400            mw = max_w
     1401            mh = max_h
     1402            if crs:
     1403                #if the client is going to downscale things anyway,
     1404                #then there is no need to send at a higher resolution than that:
     1405                crsw, crsh = crs
     1406                if crsw<max_w:
     1407                    mw = crsw
     1408                if crsh<max_h:
     1409                    mh = crsh
     1410            if width<=mw and height<=mh:
     1411                return default_value    #no problem
    10401412            #most encoders can't deal with that!
    1041             TRY_SCALE = ((2, 3), (1, 2), (1, 3), (1, 4), (1, 8), (1, 10))
    1042             for op, d in TRY_SCALE:
    1043                 if width*op/d<=max_w and height*op/d<=max_h:
    1044                     return (op, d)
    1045             raise Exception("BUG: failed to find a scaling value for window size %sx%s", width, height)
    1046         if not SCALING or not self.supports_video_scaling:
    1047             #not supported by client or disabled by env
    1048             #FIXME: what to do if width>max_w or height>max_h?
    1049             actual_scaling = 1, 1
     1413            #sort them from smallest scaling to highest:
     1414            sopts = {}
     1415            for num, den in SCALING_OPTIONS:
     1416                sopts[num/den] = (num, den)
     1417            for ratio in reversed(sorted(sopts.keys())):
     1418                num, den = sopts[ratio]
     1419                if num==1 and den==1:
     1420                    continue
     1421                if width*num/den<=mw and height*num/den<=mh:
     1422                    return (num, den)
     1423            raise Exception("BUG: failed to find a scaling value for window size %sx%s" % (width, height))
     1424        if not SCALING:
     1425            if (width>max_w or height>max_h) and first_time("scaling-required"):
     1426                if not SCALING:
     1427                    scalinglog.warn("Warning: video scaling is disabled")
     1428                else:
     1429                    scalinglog.warn("Warning: video scaling is not supported by the client")
     1430                scalinglog.warn(" but the video size is too large: %ix%i", width, height)
     1431                scalinglog.warn(" the maximum supported is %ix%i", max_w, max_h)
     1432            scaling = 1, 1
     1433        elif SCALING_HARDCODED:
     1434            scaling = get_min_required_scaling(tuple(SCALING_HARDCODED))
     1435            scalinglog("using hardcoded scaling value: %s", scaling)
    10501436        elif self.scaling_control==0:
    1051             #only enable if we have to:
    1052             actual_scaling = get_min_required_scaling()
    1053         elif SCALING_HARDCODED:
    1054             actual_scaling = tuple(SCALING_HARDCODED)
    1055             scalinglog("using hardcoded scaling: %s", actual_scaling)
    1056         elif actual_scaling is None and (width>max_w or height>max_h):
    1057             #most encoders can't deal with that!
    1058             actual_scaling = get_min_required_scaling()
    1059         elif actual_scaling is None and not self.is_shadow and self.statistics.damage_events_count>50 and (time.time()-self.statistics.last_resized)>0.5:
    1060             #no scaling window attribute defined, so use heuristics to enable:
    1061             #full frames per second (measured in pixels vs window size):
    1062             ffps = 0
    1063             stime = time.time()-5           #only look at the last 5 seconds max
    1064             lde = [x for x in list(self.statistics.last_damage_events) if x[0]>stime]
    1065             if len(lde)>10:
    1066                 #the first event's first element is the oldest event time:
    1067                 otime = lde[0][0]
    1068                 pixels = sum(w*h for _,_,_,w,h in lde)
    1069                 ffps = int(pixels/(width*height)/(time.time() - otime))
    1070 
    1071             #edge resistance for changing the current scaling value:
    1072             er = 0
    1073             if self.actual_scaling!=(1, 1):
    1074                 #if we are currently downscaling, stick with it a bit longer:
    1075                 #more so if we are downscaling a lot (1/3 -> er=1.5 + ..)
    1076                 #and yet even more if scaling_control is high (scaling_control=100 -> er= .. + 1)
    1077                 er = (0.5 * self.actual_scaling[1] / self.actual_scaling[0]) + self.scaling_control/100.0
    1078             qs = s>(q-er*10) and q<(70+er*15)
    1079             #scalinglog("calculate_scaling: er=%.1f, qs=%s, ffps=%s", er, qs, ffps)
    1080 
    1081             if self.fullscreen and (qs or ffps>=max(2, 10-er*3)):
    1082                 actual_scaling = 1,3
    1083             elif self.maximized and (qs or ffps>=max(2, 10-er*3)):
    1084                 actual_scaling = 1,2
    1085             elif width*height>=(2560-er*768)*1600 and (qs or ffps>=max(4, 25-er*5)):
    1086                 actual_scaling = 1,3
    1087             elif width*height>=(1920-er*384)*1200 and (qs or ffps>=max(5, 30-er*10)):
    1088                 actual_scaling = 2,3
    1089             elif width*height>=(1200-er*256)*1024 and (qs or ffps>=max(10, 50-er*15)):
    1090                 actual_scaling = 2,3
    1091             if actual_scaling:
    1092                 scalinglog("calculate_scaling enabled by heuristics er=%.1f, qs=%s, ffps=%s", er, qs, ffps)
    1093         if actual_scaling is None:
    1094             actual_scaling = 1, 1
    1095         v, u = actual_scaling
     1437            #video-scaling is disabled, only use scaling if we really have to:
     1438            scaling = get_min_required_scaling()
     1439        elif self.scaling:
     1440            #honour value requested for this window, unless we must scale more:
     1441            scaling = get_min_required_scaling(self.scaling)
     1442        elif (now-self.statistics.last_resized<0.5) or (now-self.last_scroll_time)<5:
     1443            #don't change during window resize or scrolling:
     1444            scaling = get_min_required_scaling(self.actual_scaling)
     1445        elif self.statistics.damage_events_count<=50:
     1446            #not enough data yet:
     1447            scaling = get_min_required_scaling()
     1448        else:
     1449            #use heuristics to choose the best scaling ratio:
     1450            mvsub = self.matches_video_subregion(width, height)
     1451            video = self.content_type=="video" or (bool(mvsub) and self.subregion_is_video())
     1452            ffps = self.get_video_fps(width, height)
     1453
     1454            if self.scaling_control is None:
     1455                #None==auto mode, derive from quality and speed only:
     1456                q_noscaling = 80 + int(video)*10
     1457                if q>=q_noscaling or ffps==0:
     1458                    scaling = get_min_required_scaling()
     1459                else:
     1460                    pps = ffps*width*height                 #Pixels/s
     1461                    if self.bandwidth_limit>0:
     1462                        #assume video compresses pixel data by ~95% (size is 20 times smaller)
     1463                        #(and convert to bytes per second)
     1464                        #ie: 240p minimum target
     1465                        target = max(SCALING_MIN_PPS, self.bandwidth_limit//8*20)
     1466                    else:
     1467                        target = SCALING_PPS_TARGET             #ie: 1080p
     1468                    if self.is_shadow:
     1469                        #shadow servers look ugly when scaled:
     1470                        target *= 16
     1471                    elif self.content_type=="text":
     1472                        #try to avoid scaling:
     1473                        target *= 4
     1474                    elif not video:
     1475                        #downscale non-video content less:
     1476                        target *= 2
     1477                    if self.image_depth==30:
     1478                        #high bit depth is normally used for high quality
     1479                        target *= 10
     1480                    #high quality means less scaling:
     1481                    target = target * (10+q)**2 // 50**2
     1482                    #high speed means more scaling:
     1483                    target = target * 60**2 // (q+20)**2
     1484                    sscaling = {}
     1485                    mrs = get_min_required_scaling()
     1486                    min_ratio = mrs[0]/mrs[1]
     1487                    for num, denom in SCALING_OPTIONS:
     1488                        #scaled pixels per second value:
     1489                        spps = pps*(num**2)/(denom**2)
     1490                        ratio = target/spps
     1491                        #ideal ratio is 1, measure distance from 1:
     1492                        score = int(abs(1-ratio)*100)
     1493                        if self.actual_scaling and self.actual_scaling==(num, denom) and (num!=1 or denom!=1):
     1494                            #if we are already downscaling,
     1495                            #try to stick to the same value longer:
     1496                            #give it a score boost (lowest score wins):
     1497                            score = int(score/1.5)
     1498                        if num/denom>min_ratio:
     1499                            #higher than minimum, should not be used unless we have no choice:
     1500                            score = int(score*100)
     1501                        sscaling[score] = (num, denom)
     1502                    scalinglog("calculate_scaling%s wid=%i, pps=%s, target=%s, scores=%s",
     1503                               (width, height, max_w, max_h), self.wid, pps, target, sscaling)
     1504                    if sscaling:
     1505                        highscore = sorted(sscaling.keys())[0]
     1506                        scaling = sscaling[highscore]
     1507                    else:
     1508                        scaling = get_min_required_scaling()
     1509            else:
     1510                #calculate scaling based on the "video-scaling" command line option,
     1511                #which is named "scaling_control" here.
     1512                #(from 1 to 100, from least to most aggressive)
     1513                if mvsub:
     1514                    if video:
     1515                        #enable scaling more aggressively
     1516                        sc = (self.scaling_control+50)*2
     1517                    else:
     1518                        sc = (self.scaling_control+25)
     1519                else:
     1520                    #not the video region, so much less aggressive scaling:
     1521                    sc = max(0, (self.scaling_control-50)//2)
     1522
     1523                #if scaling_control is high (scaling_control=100 -> er=2)
     1524                #then we will match the heuristics more quickly:
     1525                er = sc/50.0
     1526                if self.actual_scaling!=(1, 1):
     1527                    #if we are already downscaling, boost so we will stick with it a bit longer:
     1528                    #more so if we are downscaling a lot (1/3 -> er=1.5 + ..)
     1529                    er += (0.5 * self.actual_scaling[1] / self.actual_scaling[0])
     1530                qs = s>(q-er*10) and q<(50+er*15)
     1531                #scalinglog("calculate_scaling: er=%.1f, qs=%s, ffps=%s", er, qs, ffps)
     1532                if self.fullscreen and (qs or ffps>=max(2, 10-er*3)):
     1533                    scaling = 1,3
     1534                elif self.maximized and (qs or ffps>=max(2, 10-er*3)):
     1535                    scaling = 1,2
     1536                elif width*height>=(2560-er*768)*1600 and (qs or ffps>=max(4, 25-er*5)):
     1537                    scaling = 1,3
     1538                elif width*height>=(1920-er*384)*1200 and (qs or ffps>=max(5, 30-er*10)):
     1539                    scaling = 2,3
     1540                elif width*height>=(1200-er*256)*1024 and (qs or ffps>=max(10, 50-er*15)):
     1541                    scaling = 2,3
     1542                else:
     1543                    scaling = 1,1
     1544                if scaling:
     1545                    scalinglog("calculate_scaling value %s enabled by heuristics for %ix%i q=%i, s=%i, er=%.1f, qs=%s, ffps=%i, scaling-control(%i)=%i",
     1546                               scaling, width, height, q, s, er, qs, ffps, self.scaling_control, sc)
     1547        #sanity checks:
     1548        if scaling is None:
     1549            scaling = 1, 1
     1550        v, u = scaling
    10961551        if v/u>1.0:
    10971552            #never upscale before encoding!
    1098             actual_scaling = 1, 1
    1099         elif float(v)/float(u)<0.1:
     1553            scaling = 1, 1
     1554        elif v/u<0.1:
    11001555            #don't downscale more than 10 times! (for each dimension - that's 100 times!)
    1101             actual_scaling = 1, 10
    1102         scalinglog("calculate_scaling%s=%s (q=%s, s=%s, scaling_control=%s)", (width, height, max_w, max_h), actual_scaling, q, s, self.scaling_control)
    1103         return actual_scaling
     1556            scaling = 1, 10
     1557        scalinglog("calculate_scaling%s=%s (q=%s, s=%s, scaling_control=%s)",
     1558                   (width, height, max_w, max_h), scaling, q, s, self.scaling_control)
     1559        return scaling
    11041560
    11051561
     
    11111567            Runs in the 'encode' thread.
    11121568        """
    1113         if self.do_check_pipeline(encoding, width, height, src_format):
     1569        if encoding in ("auto", "grayscale"):
     1570            encodings = self.common_video_encodings
     1571        else:
     1572            encodings = [encoding]
     1573        if self.do_check_pipeline(encodings, width, height, src_format):
    11141574            return True  #OK!
    11151575
    1116         videolog("check_pipeline%s setting up a new pipeline as check failed", (encoding, width, height, src_format))
     1576        videolog("check_pipeline%s setting up a new pipeline as check failed - encodings=%s",
     1577                 (encoding, width, height, src_format), encodings)
    11171578        #cleanup existing one if needed:
    1118         csce = self._csc_encoder
    1119         if csce:
    1120             self._csc_encoder = None
    1121             csce.clean()
    1122         ve = self._video_encoder
    1123         if ve:
    1124             self._video_encoder = None
    1125             ve.clean()
     1579        self.csc_clean(self._csc_encoder)
     1580        self.ve_clean(self._video_encoder)
    11261581        #and make a new one:
    1127         scores = self.get_video_pipeline_options(encoding, width, height, src_format)
     1582        w = width & self.width_mask
     1583        h = height & self.height_mask
     1584        scores = self.get_video_pipeline_options(encodings, w, h, src_format)
    11281585        return self.setup_pipeline(scores, width, height, src_format)
    11291586
    1130     def do_check_pipeline(self, encoding, width, height, src_format):
     1587    def do_check_pipeline(self, encodings, width, height, src_format):
    11311588        """
    11321589            Checks that the current pipeline is still valid
     
    11401597        csce = self._csc_encoder
    11411598        if ve is None:
     1599            videolog("do_check_pipeline: no current video encoder")
     1600            return False
     1601        if ve.is_closed():
     1602            videolog("do_check_pipeline: current video encoder %s is closed", ve)
     1603            return False
     1604        if csce and csce.is_closed():
     1605            videolog("do_check_pipeline: csc %s is closed", csce)
    11421606            return False
    11431607
     
    11491613                                    csce.get_src_format(), src_format)
    11501614                return False
    1151             elif csce.get_src_width()!=csc_width or csce.get_src_height()!=csc_height:
     1615            if csce.get_src_width()!=csc_width or csce.get_src_height()!=csc_height:
    11521616                csclog("do_check_pipeline csc: window dimensions have changed from %sx%s to %sx%s, csc info=%s",
    11531617                                    csce.get_src_width(), csce.get_src_height(), csc_width, csc_height, csce.get_info())
    11541618                return False
    1155             elif csce.get_dst_format()!=ve.get_src_format():
    1156                 csclog.error("Error: CSC intermediate format mismatch: %s vs %s, csc info=%s",
    1157                                     csce.get_dst_format(), ve.get_src_format(), csce.get_info())
     1619            if csce.get_dst_format()!=ve.get_src_format():
     1620                csclog.error("Error: CSC intermediate format mismatch,")
     1621                csclog.error(" %s outputs %s but %s expects %sw",
     1622                             csce.get_type(), csce.get_dst_format(), ve.get_type(), ve.get_src_format())
     1623                csclog.error(" %s:", csce)
     1624                print_nested_dict(csce.get_info(), "  ", print_fn=csclog.error)
     1625                csclog.error(" %s:", ve)
     1626                print_nested_dict(ve.get_info(), "  ", print_fn=csclog.error)
    11581627                return False
    11591628
     
    11711640                return False
    11721641
    1173         if ve.get_encoding()!=encoding:
    1174             videolog("do_check_pipeline video: invalid encoding %s, expected %s",
    1175                                             ve.get_encoding(), encoding)
     1642        if ve.get_encoding() not in encodings:
     1643            videolog("do_check_pipeline video: invalid encoding %s, expected one of: %s",
     1644                                            ve.get_encoding(), csv(encodings))
    11761645            return False
    1177         elif ve.get_width()!=encoder_src_width or ve.get_height()!=encoder_src_height:
     1646        if ve.get_width()!=encoder_src_width or ve.get_height()!=encoder_src_height:
    11781647            videolog("do_check_pipeline video: window dimensions have changed from %sx%s to %sx%s",
    11791648                                            ve.get_width(), ve.get_height(), encoder_src_width, encoder_src_height)
     
    11931662        """
    11941663        assert width>0 and height>0, "invalid dimensions: %sx%s" % (width, height)
    1195         start = time.time()
    1196         if len(scores)==0:
    1197             log.error("Error: no video pipeline options found for %s at %ix%i", src_format, width, height)
     1664        start = monotonic_time()
     1665        if not scores:
     1666            if not self.is_cancelled():
     1667                videolog.error("Error: no video pipeline options found for %s %i-bit at %ix%i",
     1668                               src_format, self.image_depth, width, height)
    11981669            return False
    11991670        videolog("setup_pipeline%s", (scores, width, height, src_format))
     
    12041675                    #success!
    12051676                    return True
    1206                 else:
    1207                     #skip cleanup below
    1208                     continue
     1677                #skip cleanup below
     1678                continue
    12091679            except TransientCodecException as e:
    1210                 videolog.warn("setup_pipeline failed for %s: %s", option, e)
    1211             except:
    1212                 videolog.warn("setup_pipeline failed for %s", option, exc_info=True)
     1680                if self.is_cancelled():
     1681                    return False
     1682                videolog.warn("Warning: setup_pipeline failed for")
     1683                videolog.warn(" %s:", option)
     1684                videolog.warn(" %s", e)
     1685                del e
     1686            except Exception:
     1687                if self.is_cancelled():
     1688                    return False
     1689                videolog.warn("Warning: failed to setup video pipeline %s", option, exc_info=True)
    12131690            #we're here because an exception occurred, cleanup before trying again:
    1214             csce = self._csc_encoder
    1215             if csce:
    1216                 self._csc_encoder = None
    1217                 csce.clean()
    1218             ve = self._video_encoder
    1219             if ve:
    1220                 self._video_encoder = None
    1221                 ve.clean()
    1222         end = time.time()
    1223         videolog("setup_pipeline(..) failed! took %.2fms", (end-start)*1000.0)
    1224         videolog.error("Error: failed to setup a video pipeline for %s at %ix%i", src_format, width, height)
    1225         videolog.error(" tried the following option%s", engs(scores))
    1226         for option in scores:
    1227             videolog.error(" %s", option)
     1691            self.csc_clean(self._csc_encoder)
     1692            self.ve_clean(self._video_encoder)
     1693        end = monotonic_time()
     1694        if not self.is_cancelled():
     1695            videolog("setup_pipeline(..) failed! took %.2fms", (end-start)*1000.0)
     1696            videolog.error("Error: failed to setup a video pipeline for %s at %ix%i", src_format, width, height)
     1697            videolog.error(" tried the following option%s", engs(scores))
     1698            for option in scores:
     1699                videolog.error(" %s", option)
    12281700        return False
    12291701
     
    12481720            #so make sure it never degrades quality
    12491721            csc_speed = min(speed, 100-quality/2.0)
    1250             csc_start = time.time()
     1722            csc_start = monotonic_time()
    12511723            csce = csc_spec.make_instance()
    12521724            csce.init_context(csc_width, csc_height, src_format,
    12531725                                   enc_width, enc_height, enc_in_format, csc_speed)
    1254             csc_end = time.time()
     1726            csc_end = monotonic_time()
    12551727            csclog("setup_pipeline: csc=%s, info=%s, setup took %.2fms",
    12561728                  csce, csce.get_info(), (csc_end-csc_start)*1000.0)
     
    12691741                return False
    12701742        self._csc_encoder = csce
    1271         enc_start = time.time()
     1743        enc_start = monotonic_time()
    12721744        #FIXME: filter dst_formats to only contain formats the encoder knows about?
    1273         dst_formats = self.full_csc_modes.get(encoder_spec.encoding)
     1745        dst_formats = tuple(bytestostr(x) for x in self.full_csc_modes.strtupleget(encoder_spec.encoding))
    12741746        ve = encoder_spec.make_instance()
    1275         options = self.encoding_options.copy()
    1276         vr = self.video_subregion
    1277         if vr and vr.enabled:
    1278             options["source"] = "video"
    1279             if encoder_spec.encoding in self.supports_video_b_frames:
    1280                 #could take av-sync into account here to choose the number of b-frames:
    1281                 options["b-frames"] = 1
    1282         ve.init_context(enc_width, enc_height, enc_in_format, dst_formats, encoder_spec.encoding, quality, speed, encoder_scaling, options)
     1747        options = typedict(self.encoding_options)
     1748        options.update(self.get_video_encoder_options(encoder_spec.encoding, width, height))
     1749        ve.init_context(enc_width, enc_height, enc_in_format,
     1750                        dst_formats, encoder_spec.encoding,
     1751                        quality, speed, encoder_scaling, options)
    12831752        #record new actual limits:
    12841753        self.actual_scaling = scaling
     
    12891758        self.max_w = max_w
    12901759        self.max_h = max_h
    1291         enc_end = time.time()
     1760        enc_end = monotonic_time()
    12921761        self.start_video_frame = 0
    12931762        self._video_encoder = ve
     
    12951764                csce, ve, ve.get_info(), (enc_end-enc_start)*1000.0)
    12961765        scalinglog("setup_pipeline: scaling=%s, encoder_scaling=%s", scaling, encoder_scaling)
    1297         return  True
    1298 
    1299 
    1300     def encode_scrolling(self, image, distances, old_csums, csums, options):
    1301         tstart = time.time()
    1302         scrolllog("encode_scrolling(%s, {..}, [], [], %s)", image, options)
    1303         x, y, w, h = image.get_geometry()[:4]
    1304         yscroll_values = []
    1305         max_scroll_regions = 50
    1306         #process distances with the highest score first:
    1307         for hits in reversed(sorted(distances.values())):
    1308             for scroll in (d for d,v in distances.items() if v==hits):
    1309                 assert scroll<h
    1310                 yscroll_values.append(scroll)
    1311             if len(yscroll_values)>=max_scroll_regions:
    1312                 break
    1313         assert yscroll_values
    1314         #always add zero=no-change so we can drop those updates!
    1315         if 0 not in yscroll_values and 0 in distances:
    1316             #(but do this last so we don't end up cutting too many rectangles)
    1317             yscroll_values.append(0)
    1318         scrolllog(" will send scroll packets for yscroll=%s", yscroll_values)
    1319         #keep track of the lines we have handled already:
    1320         #(the same line may be available from multiple scroll directions)
    1321         handled = set()
     1766        return True
     1767
     1768    def get_video_encoder_options(self, encoding, width, height):
     1769        #tweaks for "real" video:
     1770        opts = {}
     1771        if not self._fixed_quality and not self._fixed_speed and self._fixed_min_quality<50:
     1772            #only allow bandwidth to drive video encoders
     1773            #when we don't have strict quality or speed requirements:
     1774            opts["bandwidth-limit"] = self.bandwidth_limit
     1775        if self.content_type:
     1776            content_type = self.content_type
     1777        elif self.matches_video_subregion(width, height) and self.subregion_is_video() and (monotonic_time()-self.last_scroll_time)>5:
     1778            content_type = "video"
     1779        else:
     1780            content_type = None
     1781        if content_type:
     1782            opts["content-type"] = content_type
     1783            if content_type=="video":
     1784                if B_FRAMES and (encoding in self.supports_video_b_frames):
     1785                    opts["b-frames"] = True
     1786        return opts
     1787
     1788
     1789    def get_fail_cb(self, packet):
     1790        coding = packet[6]
     1791        if coding in self.common_video_encodings:
     1792            return None
     1793        return super().get_fail_cb(packet)
     1794
     1795
     1796    def make_draw_packet(self, x, y, w, h, coding, data, outstride, client_options, options):
     1797        #overriden so we can invalidate the scroll data:
     1798        #log.error("make_draw_packet%s", (x, y, w, h, coding, "..", outstride, client_options)
     1799        packet = super().make_draw_packet(x, y, w, h, coding, data, outstride, client_options, options)
     1800        sd = self.scroll_data
     1801        if sd and not options.get("scroll"):
     1802            if client_options.get("scaled_size") or client_options.get("quality", 100)<20:
     1803                #don't scroll very low quality content, better to refresh it
     1804                scrolllog("low quality %s update, invalidating all scroll data (scaled_size=%s, quality=%s)",
     1805                          coding, client_options.get("scaled_size"), client_options.get("quality", 100))
     1806                self.do_free_scroll_data()
     1807            else:
     1808                sd.invalidate(x, y, w, h)
     1809        return packet
     1810
     1811
     1812    def free_scroll_data(self):
     1813        self.call_in_encode_thread(False, self.do_free_scroll_data)
     1814
     1815    def do_free_scroll_data(self):
     1816        scrolllog("do_free_scroll_data()")
     1817        sd = self.scroll_data
     1818        if sd:
     1819            self.scroll_data = None
     1820            sd.free()
     1821
     1822    def may_use_scrolling(self, image, options):
     1823        scrolllog("may_use_scrolling(%s, %s) supports_scrolling=%s, has_pixels=%s, content_type=%s, non-video encodings=%s",
     1824                  image, options, self.supports_scrolling, image.has_pixels, self.content_type, self.non_video_encodings)
     1825        if not self.supports_scrolling:
     1826            scrolllog("no scrolling: not supported")
     1827            return False
     1828        #don't download the pixels if we have a GPU buffer,
     1829        #since that means we're likely to be able to compress on the GPU too with NVENC:
     1830        if not image.has_pixels():
     1831            return False
     1832        if self.content_type=="video" or not self.non_video_encodings:
     1833            scrolllog("no scrolling: content is video")
     1834            return False
     1835        w = image.get_width()
     1836        h = image.get_height()
     1837        if w<MIN_SCROLL_IMAGE_SIZE or h<MIN_SCROLL_IMAGE_SIZE:
     1838            scrolllog("no scrolling: image size %ix%i is too small, minimum is %ix%i",
     1839                      w, h, MIN_SCROLL_IMAGE_SIZE, MIN_SCROLL_IMAGE_SIZE)
     1840            return False
     1841        scroll_data = self.scroll_data
     1842        if self.b_frame_flush_timer and scroll_data:
     1843            scrolllog("no scrolling: b_frame_flush_timer=%s", self.b_frame_flush_timer)
     1844            self.do_free_scroll_data()
     1845            return False
     1846        return self.do_scroll_encode("scroll", image, options, self.scroll_min_percent)
     1847
     1848    def scroll_encode(self, coding, image, options):
     1849        self.do_scroll_encode(coding, image, options, 0)
     1850        #do_scroll_encode() sends the packets
     1851        #so there is nothing to return:
     1852        return None
     1853
     1854    def do_scroll_encode(self, coding, image, options, min_percent=0):
     1855        x = image.get_target_x()
     1856        y = image.get_target_y()
     1857        w = image.get_width()
     1858        h = image.get_height()
     1859        scroll_data = self.scroll_data
     1860        if options.get("scroll") is True:
     1861            scrolllog("no scrolling: detection has already been used on this image")
     1862            return False
     1863        if w>=32000 or h>=32000:
     1864            scrolllog("no scrolling: the image is too large, %ix%i", w, h)
     1865            return False
     1866        try:
     1867            start = monotonic_time()
     1868            if not scroll_data:
     1869                scroll_data = ScrollData()
     1870                self.scroll_data = scroll_data
     1871                scrolllog("new scroll data: %s", scroll_data)
     1872            if not image.is_thread_safe():
     1873                #what we really want is to check that the frame has been frozen,
     1874                #so it doesn't get modified whilst we checksum or encode it,
     1875                #the "thread_safe" flag gives us that for the X11 case in most cases,
     1876                #(the other servers already copy the pixels from the "real" screen buffer)
     1877                #TODO: use a separate flag? (ximage uses this flag to know if it is safe
     1878                # to call image.free from another thread - which is theoretically more restrictive)
     1879                newstride = roundup(image.get_width()*image.get_bytesperpixel(), 4)
     1880                image.restride(newstride)
     1881                stride = image.get_rowstride()
     1882            bpp = image.get_bytesperpixel()
     1883            pixels = image.get_pixels()
     1884            if not pixels:
     1885                return False
     1886            stride = image.get_rowstride()
     1887            scroll_data.update(pixels, x, y, w, h, stride, bpp)
     1888            max_distance = min(1000, (100-min_percent)*h//100)
     1889            scroll_data.calculate(max_distance)
     1890            #marker telling us not to invalidate the scroll data from here on:
     1891            options["scroll"] = True
     1892            if min_percent>0:
     1893                max_zones = 20
     1894                scroll, count = scroll_data.get_best_match()
     1895                end = monotonic_time()
     1896                match_pct = int(100*count/h)
     1897                scrolllog("best scroll guess took %ims, matches %i%% of %i lines: %s",
     1898                          (end-start)*1000, match_pct, h, scroll)
     1899            else:
     1900                max_zones = 50
     1901                match_pct = min_percent
     1902            #if enough scrolling is detected, use scroll encoding for this frame:
     1903            if match_pct>=min_percent:
     1904                self.encode_scrolling(scroll_data, image, options, match_pct, max_zones)
     1905                return True
     1906        except Exception:
     1907            scrolllog("do_scroll_encode(%s, %s)", image, options, exc_info=True)
     1908            if not self.is_cancelled():
     1909                scrolllog.error("Error during scrolling detection")
     1910                scrolllog.error(" with image=%s, options=%s", image, options, exc_info=True)
     1911            #make sure we start again from scratch next time:
     1912            self.do_free_scroll_data()
     1913        return False
     1914
     1915    def encode_scrolling(self, scroll_data, image, options, match_pct, max_zones=20):
     1916        #generate all the packets for this screen update
     1917        #using 'scroll' encoding and picture encodings for the other regions
     1918        start = monotonic_time()
     1919        options.pop("av-sync", None)
     1920        #tells make_data_packet not to invalidate the scroll data:
     1921        ww, wh = self.window_dimensions
     1922        scrolllog("encode_scrolling([], %s, %s, %i, %i) window-dimensions=%s",
     1923                  image, options, match_pct, max_zones, (ww, wh))
     1924        x = image.get_target_x()
     1925        y = image.get_target_y()
     1926        w = image.get_width()
     1927        h = image.get_height()
     1928        raw_scroll, non_scroll = {}, {0 : h}
     1929        if x+y>ww or y+h>wh:
     1930            #window may have been resized
     1931            pass
     1932        else:
     1933            v = scroll_data.get_scroll_values()
     1934            if v:
     1935                raw_scroll, non_scroll = v
     1936                if len(raw_scroll)>=max_zones or len(non_scroll)>=max_zones:
     1937                    #avoid fragmentation, which is too costly
     1938                    #(too many packets, too many loops through the encoder code)
     1939                    scrolllog("too many items: %i scrolls, %i non-scrolls - sending just one image instead",
     1940                              len(raw_scroll), len(non_scroll))
     1941                    raw_scroll = {}
     1942                    non_scroll = {0 : h}
     1943        scrolllog(" will send scroll data=%s, non-scroll=%s", raw_scroll, non_scroll)
     1944        flush = len(non_scroll)
     1945        #convert to a screen rectangle list for the client:
    13221946        scrolls = []
    1323         max_scrolls = 1000
    1324         for s in yscroll_values:
    1325             #find all the lines that scroll by this much:
    1326             slines = match_distance(old_csums, csums, s)
    1327             assert slines, "no lines matching distance %i" % s
    1328             #remove any lines we have already handled:
    1329             lines = [v+s for v in slines if ((v+s) not in handled and v not in handled)]
    1330             if not lines:
     1947        for scroll, line_defs in raw_scroll.items():
     1948            if scroll==0:
    13311949                continue
    1332             #and them to the handled set so we don't try to paint them again:
    1333             handled = handled.union(set(lines))
    1334             if s==0:
    1335                 scrolllog(" %i lines have not changed", len(lines))
    1336             else:
    1337                 #things have actually moved
    1338                 #aggregate consecutive lines into rectangles:
    1339                 cl = consecutive_lines(lines)
    1340                 scrolllog(" scroll groups for distance=%i : %s=%s", s, lines, cl)
    1341                 for start,count in cl:
    1342                     #new rectangle
    1343                     scrolls.append((x, y+start-s, w, count, 0, s))
    1344                     if len(scrolls)>max_scrolls:
    1345                         break
    1346                 if len(scrolls)>max_scrolls:
    1347                     break
    1348         non_scroll = []
    1349         remaining = set(range(h))-handled
    1350         if remaining:
    1351             damaged_lines = sorted(list(remaining))
    1352             non_scroll = consecutive_lines(damaged_lines)
    1353             scrolllog(" non scroll: %i packets: %s", len(non_scroll), non_scroll)
    1354         flush = len(non_scroll)
    1355         #send as scroll paints packets:
     1950            for line, count in line_defs.items():
     1951                assert y+line+scroll>=0, "cannot scroll rectangle by %i lines from %i+%i" % (scroll, y, line)
     1952                assert y+line+scroll<=wh, "cannot scroll rectangle %i high by %i lines from %i+%i (window height is %i)" % (count, scroll, y, line, wh)
     1953                scrolls.append((x, y+line, w, count, 0, scroll))
     1954        del raw_scroll
     1955        #send the scrolls if we have any
     1956        #(zero change scrolls have been removed - so maybe there are none)
    13561957        if scrolls:
    13571958            client_options = options.copy()
    1358             if flush>0 and self.supports_flush:
     1959            client_options.pop("scroll", None)
     1960            if flush>0:
    13591961                client_options["flush"] = flush
    1360             packet = self.make_draw_packet(x, y, w, h, "scroll", LargeStructure("scroll data", scrolls), 0, client_options)
    1361             self.queue_damage_packet(packet)
     1962            coding = "scroll"
     1963            end = monotonic_time()
     1964            packet = self.make_draw_packet(x, y, w, h,
     1965                                           coding, LargeStructure(coding, scrolls), 0, client_options, options)
     1966            self.queue_damage_packet(packet, 0, 0, options)
     1967            compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %9s as %3i rectangles  (%5iKB)           , sequence %5i, client_options=%s",
     1968                 (end-start)*1000.0, w, h, x, y, self.wid, coding, len(scrolls), w*h*4/1024, self._damage_packet_sequence, client_options)
     1969        del scrolls
    13621970        #send the rest as rectangles:
    13631971        if non_scroll:
    1364             for start, count in non_scroll:
    1365                 sub = image.get_sub_image(0, start, w, count)
    1366                 flush -= 1
    1367                 ret = self.video_fallback(sub, options)
     1972            speed, quality = self._current_speed, self._current_quality
     1973            #boost quality a bit, because lossless saves refreshing,
     1974            #more so if we have a high match percentage (less to send):
     1975            quality = min(100, quality + 10 + max(0, match_pct-50)//2)
     1976            nsstart = monotonic_time()
     1977            client_options = options.copy()
     1978            for sy, sh in non_scroll.items():
     1979                substart = monotonic_time()
     1980                sub = image.get_sub_image(0, sy, w, sh)
     1981                encoding = self.get_best_nonvideo_encoding(w, sh, speed, quality)
     1982                assert encoding, "no nonvideo encoding found for %ix%i screen update" % (w, sh)
     1983                encode_fn = self._encoders[encoding]
     1984                ret = encode_fn(encoding, sub, options)
     1985                self.free_image_wrapper(sub)
    13681986                if not ret:
    13691987                    #cancelled?
     
    13711989                coding, data, client_options, outw, outh, outstride, _ = ret
    13721990                assert data
    1373                 client_options = options.copy()
    1374                 if flush>0 and self.supports_flush:
     1991                flush -= 1
     1992                if flush>0:
    13751993                    client_options["flush"] = flush
    1376                 packet = self.make_draw_packet(sub.get_x(), sub.get_y(), outw, outh, coding, data, outstride, client_options)
    1377                 self.queue_damage_packet(packet)
     1994                #if SAVE_TO_FILE:
     1995                #    #hard-coded for BGRA!
     1996                #    from xpra.os_util import memoryview_to_bytes
     1997                #    from PIL import Image
     1998                #    im = Image.frombuffer("RGBA", (w, sh), memoryview_to_bytes(sub.get_pixels()), "raw", "BGRA", sub.get_rowstride(), 1)
     1999                #    filename = "./scroll-%i-%i.png" % (self._sequence, len(non_scroll)-flush)
     2000                #    im.save(filename, "png")
     2001                #    log.info("saved scroll y=%i h=%i to %s", sy, sh, filename)
     2002                packet = self.make_draw_packet(sub.get_target_x(), sub.get_target_y(), outw, outh,
     2003                                               coding, data, outstride, client_options, options)
     2004                self.queue_damage_packet(packet, 0, 0, options)
     2005                psize = w*sh*4
     2006                csize = len(data)
     2007                compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %9s with ratio %5.1f%%  (%5iKB to %5iKB), sequence %5i, client_options=%s",
     2008                     (monotonic_time()-substart)*1000.0, w, sh, x+0, y+sy, self.wid, coding, 100.0*csize/psize, psize/1024, csize/1024, self._damage_packet_sequence, client_options)
     2009            scrolllog("non-scroll encoding using %s (quality=%i, speed=%i) took %ims for %i rectangles",
     2010                      encoding, self._current_quality, self._current_speed, (monotonic_time()-nsstart)*1000, len(non_scroll))
     2011        else:
     2012            #we can't send the non-scroll areas, ouch!
     2013            flush = 0
    13782014        assert flush==0
    1379         tend = time.time()
    1380         scrolllog("scroll encoding took %ims", (tend-tstart)*1000)
    1381         return None
    1382 
    1383     def video_fallback(self, image, options, order=PREFERED_ENCODING_ORDER):
    1384         #find one that is not video:
    1385         fallback_encodings = [x for x in order if (x in self.non_video_encodings and x in self._encoders and x!="mmap")]
     2015        self.last_scroll_time = monotonic_time()
     2016        scrolllog("scroll encoding total time: %ims", (self.last_scroll_time-start)*1000)
     2017        self.free_image_wrapper(image)
     2018
     2019
     2020    def do_schedule_auto_refresh(self, encoding, data, region, client_options, options):
     2021        #for scroll encoding, data is a LargeStructure wrapper:
     2022        if encoding=="scroll" and hasattr(data, "data"):
     2023            if not self.refresh_regions:
     2024                return
     2025            #check if any pending refreshes intersect the area containing the scroll data:
     2026            if not any(region.intersects_rect(r) for r in self.refresh_regions):
     2027                #nothing to do!
     2028                return
     2029            pixels_added = 0
     2030            for x, y, w, h, dx, dy in data.data:
     2031                #the region that moved
     2032                src_rect = rectangle(x, y, w, h)
     2033                for rect in self.refresh_regions:
     2034                    inter = src_rect.intersection_rect(rect)
     2035                    if inter:
     2036                        dst_rect = rectangle(inter.x+dx, inter.y+dy, inter.width, inter.height)
     2037                        pixels_added += self.add_refresh_region(dst_rect)
     2038            if pixels_added:
     2039                #if we end up with too many rectangles,
     2040                #bail out and simplify:
     2041                if len(self.refresh_regions)>=200:
     2042                    self.refresh_regions = [merge_all(self.refresh_regions)]
     2043                refreshlog("updated refresh regions with scroll data: %i pixels added", pixels_added)
     2044                refreshlog(" refresh_regions=%s", self.refresh_regions)
     2045            #we don't change any of the refresh scheduling
     2046            #if there are non-scroll packets following this one, they will
     2047            #and if not then we're OK anyway
     2048            return
     2049        super().do_schedule_auto_refresh(encoding, data, region, client_options, options)
     2050
     2051
     2052    def get_fallback_encoding(self, encodings, order):
     2053        if order is None:
     2054            if self._current_speed>=50:
     2055                order = FAST_ORDER
     2056            else:
     2057                order = PREFERRED_ENCODING_ORDER
     2058        #don't choose mmap!
     2059        fallback_encodings = tuple(x for x in order if
     2060                                   (x in encodings and x in self._encoders and x!="mmap"))
     2061        depth = self.image_depth
     2062        if depth==8 and "png/P" in fallback_encodings:
     2063            return "png/P"
     2064        if depth==30 and "rgb32" in fallback_encodings:
     2065            return "rgb32"
     2066        if depth not in (24, 32):
     2067            #jpeg cannot handle other bit depths
     2068            fallback_encodings = tuple(x for x in fallback_encodings if x!="jpeg")
    13862069        if not fallback_encodings:
    1387             log.error("no non-video fallback encodings are available!")
     2070            if not self.is_cancelled():
     2071                log.warn("Warning: no non-video fallback encodings are available!")
    13882072            return None
    1389         fallback_encoding = fallback_encodings[0]
    1390         encode_fn = self._encoders[fallback_encoding]
    1391         return encode_fn(fallback_encoding, image, options)
    1392 
    1393     def video_encode(self, encoding, image, options):
     2073        return fallback_encodings[0]
     2074
     2075    def get_video_fallback_encoding(self, order=FAST_ORDER):
     2076        return self.get_fallback_encoding(self.non_video_encodings, order)
     2077
     2078    def video_fallback(self, image, options, order=None, warn=False):
     2079        if warn:
     2080            videolog.warn("using non-video fallback encoding")
     2081        if self.image_depth==8:
     2082            if self.encoding=="grayscale":
     2083                encoding = "png/L"
     2084            else:
     2085                encoding = "png/P"
     2086        else:
     2087            encoding = self.get_video_fallback_encoding(order)
     2088            if not encoding:
     2089                return None
     2090        encode_fn = self._encoders[encoding]
     2091        #switching to non-video encoding can use a lot more bandwidth,
     2092        #try to avoid this by lowering the quality:
     2093        options["quality"] = max(5, self._current_quality-50)
     2094        return encode_fn(encoding, image, options)
     2095
     2096    def video_encode(self, encoding, image, options : dict):
    13942097        try:
    13952098            return self.do_video_encode(encoding, image, options)
     
    13972100            self.free_image_wrapper(image)
    13982101
    1399     def do_video_encode(self, encoding, image, options):
     2102    def do_video_encode(self, encoding, image, options : dict):
    14002103        """
    14012104            This method is used by make_data_packet to encode frames using video encoders.
     
    14062109            Runs in the 'encode' thread.
    14072110        """
     2111        log("do_video_encode(%s, %s, %s)", encoding, image, options)
    14082112        x, y, w, h = image.get_geometry()[:4]
    1409         assert self.supports_video_subregion or (x==0 and y==0), "invalid position: %s,%s" % (x,y)
    14102113        src_format = image.get_pixel_format()
     2114        stride = image.get_rowstride()
    14112115        if self.pixel_format!=src_format:
    1412             videolog.warn("image pixel format changed from %s to %s", self.pixel_format, src_format)
     2116            if self.is_cancelled():
     2117                return None
     2118            videolog.warn("Warning: image pixel format unexpectedly changed from %s to %s",
     2119                          self.pixel_format, src_format)
    14132120            self.pixel_format = src_format
    14142121
    1415         def video_fallback():
    1416             videolog.warn("using non-video fallback encoding")
     2122        if SAVE_VIDEO_FRAMES:
     2123            from xpra.os_util import memoryview_to_bytes
     2124            from PIL import Image
     2125            img_data = image.get_pixels()
     2126            rgb_format = image.get_pixel_format() #ie: BGRA
     2127            rgba_format = rgb_format.replace("BGRX", "BGRA")
     2128            img = Image.frombuffer("RGBA", (w, h), memoryview_to_bytes(img_data), "raw", rgba_format, stride)
     2129            kwargs = {}
     2130            if SAVE_VIDEO_FRAMES=="jpeg":
     2131                kwargs = {
     2132                          "quality"     : 0,
     2133                          "optimize"    : False,
     2134                          }
     2135            t = monotonic_time()
     2136            tstr = time.strftime("%H-%M-%S", time.localtime(t))
     2137            filename = "W%i-VDO-%s.%03i.%s" % (self.wid, tstr, (t*1000)%1000, SAVE_VIDEO_FRAMES)
     2138            if SAVE_VIDEO_PATH:
     2139                filename = os.path.join(SAVE_VIDEO_PATH, filename)
     2140            videolog("do_video_encode: saving %4ix%-4i pixels, %7i bytes to %s", w, h, (stride*h), filename)
     2141            img.save(filename, SAVE_VIDEO_FRAMES, **kwargs)
     2142
     2143        if self.may_use_scrolling(image, options):
     2144            #scroll encoding has dealt with this image
     2145            return None
     2146
     2147        if not self.common_video_encodings:
     2148            #we have to send using a non-video encoding as that's all we have!
    14172149            return self.video_fallback(image, options)
     2150        if self.image_depth not in (24, 30, 32):
     2151            #this image depth is not supported for video
     2152            return self.video_fallback(image, options)
     2153
     2154        if self.encoding=="grayscale":
     2155            from xpra.codecs.csc_libyuv.colorspace_converter import argb_to_gray    #@UnresolvedImport
     2156            image = argb_to_gray(image)
    14182157
    14192158        vh = self.video_helper
     
    14212160            return None         #shortcut when closing down
    14222161        if not self.check_pipeline(encoding, w, h, src_format):
     2162            if self.is_cancelled():
     2163                return None
    14232164            #just for diagnostics:
    1424             supported_csc_modes = self.full_csc_modes.get(encoding, [])
     2165            supported_csc_modes = self.full_csc_modes.strtupleget(encoding)
    14252166            encoder_specs = vh.get_encoder_specs(encoding)
    14262167            encoder_types = []
     
    14342175                    if especs.codec_type not in encoder_types:
    14352176                        encoder_types.append(especs.codec_type)
    1436             videolog.error("Error: failed to setup a video pipeline for %s encoding with source format %s", encoding, src_format)
    1437             videolog.error(" all encoders: %s", ", ".join(list(set([es.codec_type for sublist in encoder_specs.values() for es in sublist]))))
    1438             videolog.error(" supported CSC modes: %s", ", ".join(supported_csc_modes))
    1439             videolog.error(" supported encoders: %s", ", ".join(encoder_types))
    1440             videolog.error(" encoders CSC modes: %s", ", ".join(ecsc))
     2177            videolog.error("Error: failed to setup a video pipeline for %s encoding with source format %s",
     2178                           encoding, src_format)
     2179            all_encs = set(es.codec_type for sublist in encoder_specs.values() for es in sublist)
     2180            videolog.error(" all encoders: %s", csv(tuple(all_encs)))
     2181            videolog.error(" supported CSC modes: %s", csv(supported_csc_modes))
     2182            videolog.error(" supported encoders: %s", csv(encoder_types))
     2183            videolog.error(" encoders CSC modes: %s", csv(ecsc))
    14412184            if FORCE_CSC:
    1442                 log.error(" forced csc mode: %s", FORCE_CSC_MODE)
    1443             return video_fallback()
     2185                videolog.error(" forced csc mode: %s", FORCE_CSC_MODE)
     2186            return self.video_fallback(image, options, warn=True)
    14442187        ve = self._video_encoder
    14452188        if not ve:
    1446             return video_fallback()
    1447 
    1448         #check for scrolling:
    1449         if self.supports_scrolling:
    1450             try:
    1451                 start = time.time()
    1452                 lsd = self.scroll_data
    1453                 pixels = image.get_pixels()
    1454                 stride = image.get_rowstride()
    1455                 width = image.get_width()
    1456                 height = image.get_height()
    1457                 csums = CRC_Image(pixels, width, height, stride)
    1458                 self.scroll_data = (width, height, csums)
    1459                 if lsd:
    1460                     lw, lh, lcsums = lsd
    1461                     if lw==width and lh==height:
    1462                         #same size, try to find scrolling value
    1463                         assert len(csums)==len(lcsums)
    1464                         distances = calculate_distances(csums, lcsums, 2, 500)
    1465                         if len(distances)>0:
    1466                             best = max(distances.values())
    1467                             scroll = distances.keys()[distances.values().index(best)]
    1468                             end = time.time()
    1469                             best_pct = int(100*best/height)
    1470                             scrolllog("best scroll guess took %ims, matches %i%% of %i lines: %s", (end-start)*1000, best_pct, height, scroll)
    1471                             #at least 40% of the picture was found as scroll areas:
    1472                             if best_pct>=SCROLL_MIN_PERCENT:
    1473                                 return self.encode_scrolling(image, distances, lcsums, csums, options)
    1474             except Exception:
    1475                 scrolllog.error("Error during scrolling detection!", exc_info=True)
     2189            return self.video_fallback(image, options, warn=True)
     2190        if not ve.is_ready():
     2191            log("video encoder %s is not ready yet, using temporary fallback", ve)
     2192            return self.video_fallback(image, options, order=FAST_ORDER, warn=False)
     2193
     2194        #we're going to use the video encoder,
     2195        #so make sure we don't time it out:
     2196        self.cancel_video_encoder_timer()
    14762197
    14772198        #dw and dh are the edges we don't handle here
    14782199        width = w & self.width_mask
    14792200        height = h & self.height_mask
    1480         videolog("video_encode%s image size: %sx%s, encoder/csc size: %sx%s", (encoding, image, options), w, h, width, height)
    1481 
    1482         csc_image, csc, enc_width, enc_height = self.csc_image(image, width, height)
    1483 
    1484         start = time.time()
     2201        videolog("video_encode%s image size: %4ix%-4i, encoder/csc size: %4ix%-4i",
     2202                 (encoding, image, options), w, h, width, height)
     2203
     2204        csce, csc_image, csc, enc_width, enc_height = self.csc_image(image, width, height)
     2205
     2206        start = monotonic_time()
    14852207        quality = max(0, min(100, self._current_quality))
    14862208        speed = max(0, min(100, self._current_speed))
    1487         ret = ve.compress_image(csc_image, quality, speed, options)
     2209        options.update(self.get_video_encoder_options(ve.get_encoding(), width, height))
     2210        try:
     2211            ret = ve.compress_image(csc_image, quality, speed, options)
     2212        except Exception as e:
     2213            videolog("%s.compress_image%s", ve, (csc_image, quality, speed, options), exc_info=True)
     2214            if self.is_cancelled():
     2215                return None
     2216            videolog.error("Error: failed to encode %s video frame using %s:", ve.get_encoding(), ve.get_type())
     2217            videolog.error(" %s", e)
     2218            videolog.error(" source: %s", csc_image)
     2219            videolog.error(" options:")
     2220            print_nested_dict(options, prefix="   ", print_fn=videolog.error)
     2221            videolog.error(" encoder:")
     2222            print_nested_dict(ve.get_info(), prefix="   ", print_fn=videolog.error)
     2223            if csce:
     2224                videolog.error(" csc %s:", csce.get_type())
     2225                print_nested_dict(csce.get_info(), prefix="   ", print_fn=videolog.error)
     2226            return None
     2227        finally:
     2228            if image!=csc_image:
     2229                self.free_image_wrapper(csc_image)
     2230            del csc_image
    14882231        if ret is None:
    1489             videolog.error("video_encode: ouch, %s compression failed", encoding)
     2232            if not self.is_cancelled():
     2233                videolog.error("Error: %s video compression failed", encoding)
    14902234            return None
    14912235        data, client_options = ret
    1492         end = time.time()
    1493 
    1494         if csc_image!=image:
    1495             self.free_image_wrapper(csc_image)
    1496             del csc_image
    1497 
     2236        end = monotonic_time()
     2237
     2238        #populate client options:
     2239        frame = client_options.get("frame", 0)
     2240        if frame<self.start_video_frame:
     2241            #tell client not to bother updating the screen,
     2242            #as it must have received a non-video frame already
     2243            client_options["paint"] = False
     2244
     2245        if frame==0 and SAVE_VIDEO_STREAMS:
     2246            self.close_video_stream_file()
     2247            elapsed = monotonic_time()-self.start_time
     2248            stream_filename = "window-%i-%.1f-%s.%s" % (self.wid, elapsed, ve.get_type(), ve.get_encoding())
     2249            if SAVE_VIDEO_PATH:
     2250                stream_filename = os.path.join(SAVE_VIDEO_PATH, stream_filename)
     2251            self.video_stream_file = open(stream_filename, "wb")
     2252            log.info("saving new %s stream for window %i to %s", ve.get_encoding(), self.wid, stream_filename)
     2253        if self.video_stream_file:
     2254            self.video_stream_file.write(data)
     2255            self.video_stream_file.flush()
     2256
     2257        #tell the client about scaling (the size of the encoded picture):
     2258        #(unless the video encoder has already done so):
     2259        scaled_size = None
     2260        if csce and ("scaled_size" not in client_options) and (enc_width!=width or enc_height!=height):
     2261            scaled_size = enc_width, enc_height
     2262            client_options["scaled_size"] = scaled_size
     2263
     2264        #deal with delayed b-frames:
    14982265        delayed = client_options.get("delayed", 0)
    1499         frame = client_options.get("frame", 0)
    15002266        self.cancel_video_encoder_flush()
    15012267        if delayed>0:
    1502             self.schedule_video_encoder_flush(ve, csc, frame, x, y, client_options)
     2268            self.schedule_video_encoder_flush(ve, csc, frame, x, y, scaled_size)
    15032269            if not data:
    1504                 if frame==0:
     2270                if self.non_video_encodings and frame==0:
    15052271                    #first frame has not been sent yet,
    15062272                    #so send something as non-video
     
    15092275                    return self.video_fallback(image, options, order=FAST_ORDER)
    15102276                return None
    1511         if frame<self.start_video_frame:
    1512             #tell client not to bother updating the screen,
    1513             #as it must have received a non-video frame already
    1514             client_options["paint"] = False
    1515 
    1516         #tell the client which colour subsampling we used:
    1517         #(note: see csc_equiv!)
    1518         client_options["csc"] = self.csc_equiv(csc)
    1519         #tell the client about scaling (the size of the encoded picture):
    1520         #(unless the video encoder has already done so):
    1521         if self._csc_encoder and ("scaled_size" not in client_options) and (enc_width!=width or enc_height!=height):
    1522             client_options["scaled_size"] = enc_width, enc_height
    1523         videolog("video_encode %s encoder: %s %sx%s result is %s bytes (%.1f MPixels/s), client options=%s",
    1524                             ve.get_type(), encoding, enc_width, enc_height, len(data), (enc_width*enc_height/(end-start+0.000001)/1024.0/1024.0), client_options)
    1525         return ve.get_encoding(), Compressed(encoding, data), client_options, width, height, 0, 24
     2277        else:
     2278            #there are no delayed frames,
     2279            #make sure we timeout the encoder if no new frames come through:
     2280            self.schedule_video_encoder_timer()
     2281        actual_encoding = ve.get_encoding()
     2282        videolog("video_encode %s encoder: %4s %4ix%-4i result is %7i bytes, %6.1f MPixels/s, client options=%s",
     2283                            ve.get_type(), actual_encoding, enc_width, enc_height, len(data or ""),
     2284                            (enc_width*enc_height/(end-start+0.000001)/1024.0/1024.0), client_options)
     2285        return actual_encoding, Compressed(actual_encoding, data), client_options, width, height, 0, 24
    15262286
    15272287    def cancel_video_encoder_flush(self):
     2288        self.cancel_video_encoder_flush_timer()
     2289        self.b_frame_flush_data = None
     2290
     2291    def cancel_video_encoder_flush_timer(self):
    15282292        bft = self.b_frame_flush_timer
    15292293        if bft:
     
    15312295            self.source_remove(bft)
    15322296
    1533     def schedule_video_encoder_flush(self, ve, csc, frame, x , y, client_options):
    1534         flush_delay = max(100, min(500, int(self.batch_config.delay*10)))
    1535         videolog("schedule_video_encoder_flush%s flush delay=%i", (ve, csc, frame, x, y, client_options), flush_delay)
    1536         self.b_frame_flush_timer = self.timeout_add(flush_delay, self.flush_video_encoder, ve, csc, frame, x, y)
    1537 
    1538     def flush_video_encoder(self, ve, csc, frame, x, y):
    1539         #this runs in the UI thread..
     2297    def schedule_video_encoder_flush(self, ve, csc, frame, x , y, scaled_size):
     2298        flush_delay = max(150, min(500, int(self.batch_config.delay*10)))
     2299        self.b_frame_flush_data = (ve, csc, frame, x, y, scaled_size)
     2300        self.b_frame_flush_timer = self.timeout_add(flush_delay, self.flush_video_encoder)
     2301
     2302    def flush_video_encoder_now(self):
     2303        #this can be called before the timer is due
     2304        self.cancel_video_encoder_flush_timer()
     2305        self.flush_video_encoder()
     2306
     2307    def flush_video_encoder(self):
     2308        #this runs in the UI thread as scheduled by schedule_video_encoder_flush,
    15402309        #but we want to run from the encode thread to access the encoder:
    15412310        self.b_frame_flush_timer = None
     2311        if self.b_frame_flush_data:
     2312            self.call_in_encode_thread(True, self.do_flush_video_encoder)
     2313
     2314    def do_flush_video_encoder(self):
     2315        flush_data = self.b_frame_flush_data
     2316        videolog("do_flush_video_encoder: %s", flush_data)
     2317        if not flush_data:
     2318            return
     2319        ve, csc, frame, x, y, scaled_size = flush_data
    15422320        if self._video_encoder!=ve or ve.is_closed():
    15432321            return
    1544         self.call_in_encode_thread(True, self.do_flush_video_encoder, ve, csc, frame, x, y)
    1545 
    1546     def do_flush_video_encoder(self, ve, csc, frame, x, y):
    1547         videolog("do_flush_video_encoder%s", (ve, csc, frame, x, y))
    1548         if self._video_encoder!=ve or ve.is_closed():
    1549             return
    1550         if frame==0:
     2322        if frame==0 and ve.get_type()=="x264":
    15512323            #x264 has problems if we try to re-use a context after flushing the first IDR frame
    1552             self._video_encoder = None
    1553             ve.clean()
    1554             self.idle_add(self.refresh, {"novideo" : True})
     2324            self.ve_clean(self._video_encoder)
     2325            if self.non_video_encodings:
     2326                log("do_flush_video_encoder() scheduling novideo refresh")
     2327                self.idle_add(self.refresh, {"novideo" : True})
     2328                videolog("flushed frame 0, novideo refresh requested")
    15552329            return
    15562330        w = ve.get_width()
     
    15622336            self.cleanup_codecs()
    15632337        if not v:
    1564             videolog("do_flush_video_encoder%s=%s", (ve, frame, x, y), v)
     2338            videolog("do_flush_video_encoder: %s flush=%s", flush_data, v)
    15652339            return
    15662340        data, client_options = v
    15672341        if not data:
    1568             videolog("do_flush_video_encoder%s no data", (ve, frame, x, y))
     2342            videolog("do_flush_video_encoder: %s no data: %s", flush_data, v)
    15692343            return
    1570         client_options["csc"] = self.csc_equiv(csc)
     2344        if self.video_stream_file:
     2345            self.video_stream_file.write(data)
     2346            self.video_stream_file.flush()
    15712347        if frame<self.start_video_frame:
    15722348            client_options["paint"] = False
    1573         videolog("do_flush_video_encoder%s=(%s %s bytes, %s)", (ve, frame, x, y), len(data or ()), type(data), client_options)
    1574         packet = self.make_draw_packet(x, y, w, h, encoding, Compressed(encoding, data), 0, client_options)
     2349        if scaled_size:
     2350            client_options["scaled_size"] = scaled_size
     2351        client_options["flush-encoder"] = True
     2352        videolog("do_flush_video_encoder %s : (%s %s bytes, %s)",
     2353                 flush_data, len(data or ()), type(data), client_options)
     2354        #warning: 'options' will be missing the "window-size",
     2355        #so we may end up not honouring gravity during window resizing:
     2356        options = {}
     2357        packet = self.make_draw_packet(x, y, w, h, encoding, Compressed(encoding, data), 0, client_options, options)
    15752358        self.queue_damage_packet(packet)
    15762359        #check for more delayed frames since we want to support multiple b-frames:
    1577         self.flush_video_encoder(ve, csc, frame, x, y)
     2360        if not self.b_frame_flush_timer and client_options.get("delayed", 0)>0:
     2361            self.schedule_video_encoder_flush(ve, csc, frame, x, y, scaled_size)
     2362        else:
     2363            self.schedule_video_encoder_timer()
     2364
     2365
     2366    def cancel_video_encoder_timer(self):
     2367        vet = self.video_encoder_timer
     2368        if vet:
     2369            self.video_encoder_timer = None
     2370            self.source_remove(vet)
     2371
     2372    def schedule_video_encoder_timer(self):
     2373        if not self.video_encoder_timer:
     2374            vs = self.video_subregion
     2375            if vs and vs.detection:
     2376                timeout = VIDEO_TIMEOUT
     2377            else:
     2378                timeout = VIDEO_NODETECT_TIMEOUT
     2379            if timeout>0:
     2380                self.video_encoder_timer = self.timeout_add(timeout*1000, self.video_encoder_timeout)
     2381
     2382    def video_encoder_timeout(self):
     2383        videolog("video_encoder_timeout() will close video encoder=%s", self._video_encoder)
     2384        self.video_encoder_timer = None
     2385        self.video_context_clean()
    15782386
    15792387
     
    15912399        if csce is None:
    15922400            #no csc step!
    1593             return image, image.get_pixel_format(), width, height
    1594 
    1595         start = time.time()
     2401            return None, image, image.get_pixel_format(), width, height
     2402
     2403        start = monotonic_time()
    15962404        csc_image = csce.convert_image(image)
    1597         end = time.time()
    1598         csclog("csc_image(%s, %s, %s) converted to %s in %.1fms (%.1f MPixels/s)",
     2405        end = monotonic_time()
     2406        csclog("csc_image(%s, %s, %s) converted to %s in %.1fms, %6.1f MPixels/s",
    15992407                        image, width, height,
    16002408                        csc_image, (1000.0*end-1000.0*start), (width*height/(end-start+0.000001)/1024.0/1024.0))
     
    16022410            raise Exception("csc_image: conversion of %s to %s failed" % (image, csce.get_dst_format()))
    16032411        assert csce.get_dst_format()==csc_image.get_pixel_format()
    1604         return csc_image, csce.get_dst_format(), csce.get_dst_width(), csce.get_dst_height()
     2412        return csce, csc_image, csce.get_dst_format(), csce.get_dst_width(), csce.get_dst_height()
Note: See TracChangeset for help on using the changeset viewer.