xpra icon
Bug tracker and wiki

This bug tracker and wiki are being discontinued
please use https://github.com/Xpra-org/xpra instead.


Ticket #800: delayed-frames.patch

File delayed-frames.patch, 11.9 KB (added by Antoine Martin, 6 years ago)

work in progress to support b frames

  • xpra/client/ui_client_base.py

     
    10301030            "set_enabled"               : True,
    10311031            })
    10321032        updict(capabilities, "encoding", {
     1033            "b-frames"                  : True,
    10331034            "flush"                     : True,
    10341035            "scaling.control"           : self.scaling,
    10351036            "client_options"            : True,
  • xpra/codecs/enc_x264/encoder.pyx

     
    9393    int X264_TYPE_B
    9494
    9595    const char * const *x264_preset_names
     96    const char * const *x264_tune_names
    9697
    9798    ctypedef struct rc:
    9899        int         i_rc_method
     
    223224cdef set_f_rf(x264_param_t *param, float q):
    224225    param.rc.f_rf_constant = q
    225226
    226 cdef const char * const *get_preset_names():
    227     return x264_preset_names;
    228227
     228cdef const char * n
     229cdef int i = 0
     230PRESET_NAMES = {}
     231PRESET_VALUES = {}
     232while True:
     233    n = x264_preset_names[i]
     234    if n==NULL:
     235        break
     236    PRESET_NAMES[i] = str(n)
     237    PRESET_VALUES[str(n)] = i
     238    i += 1
    229239
     240TUNE_NAMES = {}
     241TUNE_VALUES = {}
     242i = 0
     243while True:
     244    n = x264_tune_names[i]
     245    if n==NULL:
     246        break
     247    TUNE_NAMES[i] = str(n)
     248    TUNE_VALUES[str(n)] = i
     249    i += 1
     250log("x264 presets: %s", PRESET_NAMES)
     251log("x264 tune: %s", TUNE_NAMES)
     252
    230253#we choose presets from 1 to 7
    231254#(we exclude placebo)
    232 cdef int get_preset_for_speed(int speed):
     255def get_preset_for_speed(int speed):
    233256    if speed > 99:
    234257        #only allow "ultrafast" if pct > 99
    235258        return 0
    236     return 7 - max(0, min(6, speed / 15))
     259    return PRESET_NAMES[7 - max(0, min(6, speed / 15))]
    237260
     261def get_tune(options):
     262    if options.get("b-frames", 0):
     263        return "film"
     264    return "zerolatency"
     265
    238266#the x264 quality option ranges from 0 (best) to 51 (lowest)
    239267cdef float get_x264_quality(int pct):
    240268    return <float> (50.0 - (min(100, max(0, pct)) * 49.0 / 100.0))
     
    374402
    375403
    376404cdef class Encoder:
    377     cdef unsigned long frames
     405    cdef unsigned long frames_in
     406    cdef unsigned long frames_out
    378407    cdef x264_t *context
    379408    cdef int width
    380409    cdef int height
     
    382411    cdef object profile
    383412    cdef double time
    384413    cdef int colorspace
    385     cdef int preset
     414    cdef object preset
     415    cdef object tune
    386416    cdef int quality
    387417    cdef int speed
    388418    cdef unsigned long long bytes_in
    389419    cdef unsigned long long bytes_out
    390420    cdef object last_frame_times
     421    cdef object slice_stats
    391422    cdef uint64_t first_frame_timestamp
    392423
    393424    cdef object __weakref__
     
    403434        self.quality = quality
    404435        self.speed = speed
    405436        self.preset = get_preset_for_speed(speed)
     437        self.tune = get_tune(options)
    406438        self.src_format = src_format
    407439        self.colorspace = cs_info[0]
    408         self.frames = 0
     440        self.frames_in = 0
     441        self.frames_out = 0
    409442        self.last_frame_times = deque(maxlen=200)
    410443        self.time = 0
    411444        self.first_frame_timestamp = 0
     445        self.slice_stats = {}
    412446        self.profile = self._get_profile(options, self.src_format)
    413447        if self.profile is not None and self.profile not in cs_info[2]:
    414448            log.warn("invalid profile specified for %s: %s (must be one of: %s)" % (src_format, self.profile, cs_info[2]))
     
    419453
    420454    cdef init_encoder(self):
    421455        cdef x264_param_t param
    422         cdef const char *preset
    423         preset = get_preset_names()[self.preset]
    424         x264_param_default_preset(&param, preset, "zerolatency")
     456        x264_param_default_preset(&param, self.preset, self.tune)
    425457        param.i_threads = X264_THREADS
    426458        if X264_THREADS!=1:
    427459            param.b_sliced_threads = 1
     
    446478        if self.context!=NULL:
    447479            x264_encoder_close(self.context)
    448480            self.context = NULL
    449         self.frames = 0
     481        self.frames_in = 0
     482        self.frames_out = 0
    450483        self.width = 0
    451484        self.height = 0
    452485        self.src_format = ""
     
    468501            return {}
    469502        info = get_info()
    470503        info.update({"profile"   : self.profile,
    471                      "preset"    : get_preset_names()[self.preset],
    472                      "frames"    : self.frames,
     504                     "preset"    : self.preset,
     505                     "tune"      : self.tune,
     506                     "frames.out": self.frames_out,
     507                     "frames.in" : self.frames_in,
    473508                     "width"     : self.width,
    474509                     "height"    : self.height,
    475510                     "speed"     : self.speed,
     
    480515            info["bytes_in"] = self.bytes_in
    481516            info["bytes_out"] = self.bytes_out
    482517            info["ratio_pct"] = int(100.0 * self.bytes_out / self.bytes_in)
    483         if self.frames>0 and self.time>0:
    484             pps = float(self.width) * float(self.height) * float(self.frames) / self.time
     518        if self.frames_out>0 and self.time>0:
     519            pps = float(self.width) * float(self.height) * float(self.frames_out) / self.time
    485520            info["total_time_ms"] = int(self.time*1000.0)
    486521            info["pixels_per_second"] = int(pps)
     522        if self.slice_stats:
     523            for k,v in self.slice_stats.items():
     524                info["slice.%s" % k] = v
    487525        #calculate fps:
    488526        cdef int f = 0
    489527        cdef double now = time.time()
     
    555593        cdef int i                        #@DuplicatedSignature
    556594        start = time.time()
    557595
    558         if self.frames==0:
     596        if self.frames_in==0:
    559597            self.first_frame_timestamp = image.get_timestamp()
    560598
    561599        if speed>=0 and abs(self.speed-speed)>5:
     
    570608        x264_picture_init(&pic_out)
    571609        x264_picture_init(&pic_in)
    572610
     611        if self.frames_in==0:
     612            pic_in.i_type = X264_TYPE_IDR
     613        else:
     614            #only use B frames if the "tune" supports it and there are frames pending:
     615            if self.tune!="zerolatency" and options.get("pending", 0)>0:
     616                pic_in.i_type = X264_TYPE_B
     617            else:
     618                pic_in.i_type = X264_TYPE_P
     619        log.info("slice type requested: %s", SLICE_TYPES.get(pic_in.i_type, pic_in.i_type))
    573620        if self.src_format.find("RGB")>=0 or self.src_format.find("BGR")>=0:
    574621            assert len(pixels)>0
    575622            assert istrides>0
     
    593640        with nogil:
    594641            frame_size = x264_encoder_encode(self.context, &nals, &i_nals, &pic_in, &pic_out)
    595642        if frame_size < 0:
    596             log.error("x264 encoding error: frame_size is invalid!")
     643            log.error("x264 encoding error: returned frame_size %i is invalid", frame_size)
    597644            return None
    598645        slice_type = SLICE_TYPES.get(pic_out.i_type, pic_out.i_type)
    599         log("x264 encode frame %i as %4s slice with %i nals, total %7i bytes", self.frames, slice_type, i_nals, frame_size)
     646        self.slice_stats[slice_type] = self.slice_stats.get(slice_type, 0)+1
     647        log("x264 encode frame %i as %4s slice with %i nals, total %7i bytes", self.frames_in, slice_type, i_nals, frame_size)
    600648        if LOG_NALS:
    601649            for i in range(i_nals):
    602650                log.info(" nal %s priority:%10s, type:%10s, payload=%#x, payload size=%#x",
    603651                         i, NAL_PRIORITIES.get(nals[i].i_ref_idc, nals[i].i_ref_idc), NAL_TYPES.get(nals[i].i_type, nals[i].i_type), <unsigned long> nals[i].p_payload, nals[i].i_payload)
    604652            #log.info("x264 nal %s: %s", i, (<char *>nals[i].p_payload)[:64])
    605         out = <char *>nals[0].p_payload
    606         cdata = out[:frame_size]
    607         self.bytes_out += frame_size
    608         #info for client:
    609         client_options = {
    610                 "frame"     : self.frames,
    611                 "pts"       : pic_out.i_pts,
    612                 "quality"   : min(99, quality),
    613                 "speed"     : speed,
    614                 "type"      : slice_type}
    615         #accounting:
    616         end = time.time()
    617         self.time += end-start
    618         self.frames += 1
    619         self.last_frame_times.append((start, end))
     653        if frame_size>0:
     654            assert nals!=NULL, "no nals!"
     655            out = <char *>nals[0].p_payload
     656            cdata = out[:frame_size]
     657            self.bytes_out += frame_size
     658            #info for client:
     659            client_options = {
     660                    "frame"     : self.frames_out,
     661                    "pts"       : pic_out.i_pts,
     662                    "quality"   : min(99, quality),
     663                    "speed"     : speed,
     664                    "type"      : slice_type}
     665            self.frames_out += 1
     666            #accounting:
     667            end = time.time()
     668            self.time += end-start
     669            self.last_frame_times.append((start, end))
     670        else:
     671            assert self.tune!="zerolatency", "no frame data with %s" % self.tune
     672            cdata = ""
     673            client_options = {
     674                    "frame"     : self.frames_out,
     675                    }
     676        self.frames_in += 1
    620677        assert self.context!=NULL
    621678        return  cdata, client_options
    622679
     
    625682        assert pct>=0 and pct<=100, "invalid percentage: %s" % pct
    626683        assert self.context!=NULL, "context is closed!"
    627684        cdef x264_param_t param                     #@DuplicatedSignature
    628         cdef int new_preset = get_preset_for_speed(pct)
     685        new_preset = get_preset_for_speed(pct)
    629686        if new_preset == self.preset:
    630687            return
    631688        self.speed = pct
     
    632689        #retrieve current parameters:
    633690        x264_encoder_parameters(self.context, &param)
    634691        #apply new preset:
    635         x264_param_default_preset(&param, get_preset_names()[new_preset], "zerolatency")
     692        x264_param_default_preset(&param, new_preset, self.tune)
    636693        #ensure quality remains what it was:
    637694        set_f_rf(&param, get_x264_quality(self.quality))
    638695        #apply it:
  • xpra/server/window_video_source.py

     
    88import time
    99import operator
    1010
    11 from xpra.net.compression import Compressed
    1211from xpra.codecs.codec_constants import get_subsampling_divs, \
    1312                                        TransientCodecException, RGB_FORMATS, PIXEL_SUBSAMPLING, LOSSY_PIXEL_FORMATS
    1413from xpra.server.window_source import WindowSource, STRICT_MODE, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY
     
    12451244        start = time.time()
    12461245        quality = max(0, min(100, self._current_quality))
    12471246        speed = max(0, min(100, self._current_speed))
     1247        #tell the video encoder if we have more frames in the queue,
     1248        #so it can use B frames:
     1249        options["pending"] = len(self.encode_queue)>0
    12481250        ret = self._video_encoder.compress_image(csc_image, quality, speed, options)
    12491251        if ret is None:
    12501252            log.error("video_encode: ouch, %s compression failed", encoding)
    12511253            return None
    12521254        data, client_options = ret
     1255        if data:
     1256            log("video_encode: no data (buffering?)")
     1257            return None
    12531258        end = time.time()
    12541259
    12551260        self.free_image_wrapper(csc_image)
     
    12641269            client_options["scaled_size"] = enc_width, enc_height
    12651270        log("video_encode encoder: %s %sx%s result is %s bytes (%.1f MPixels/s), client options=%s",
    12661271                            encoding, enc_width, enc_height, len(data), (enc_width*enc_height/(end-start+0.000001)/1024.0/1024.0), client_options)
    1267         return self._video_encoder.get_encoding(), Compressed(encoding, data), client_options, width, height, 0, 24
     1272        return self._video_encoder.get_encoding(), data, client_options, width, height, 0, 24
    12681273
    12691274    def csc_image(self, image, width, height):
    12701275        """