xpra icon
Bug tracker and wiki

Ticket #2048: threaded-encoder-init.patch

File threaded-encoder-init.patch, 11.7 KB (added by Antoine Martin, 6 months ago)

"works" but cleanup causes a segfault

  • xpra/codecs/codec_constants.py

     
    119119        log = Logger("encoding")
    120120        cur = self.get_instance_count()
    121121        if (self.max_instances>0 and cur>=self.max_instances) or cur>=_codec_spec.WARN_LIMIT:
     122            instances = tuple(self.instances.keys())
    122123            log.warn("Warning: already %s active instances of %s: %s",
    123                      cur, self.codec_class, tuple(self.instances.keys()))
     124                     cur, self.codec_class, instances)
    124125            from xpra.util import dump_references
    125             dump_references(log, self.instances.keys())
     126            dump_references(log, instances)
    126127        else:
    127128            log("make_instance() %s - instance count=%s", self.codec_type, cur)
    128129        v = self.codec_class()
  • xpra/codecs/enc_ffmpeg/encoder.pyx

     
    1515
    1616from xpra.codecs.image_wrapper import ImageWrapper
    1717from xpra.codecs.codec_constants import get_subsampling_divs, video_spec
    18 from xpra.codecs.libav_common.av_log cimport override_logger, restore_logger, av_error_str #@UnresolvedImport
     18from xpra.codecs.libav_common.av_log cimport override_logger, restore_logger, av_error_str #@UnresolvedImport pylint: disable=syntax-error
    1919from xpra.codecs.libav_common.av_log import suspend_nonfatal_logging, resume_nonfatal_logging
    2020from xpra.util import AtomicInteger, csv, print_nested_dict, reverse_dict, envint, envbool
    2121from xpra.os_util import bytestostr, strtobytes
     
    762762    cdef AVCodec *audio_codec
    763763    cdef AVStream *audio_stream
    764764    cdef AVCodecContext *audio_ctx
     765    cdef uint8_t ready
    765766
    766767    cdef object __weakref__
    767768
     
    817818            raise
    818819        else:
    819820            log("enc_ffmpeg.Encoder.init_context(%s, %s, %s) self=%s", self.width, self.height, self.src_format, self.get_info())
     821        self.ready = 1
    820822
     823    def is_ready(self):
     824        return bool(self.ready)
    821825
     826
    822827    def init_muxer(self, uintptr_t gen):
    823828        global GEN_TO_ENCODER
    824829        cdef AVOutputFormat *oformat = NULL
  • xpra/codecs/enc_proxy/encoder.py

     
    5151        self.time = 0
    5252        self.first_frame_timestamp = 0
    5353
     54    def is_ready(self):
     55        return True
     56
    5457    def get_info(self):             #@DuplicatedSignature
    5558        info = get_info()
    5659        if self.src_format is None:
  • xpra/codecs/enc_x264/encoder.pyx

     
    1515from xpra.os_util import bytestostr, strtobytes, get_cpu_count
    1616from xpra.codecs.codec_constants import get_subsampling_divs, video_spec
    1717from collections import deque
    18 from xpra.buffers.membuf cimport object_as_buffer
     18from xpra.buffers.membuf cimport object_as_buffer   #pylint: disable=syntax-error
    1919
    2020from xpra.monotonic_time cimport monotonic_time
    2121from libc.stdint cimport int64_t, uint64_t, uint8_t, uintptr_t
     
    483483    cdef object frame_types
    484484    cdef object blank_buffer
    485485    cdef uint64_t first_frame_timestamp
     486    cdef uint8_t ready
    486487
    487488    cdef object __weakref__
    488489
     
    528529            log.info("saving %s stream to %s", encoding, filename)
    529530        if BLANK_VIDEO:
    530531            self.blank_buffer = b"\0" * (self.width * self.height * 4)
     532        self.ready = 1
    531533
     534    def is_ready(self):
     535        return bool(self.ready)
     536
    532537    def get_tune(self):
    533538        log("x264: get_tune() TUNE=%s, fast_decode=%s, content_type=%s", TUNE, self.fast_decode, self.content_type)
    534539        if TUNE:
  • xpra/codecs/enc_x265/encoder.pyx

     
    1313
    1414from xpra.util import envbool
    1515from xpra.codecs.codec_constants import get_subsampling_divs, RGB_FORMATS, video_spec
    16 from xpra.buffers.membuf cimport object_as_buffer
     16from xpra.buffers.membuf cimport object_as_buffer   #pylint: disable=syntax-error
    1717
    1818from libc.stdint cimport int64_t, uint64_t, uint8_t, uint32_t, uintptr_t
    1919from xpra.monotonic_time cimport monotonic_time
     
    329329    cdef double time
    330330    cdef unsigned long frames
    331331    cdef int64_t first_frame_timestamp
     332    cdef uint8_t ready
    332333
    333334    cdef object __weakref__
    334335
     
    346347        self.preset = b"ultrafast"
    347348        self.profile = PROFILE_MAIN
    348349        self.init_encoder()
     350        self.ready = 1
    349351
     352    def is_ready(self):
     353        return bool(self.ready)
     354
    350355    cdef init_encoder(self):
    351356        global log_level
    352357        cdef const char *preset
  • xpra/codecs/nvenc/encoder.pyx

     
    1414from collections import deque, OrderedDict
    1515import ctypes
    1616from ctypes import cdll as loader, POINTER
    17 
     17from threading import RLock
    1818from pycuda import driver
    1919
    2020from xpra.os_util import WIN32, OSX, LINUX, PYTHON3, bytestostr, strtobytes
     21from xpra.make_thread import start_thread
    2122from xpra.util import AtomicInteger, engs, csv, pver, envint, envbool, first_time
    2223from xpra.codecs.cuda_common.cuda_context import (
    2324    init_all_devices, get_devices, select_device, get_device_info, get_device_name,
     
    5758cdef int DEBUG_API = envbool("XPRA_NVENC_DEBUG_API", False)
    5859cdef int GPU_MEMCOPY = envbool("XPRA_NVENC_GPU_MEMCOPY", True)
    5960cdef int CONTEXT_LIMIT = envint("XPRA_NVENC_CONTEXT_LIMIT", 32)
     61cdef int THREADED_INIT = envbool("XPRA_NVENC_THREADED_INIT", True)
    6062
    6163
     64device_lock = RLock()
     65#to prevent garbage collection during threaded_clean:
     66instances = set()
     67
     68
    6269cdef int QP_MAX_VALUE = 51   #newer versions of ffmpeg can decode up to 63
    6370
    6471YUV444_CODEC_SUPPORT = {
     
    14791486    cdef object last_frame_times
    14801487    cdef uint64_t bytes_in
    14811488    cdef uint64_t bytes_out
     1489    cdef uint8_t ready
     1490    cdef uint8_t closed
     1491    cdef uint8_t threaded_init
    14821492
    14831493    cdef object __weakref__
    14841494
     
    15511561        self.pixel_format = ""
    15521562        self.last_frame_times = deque(maxlen=200)
    15531563        self.update_bitrate()
    1554         cdef double start = monotonic_time()
    15551564
    15561565        #the pixel format we feed into the encoder
    15571566        self.pixel_format = self.get_target_pixel_format(self.quality)
    15581567        self.lossless = self.get_target_lossless(self.pixel_format, self.quality)
     1568        log("using %s %s compression at %s%% quality with pixel format %s",
     1569            ["lossy","lossless"][self.lossless], encoding, self.quality, self.pixel_format)
    15591570
     1571        instances.add(self)
     1572        self.threaded_init = options.get("threaded-init", THREADED_INIT)
     1573        if self.threaded_init:
     1574            start_thread(self.threaded_init_device, "threaded-init-device", daemon=True, args=(options,))
     1575        else:
     1576            self.init_device(options)
     1577
     1578
     1579    def threaded_init_device(self, options):
     1580        global device_lock
     1581        with device_lock:
     1582            import time
     1583            time.sleep(1)
     1584            try:
     1585                self.init_device(options)
     1586            except Exception as e:
     1587                log("threaded_init_device(%s)", options, exc_info=True)
     1588                log.warn("Warning: failed to initialize device:")
     1589                log.warn(" %s", e)
     1590                self.clean()
     1591
     1592    def init_device(self, options):
     1593        cdef double start = monotonic_time()
    15601594        self.select_cuda_device(options)
    1561         log("using %s %s compression at %s%% quality with pixel format %s", ["lossy","lossless"][self.lossless], encoding, self.quality, self.pixel_format)
    15621595        try:
    15631596            self.init_cuda()
    15641597            try:
     
    15751608            log("init_cuda failed", exc_info=True)
    15761609            record_device_failure(self.cuda_device_id)
    15771610            raise
    1578 
    15791611        cdef double end = monotonic_time()
    1580         log("init_context%s took %1.fms", (width, height, src_format, quality, speed, options), (end-start)*1000.0)
     1612        self.ready = 1
     1613        log("init_device(%s) took %1.fms", options, (end-start)*1000.0)
    15811614
     1615    def is_ready(self):
     1616        return bool(self.ready)
     1617
     1618
    15821619    def select_cuda_device(self, options={}):
    15831620        self.cuda_device_id, self.cuda_device = select_device(options.get("cuda_device", -1), min_compute=MIN_COMPUTE)
    15841621        if self.cuda_device_id<0 or not self.cuda_device:
     
    19922029        return "nvenc(%s/%s/%s - %s - %4ix%-4i)" % (self.src_format, self.pixel_format, self.codec_name, self.preset_name, self.width, self.height)
    19932030
    19942031    def is_closed(self):
    1995         return self.context==NULL
     2032        return bool(self.closed)
    19962033
    19972034    def __dealloc__(self):
    19982035        self.clean()
    19992036
    2000     def clean(self):                        #@DuplicatedSignature
     2037
     2038    def clean(self):                          #@DuplicatedSignature
     2039        self.closed = 1
     2040        if self.threaded_init:
     2041            start_thread(self.threaded_clean, "threaded-clean", daemon=True)
     2042        else:
     2043            self.do_clean()
     2044
     2045    def threaded_clean(self):
     2046        global device_lock
     2047        with device_lock:
     2048            self.do_clean()
     2049
     2050    def do_clean(self):                        #@DuplicatedSignature
    20012051        log("clean() cuda_context=%s, encoder context=%#x", self.cuda_context, <uintptr_t> self.context)
    20022052        if self.cuda_context:
    20032053            self.cuda_context.push()
     
    20482098        self.bytes_in = 0
    20492099        self.bytes_out = 0
    20502100        log("clean() done")
     2101        try:
     2102            instances.remove(self)
     2103        except KeyError:
     2104            pass
    20512105
    20522106
    20532107    cdef cuda_clean(self):
     
    27972851
    27982852        for device_id in devices:
    27992853            log("testing encoder with device %s", device_id)
    2800             options = {"cuda_device" : device_id}
     2854            options = {
     2855                "cuda_device"   : device_id,
     2856                "threaded-init" : False,
     2857                }
    28012858            try:
    28022859                test_encoder = Encoder()
    28032860                test_encoder.select_cuda_device(options)
  • xpra/server/window/window_video_source.py

     
    15251525        #but to make the code less dense:
    15261526        ve = self._video_encoder
    15271527        csce = self._csc_encoder
    1528         if ve is None or ve.is_closed() or (csce and csce.is_closed()):
     1528        if ve is None:
     1529            videolog("do_check_pipeline: no current video encoder")
    15291530            return False
     1531        if ve.is_closed():
     1532            videolog("do_check_pipeline: current video encoder %s is closed", ve)
     1533            return False
     1534        if csce and csce.is_closed():
     1535            videolog("do_check_pipeline: csc %s is closed", csce)
     1536            return False
    15301537
    15311538        if csce:
    15321539            csc_width = width & self.width_mask
     
    20602067        ve = self._video_encoder
    20612068        if not ve:
    20622069            return self.video_fallback(image, options, warn=True)
     2070        if not ve.is_ready():
     2071            log.info("video encoder is not ready, using temporary fallback")
     2072            return self.video_fallback(image, options, warn=False)
    20632073
    20642074        #we're going to use the video encoder,
    20652075        #so make sure we don't time it out: