[OpenSIPS-Users] 3.2.3 TLS issue

Gregory Massel greg at switchtel.co.za
Thu Dec 9 00:46:38 EST 2021


Herewith some backtraces:

#0  __strlen_sse2 () at ../sysdeps/x86_64/multiarch/../strlen.S:120
No locals.
#1  0x00007f3e6c2b25d3 in _IO_vfprintf_internal (s=s at entry=0x557c28170060,
     format=format at entry=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=0x7ffce94b5188)
     at vfprintf.c:1643
         len = <optimized out>
         string_malloced = 0
         string = 0x1343c6c031e289 <error: Cannot access memory at address 0x1343c6c031e289>
         width = <optimized out>
         signed_number = <optimized out>
         is_short = 0
         spec = 115 's'
         ptr = <optimized out>
         ptr = <optimized out>
         outc = <optimized out>
         step0_jumps = {0, 160, 64, 1664, 1576, 1464, 1360, 2216, 2912, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256,
           2624, -1344, -1344, 2536}
         base = <optimized out>
         pad = <optimized out>
         offset = <optimized out>
         offset = <optimized out>
         step1_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
         args_value = <optimized out>
         is_negative = <optimized out>
         number = <optimized out>
         use_outdigits = <optimized out>
         step2_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
         alt = 0
         left = 0
         showsign = <optimized out>
         group = 0
         is_char = 0
         step3a_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, -760, 3024, 3000, 3104, 2128, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0}
         the_arg = {pa_wchar = 2037198692 L'\x796d2f64', pa_int = 2037198692, pa_long_int = 7236283442049920868, pa_long_long_int = 7236283442049920868,
           pa_u_int = 2037198692, pa_u_long_int = 7236283442049920868, pa_u_long_long_int = 7236283442049920868, pa_double = 5.6278881776203929e+175,
           pa_long_double = <invalid float value>, pa_string = 0x646c7173796d2f64 <error: Cannot access memory at address 0x646c7173796d2f64>,
           pa_wstring = 0x646c7173796d2f64 <error: Cannot access memory at address 0x646c7173796d2f64>, pa_pointer = 0x646c7173796d2f64, pa_user = 0x646c7173796d2f64}
         space = <optimized out>
         is_long_double = <optimized out>
---Type <return> to continue, or q <return> to quit---
         is_long = <optimized out>
         step3b_jumps = {0 <repeats 11 times>, 1760, 0, 0, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
         step4_jumps = {0 <repeats 14 times>, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
         prec = <optimized out>
         _buffer = {__routine = 0xffffffffffffffff, __arg = 0x100000000, __canceltype = 0, __prev = 0x756b43e300000000}
         _avail = <optimized out>
         thousands_sep = 0x0
         grouping = 0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>
         done = 87
         f = <optimized out>
         lead_str_end = 0x557c27af4386 "%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n"
         end_of_spec = <optimized out>
         work_buffer = "\260MK\351\374\177\000\000\v\000\000\000>\177\000\000\332u\vF>\177\000\000\000\000\000\000\000\000\000\000h\r\000\000\000\000\000\000\373u\vF>\177\000\000\331u\vF>\177\000\000\v\000\000\000\000\000\000\000\240MK\351\374\177\000\000\200NK\351\374\177\000\000@\037\355F>\177\000\000\060\037\355F>\177\000\000 \037\355F>\177\000\000\020\037\355F>\177\000\000\000\037\355F>\177\000\000\360\036\355F>\177\000\000 \036\355F>\177\000\000\060\036\355F>\177\000\000\320\036\355F>\177\000\000`\036\355F>\177\000\000\360MK\351\374\177\000\000\000\006\332\016\343Cku\320MK\351\374\177\000\000\320MK\351\374\177\000\000\000OK\351\374\177\000\000"...
         workstart = <optimized out>
         workend = <optimized out>
         ap_save = {{gp_offset = 24, fp_offset = 48, overflow_arg_area = 0x7ffce94b52f0, reg_save_area = 0x7ffce94b5200}}
         nspecs_done = 3
         save_errno = 11
         readonly_format = 0
         __PRETTY_FUNCTION__ = "_IO_vfprintf_internal"
         __result = <optimized out>
#2  0x00007f3e6c3875f8 in ___vfprintf_chk (fp=fp at entry=0x557c28170060, flag=flag at entry=1,
     format=format at entry=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=ap at entry=0x7ffce94b5188)
     at vfprintf_chk.c:33
         _IO_acquire_lock_file = 0x557c28170060
         done = <optimized out>
#3  0x00007f3e6c36f37c in __GI___vsyslog_chk (ap=0x7ffce94b5188,
     fmt=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", flag=1, pri=<optimized out>)
     at ../misc/syslog.c:222
         buf = 0x0
         now_tm = {tm_sec = 4, tm_min = 40, tm_hour = 2, tm_mday = 9, tm_mon = 11, tm_year = 121, tm_wday = 4, tm_yday = 342, tm_isdst = 0, tm_gmtoff = 7200,
           tm_zone = 0x557c2812da20 "SAST"}
         fd = <optimized out>
         failbuf = "`\000\027(|U\000\000\001\000\000\000\000\000\000\000\v\000\000\000\000\000\000\000\310u\vF>"
         now = 1639010404
         f = 0x557c28170060
---Type <return> to continue, or q <return> to quit---
         bufsize = 0
         msgoff = 21
         saved_errno = 11
         clarg = <optimized out>
         now_tm = <optimized out>
         now = <optimized out>
         fd = <optimized out>
         f = <optimized out>
         buf = <optimized out>
         bufsize = <optimized out>
         msgoff = <optimized out>
         saved_errno = <optimized out>
         failbuf = <optimized out>
         clarg = <optimized out>
         numbuf = <optimized out>
         nump = <optimized out>
         endp = <optimized out>
         pid = <optimized out>
         __result = <optimized out>
         iov = <optimized out>
         v = <optimized out>
         __clframe = <optimized out>
         __clframe = <optimized out>
         ignore1 = <optimized out>
         ignore2 = <optimized out>
         ignore3 = <optimized out>
         ignore = <optimized out>
#4  __syslog_chk (pri=<optimized out>, flag=1, fmt=0x557c27af4378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n")
     at ../misc/syslog.c:129
         ap = {{gp_offset = 48, fp_offset = 48, overflow_arg_area = 0x7ffce94b52f8, reg_save_area = 0x7ffce94b5200}}
#5  0x0000557c279fa286 in fm_free ()
No symbol table info available.
#6  0x00007f3e45f62994 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7  0x00007f3e46001799 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8  0x00007f3e460017e9 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#9  0x00007f3e45fb1263 in wolfSSL_Cleanup () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#10 0x00007f3e45f625af in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#11 0x0000557c279c7c78 in ?? ()
No symbol table info available.
#12 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#13 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#14 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#15 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#16 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#17 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#18 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#19 0x0000557c279c7cb1 in ?? ()
No symbol table info available.
#20 0x0000557c279ca654 in destroy_modules ()
No symbol table info available.
#21 0x0000557c279ea2e0 in cleanup ()
No symbol table info available.
#22 0x0000557c279ead2b in shutdown_opensips ()
No symbol table info available.
#23 0x0000557c279c68e7 in handle_sigs ()
No symbol table info available.
#24 0x0000557c27922343 in main ()
No symbol table info available.


AND:

#0  __strlen_sse2 () at ../sysdeps/x86_64/multiarch/../strlen.S:120
No locals.
#1  0x00007f5ac10265d3 in _IO_vfprintf_internal (s=s at entry=0x55fdfaba8060,
     format=format at entry=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=0x7fffffd36458)
     at vfprintf.c:1643
         len = <optimized out>
         string_malloced = 0
         string = 0xfffcd1850f023b48 <error: Cannot access memory at address 0xfffcd1850f023b48>
         width = <optimized out>
         signed_number = <optimized out>
         is_short = 0
         spec = 115 's'
         ptr = <optimized out>
         ptr = <optimized out>
         outc = <optimized out>
         step0_jumps = {0, 160, 64, 1664, 1576, 1464, 1360, 2216, 2912, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256,
           2624, -1344, -1344, 2536}
         base = <optimized out>
         pad = <optimized out>
         offset = <optimized out>
         offset = <optimized out>
         step1_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
         args_value = <optimized out>
         is_negative = <optimized out>
         number = <optimized out>
         use_outdigits = <optimized out>
         step2_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 1848, 1760, -768, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, -1344, -1344, 0}
         alt = 0
         left = 0
         showsign = <optimized out>
         group = 0
         is_char = 0
         step3a_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 0, 0, -760, 3024, 3000, 3104, 2128, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0}
         the_arg = {pa_wchar = 0 L'\000', pa_int = 0, pa_long_int = 0, pa_long_long_int = 0, pa_u_int = 0, pa_u_long_int = 0, pa_u_long_long_int = 0, pa_double = 0,
           pa_long_double = <invalid float value>, pa_string = 0x0, pa_wstring = 0x0, pa_pointer = 0x0, pa_user = 0x0}
         space = <optimized out>
         is_long_double = <optimized out>
         is_long = <optimized out>
         step3b_jumps = {0 <repeats 11 times>, 1760, 0, 0, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
---Type <return> to continue, or q <return> to quit---
         step4_jumps = {0 <repeats 14 times>, -760, 3024, 3000, 3104, 2128, 3120, -304, -704, 384, 312, -80, -1256, 2624, 0, 0, 0}
         prec = <optimized out>
         _buffer = {__routine = 0x14, __arg = 0x8b6761cc37031500, __canceltype = -1060996992, __prev = 0x7f5ac0bd4020}
         _avail = <optimized out>
         thousands_sep = 0x0
         grouping = 0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff>
         done = 87
         f = <optimized out>
         lead_str_end = 0x55fdf9265386 "%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n"
         end_of_spec = <optimized out>
         work_buffer = "\036\377,\347ir\231crxiz\v\327Q_\000\000\000\000\000\000\000\000\000\025\003\067\314ag\213\243B\324P\213j\301\256\003\000\000\000\000\000\000\000\070A\275\300Z\177\000\000\000\000\000\000\000\000\000\000\024\000\000\000\000\000\000\000\273\343%\371\375U\000\000\000п\300Z\177\000\000Q\215\023\371\375U\000\000p`\323\377\377\177\000\000\340\000\000\000\000\000\000\000@\000\000\000\017\000\000\000\a\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000x\260\262\372\375U\000\000\020\001\000\000\000\000\000\000\260\377\377\377\377\377\377\377\003\000\000\000\000\000\000\000\000\025\003\067\314ag\213\240`\323\377\377\177\000\000\240`\323\377\377\177\000\000\320a\323\377\377\177\000\000\225"...
         workstart = <optimized out>
         workend = <optimized out>
         ap_save = {{gp_offset = 24, fp_offset = 48, overflow_arg_area = 0x7fffffd365c0, reg_save_area = 0x7fffffd364d0}}
         nspecs_done = 3
         save_errno = 11
         readonly_format = 0
         __PRETTY_FUNCTION__ = "_IO_vfprintf_internal"
         __result = <optimized out>
#2  0x00007f5ac10fb5f8 in ___vfprintf_chk (fp=fp at entry=0x55fdfaba8060, flag=flag at entry=1,
     format=format at entry=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", ap=ap at entry=0x7fffffd36458)
     at vfprintf_chk.c:33
         _IO_acquire_lock_file = 0x55fdfaba8060
         done = <optimized out>
#3  0x00007f5ac10e337c in __GI___vsyslog_chk (ap=0x7fffffd36458,
     fmt=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n", flag=1, pri=<optimized out>)
     at ../misc/syslog.c:222
         buf = 0x0
         now_tm = {tm_sec = 37, tm_min = 39, tm_hour = 2, tm_mday = 9, tm_mon = 11, tm_year = 121, tm_wday = 4, tm_yday = 342, tm_isdst = 0, tm_gmtoff = 7200,
           tm_zone = 0x55fdfab65a20 "SAST"}
         fd = <optimized out>
         failbuf = "0\000\000\000\060\000\000\000\200e\323\377\377\177\000\000\300d\323\377\377\177\000\000\000\025\003", <incomplete sequence \314>
         now = 1639010377
         f = 0x55fdfaba8060
         bufsize = 0
---Type <return> to continue, or q <return> to quit---
         msgoff = 21
         saved_errno = 11
         clarg = <optimized out>
         now_tm = <optimized out>
         now = <optimized out>
         fd = <optimized out>
         f = <optimized out>
         buf = <optimized out>
         bufsize = <optimized out>
         msgoff = <optimized out>
         saved_errno = <optimized out>
         failbuf = <optimized out>
         clarg = <optimized out>
         numbuf = <optimized out>
         nump = <optimized out>
         endp = <optimized out>
         pid = <optimized out>
         __result = <optimized out>
         iov = <optimized out>
         v = <optimized out>
         __clframe = <optimized out>
         __clframe = <optimized out>
         ignore1 = <optimized out>
         ignore2 = <optimized out>
         ignore3 = <optimized out>
         ignore = <optimized out>
#4  __syslog_chk (pri=<optimized out>, flag=1, fmt=0x55fdf9265378 "CRITICAL:core:%s: freeing already freed %s pointer (%p), first free: %s: %s(%ld) - aborting!\n")
     at ../misc/syslog.c:129
         ap = {{gp_offset = 48, fp_offset = 48, overflow_arg_area = 0x7fffffd365c8, reg_save_area = 0x7fffffd364d0}}
#5  0x000055fdf916b286 in fm_free ()
No symbol table info available.
#6  0x00007f5a9acd6994 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7  0x00007f5a9ad75799 in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8  0x00007f5a9ace6e25 in _wolfssl_tls_read () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#9  0x00007f5a9a87d55f in tls_read () from /usr/lib/x86_64-linux-gnu/opensips/modules/tls_mgm.so
No symbol table info available.
---Type <return> to continue, or q <return> to quit---
#10 0x00007f5a9aa8e51e in ?? () from /usr/lib/x86_64-linux-gnu/opensips/modules/proto_tls.so
No symbol table info available.
#11 0x000055fdf92197fa in ?? ()
No symbol table info available.
#12 0x000055fdf921b042 in tcp_worker_proc_loop ()
No symbol table info available.
#13 0x000055fdf9227046 in tcp_start_processes ()
No symbol table info available.
#14 0x000055fdf9093059 in main ()
No symbol table info available.


And


#0 __GI_raise (sig=sig at entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
set = {__val = {0, 10045105023597352192, 11, 94549321613408, 0, 
10045105023597352192, 11, 1, 94549321613408, 1, 11, 94549295125368, 21, 
140027762717882,
140737485432176, 1639010387}}
pid = <optimized out>
tid = <optimized out>
ret = <optimized out>
#1 0x00007f5ac1009921 in __GI_abort () at abort.c:79
save_stage = 1
act = {__sigaction_handler = {sa_handler = 0x7fffffd367f0, sa_sigaction 
= 0x7fffffd367f0}, sa_mask = {__val = {140737485432576, 140737485432560, 16,
140027221347480, 140027224679288, 140027221347744, 94549293278303, 
140737485432544, 140027116336909, 0, 0, 0, 0, 140027224966528, 37504, 
94549295118192}},
sa_flags = -115953018, sa_restorer = 0x0}
sigs = {__val = {32, 0 <repeats 15 times>}}
__cnt = <optimized out>
__set = <optimized out>
__cnt = <optimized out>
__set = <optimized out>
#2 0x000055fdf916b25b in fm_free ()
No symbol table info available.
#3 0x00007f5a9acd6994 in ?? () from 
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#4 0x00007f5a9ad75799 in ?? () from 
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#5 0x00007f5a9ad757e9 in ?? () from 
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#6 0x00007f5a9ad25263 in wolfSSL_Cleanup () from 
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#7 0x00007f5a9acd65af in ?? () from 
/usr/lib/x86_64-linux-gnu/opensips/modules/tls_wolfssl.so
No symbol table info available.
#8 0x000055fdf9138c78 in ?? ()
No symbol table info available.
#9 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#10 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#11 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#12 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#13 0x000055fdf9138cb1 in ?? ()
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#14 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#15 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#16 0x000055fdf9138cb1 in ?? ()
No symbol table info available.
#17 0x000055fdf913b654 in destroy_modules ()
No symbol table info available.
#18 0x000055fdf915b2e0 in cleanup ()
No symbol table info available.
#19 0x000055fdf915bd2b in shutdown_opensips ()
No symbol table info available.
#20 0x000055fdf9137798 in handle_sigs ()
No symbol table info available.
#21 0x000055fdf9093343 in main ()
No symbol table info available.

Regards

Greg


On 2021-11-18 23:36, Vlad Patrascu wrote:
>
> Hi Gregory,
>
> It is difficult to draw any conclusions without a backtrace so I'm 
> hoping you'll be able to get the proper core dumps again in order to 
> investigate this further.
>
> Regards,
> -- 
> Vlad Patrascu
> OpenSIPS Core Developer
> http://www.opensips-solutions.com
> On 12.11.2021 14:44, Gregory Massel wrote:
>>
>> Sorry, I do have the core files, however, my /usr/sbin/opensips is 
>> now 3.2.2 and the core dumps were taken against 3.2.3.
>>
>> I should have thought to get the backtrace before downgrading.
>>
>> This is a production server, however, I will try and schedule some 
>> downtime and then restore 3.2.3 to generate the backtrace and will 
>> try out the nightly release.
>>
>> --Greg
>>
>>
>>
>> _______________________________________________
>> Users mailing list
>> Users at lists.opensips.org
>> http://lists.opensips.org/cgi-bin/mailman/listinfo/users
>
> _______________________________________________
> Users mailing list
> Users at lists.opensips.org
> http://lists.opensips.org/cgi-bin/mailman/listinfo/users
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.opensips.org/pipermail/users/attachments/20211209/17965f7c/attachment-0001.html>


More information about the Users mailing list