[Tracker] tracker-extract crashes in GLib: gmem.c:168: failed to allocate 2330172 bytes

[Ralph Böhme <rb netafp com>]

Hi all,

I have Tracker 1.0.1 running (on Solaris) where tracker-extract is crashing reproducibly when forcing a 
reindex of a test dataset (1000 text files).

tracker-extract.log:
...
12 May 2014, 14:46:19: Tracker:   Read 65535 bytes from file, 196621 bytes remaining until configured 
threshold is reached
12 May 2014, 14:46:19: Tracker:   Read 65535 bytes from file, 131086 bytes remaining until configured 
threshold is reached
12 May 2014, 14:46:19: Tracker:   Read 65535 bytes from file, 65551 bytes remaining until configured 
threshold is reached
12 May 2014, 14:46:19: Tracker:   Read 65535 bytes from file, 16 bytes remaining until configured threshold 
is reached
12 May 2014, 14:46:19: Tracker:   Read 16 bytes from file, 0 bytes remaining until configured threshold is 
reached
12 May 2014, 14:46:19: GLib: gmem.c:168: failed to allocate 2330172 bytes

RIP!

I have a corefile, here's the SBT:

Crashing LWP is #3:

# pstack /var/cores/core.tracker-extract.2763 
core '/var/cores/core.tracker-extract.2763' of 2763:    /opt/csw/libexec/tracker-extract
-----------------  lwp# 1 / thread# 1  --------------------
 fe6c8d75 __pollsys (84cad00, 2, 0, 0) + 15
 fe66de71 poll     (84cad00, 2, ffffffff, fe481a3c, fe414749) + 81
 fe3d855b g_poll   (84cad00, 2, ffffffff, 84cad00, 2, fe481a3c) + 2b
 fe3c86f9 g_main_context_iterate.isra.29 (1, fe7be504, fe790a00, b00, 8063f5f, 84cacb0) + 189
 fe3c8b53 g_main_loop_run (84cacb0, 0, feffead8, 100000, feffeaf0, 8062f65) + d3
 08063f5f main     (1, feffeb48, feffeb50) + 53f
 08058020 _start   (1, feffec4c, 0, feffec6d, feffec79, feffec9f) + 80
-----------------  lwp# 2 / thread# 2  --------------------
 fe6c8d75 __pollsys (815b5b8, 2, 0, 0) + 15
 fe66de71 poll     (815b5b8, 2, ffffffff, fe481a3c, fe414749) + 81
 fe3d855b g_poll   (815b5b8, 2, ffffffff, 815b5b8, 2, fe481a3c) + 2b
 fe3c86f9 g_main_context_iterate.isra.29 (1, fe7be504, fe110018, 1c20, fe23624c, 0) + 189
 fe3c8b53 g_main_loop_run (815b5e0, 8141350, 0, fe6bd275, fe481a3c, 8141350) + d3
 fe23624c gdbus_shared_thread_func (815b630, 8141350, 208, fe6c37e9, 0, 0) + 2c
 fe3f3522 g_thread_proxy (8141350, fe762000, fddbefe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde00a40) + 9d
 fe6c46f0 _lwp_start (fde00a40, 0, 0, 0, 0, 0)
-----------------  lwp# 3 / thread# 3  --------------------
 fe3cfed2 g_logv   (fe4669d9, 4, fe46d604, fdb3ec3c, 238e3c) + 1d2
 fe3d00c2 g_log    (fe4669d9, 4, fe46d604, fe46830f, 238e3c, 1494f725) + 32
 fe3ce730 g_realloc (14833008, 238e3c, fdb3ec90, 14213008, fdb3ecec, 14213008) + 80
 fe4191ba _g_gnulib_vasnprintf (0, fdb3ed4c, fd0bc35f, fdb3edf8, 14833008, 4) + 66a
 fe41a333 _g_gnulib_vasprintf (fdb3edac, fd0bc35f, fdb3edf8, fe481a3c, fe481a3c, 8b15e68) + 33
 fe413bd4 g_vasprintf (fdb3edac, fd0bc35f, fdb3edf8, 14213008, 8b24120, ffffffff) + 34
 fe3ee0d6 g_string_append_vprintf (8168260, fd0bc35f, fdb3edf8, fd0cc588, 8b15e68) + 46
 fe3ee2fb g_string_append_printf (8168260, fd0bc35f, 14213008, 1d, 8b15e68, 0) + 2b
 fd0ac385 tracker_sparql_builder_object_string (8b15e68, 14632008, fdb3ee5c, fd0abcee, 8168260, fcad11bd) + b5
 fd0ac55c tracker_sparql_builder_object_unvalidated (8b15e68, 14632008, fcad11f0, 86c26c8, 100000, 50) + 10c
 fcad106e tracker_extract_get_metadata (8b156e0, 80, 8065bee, 83cb408, 816b934, 816b920) + 10e
 08058ca3 get_file_metadata (84c4650, 8b15660, 8056bd0, fe481a3c, fe414749, fe481a3c) + c3
 080593f2 get_metadata (135d9230, 0, 0, fe48354c, 8140200, fe481a3c) + 52
 fe3f407b g_thread_pool_thread_proxy (816e808, 8140200, 208, fe6c37e9, 0, 0) + bb
 fe3f3522 g_thread_proxy (8140200, fe762000, fdb3efe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde01240) + 9d
 fe6c46f0 _lwp_start (fde01240, 0, 0, 0, 0, 0)
-----------------  lwp# 4 / thread# 4  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fda3fe98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86e7948, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fda3ff08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fda3ff08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fda3ff68, 8b21500, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140230, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140230, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140230, fe762000, fda3ffe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde01a40) + 9d
 fe6c46f0 _lwp_start (fde01a40, 0, 0, 0, 0, 0)
-----------------  lwp# 5 / thread# 5  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd940e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 81060c8, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd940f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd940f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd940f68, 86a2200, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140260, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140260, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140260, fe762000, fd940fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde02240) + 9d
 fe6c46f0 _lwp_start (fde02240, 0, 0, 0, 0, 0)
-----------------  lwp# 6 / thread# 6  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd841e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 8105888, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd841f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd841f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd841f68, 86b4cc0, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140290, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140290, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140290, fe762000, fd841fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde02a40) + 9d
 fe6c46f0 _lwp_start (fde02a40, 0, 0, 0, 0, 0)
-----------------  lwp# 7 / thread# 7  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd742e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 8105048, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd742f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd742f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd742f68, 86ce9c0, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 81402c0, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 81402c0, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (81402c0, fe762000, fd742fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde03240) + 9d
 fe6c46f0 _lwp_start (fde03240, 0, 0, 0, 0, 0)
-----------------  lwp# 8 / thread# 8  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd643e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86e9208, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd643f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd643f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd643f68, 8e25660, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 81402f0, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 81402f0, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (81402f0, fe762000, fd643fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde03a40) + 9d
 fe6c46f0 _lwp_start (fde03a40, 0, 0, 0, 0, 0)
-----------------  lwp# 9 / thread# 9  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd544e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86e9a48, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd544f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd544f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd544f68, 86ed8e0, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140320, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140320, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140320, fe762000, fd544fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde04240) + 9d
 fe6c46f0 _lwp_start (fde04240, 0, 0, 0, 0, 0)
-----------------  lwp# 10 / thread# 10  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd445e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86ea288, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd445f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd445f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd445f68, 86d4a20, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140350, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140350, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140350, fe762000, fd445fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde04a40) + 9d
 fe6c46f0 _lwp_start (fde04a40, 0, 0, 0, 0, 0)
-----------------  lwp# 11 / thread# 11  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd346e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86e7108, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd346f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd346f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, 0, 8b26aa0, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 8140380, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 8140380, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (8140380, fe762000, fd346fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde05240) + 9d
 fe6c46f0 _lwp_start (fde05240, 0, 0, 0, 0, 0)
-----------------  lwp# 12 / thread# 12  --------------------
 fe6c4749 __lwp_park (818ca80, 81713a8) + 19
 fe6be0e3 cond_wait_queue (818ca80, 81713a8, 0, fe6be600) + 63
 fe6be678 __cond_wait (818ca80, 81713a8, fd247e98, fe6be6c1) + 89
 fe6be6cf cond_wait (818ca80, 81713a8, 86e89c8, fe6be707) + 27
 fe6be71c pthread_cond_wait (818ca80, 81713a8, fd247f08, 81713a8, fe481a3c, 816b920) + 24
 fe414b25 g_cond_wait (816b928, 816b920, fd247f08, fe6bca7e, 81713a8, 0) + 35
 fe398dff g_async_queue_pop_intern_unlocked (ffffffff, ffffffff, fd247f68, 8b127a0, fe481a3c) + ef
 fe3994e3 g_async_queue_pop_unlocked (816b920, 0, 0, fe48354c, 81403b0, fe481a3c) + 33
 fe3f40bb g_thread_pool_thread_proxy (816e808, 81403b0, 208, fe6c37e9, 0, 0) + fb
 fe3f3522 g_thread_proxy (81403b0, fe762000, fd247fe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde05a40) + 9d
 fe6c46f0 _lwp_start (fde05a40, 0, 0, 0, 0, 0)
-----------------  lwp# 13 / thread# 13  --------------------
 fe6c4749 __lwp_park (818c968, 81bdc08) + 19
 fe6be0e3 cond_wait_queue (818c968, 81bdc08, fb3fee18, fe6be32b) + 63
 fe6be501 cond_wait_common (818c968, 81bdc08, fb3fee18, fe6be749) + 1e7
 fe6be7a1 __cond_timedwait (818c968, 81bdc08, fb3fee98, fe6be834) + 69
 fe6be845 cond_timedwait (818c968, 81bdc08, fb3fee98, fe6be87b) + 2a
 fe6be893 pthread_cond_timedwait (818c968, 81bdc08, fb3fee98, 0, 81bdc08, 0) + 27
 fe414c59 g_cond_wait_until (816b8e0, 816b8d8, b8d95172, 1, 81bdc08, 0) + 89
 fe398da0 g_async_queue_pop_intern_unlocked (b8d95172, 1, 82de488, fe481a3c, fb3fef40, fe7be504) + 90
 fe399684 g_async_queue_timeout_pop (816b8d8, e4e1c0, 0, fe48354c, 8141c60, 0) + 54
 fe3f4165 g_thread_pool_thread_proxy (82de488, 8141c60, 208, fe6c37e9, 0, 0) + 1a5
 fe3f3522 g_thread_proxy (8141c60, fe762000, fb3fefe8, fe6c43f9) + 72
 fe6c444c _thrp_setup (fde06240) + 9d
 fe6c46f0 _lwp_start (fde06240, 0, 0, 0, 0, 0)
-----------------  lwp# 14 / thread# 14  --------------------
 fe6c4749 __lwp_park (ef391638, ef391658) + 19
 fe6be0e3 cond_wait_queue (ef391638, ef391658, fb09ef38, fe6be32b) + 63
 fe6be501 cond_wait_common (ef391638, ef391658, fb09ef38, fe6be749) + 1e7
 fe6be7fd __cond_timedwait (ef391638, ef391658, fb09efa8, fe6be834) + c5
 fe6be845 cond_timedwait (ef391638, ef391658) + 2a
 ef368177 umem_update_thread (0, fe762000, fb09efe8, fe6c43f9) + 18f
 fe6c444c _thrp_setup (fde06a40) + 9d
 fe6c46f0 _lwp_start (fde06a40, 0, 0, 0, 0, 0)
root beast:~s

I have no idea why the realloc() fails in glib. My first thought was that there may be a memory leek 
somewhere, but checking this with Solaris libumem didn't come up with any serios leaks:

root beast:~# mdb /opt/csw/libexec/tracker-extract /var/cores/core.tracker-extract.2763 
Loading modules: [ libumem.so.1 libc.so.1 ld.so.1 ]
> ::findleaks
CACHE     LEAKED   BUFCTL CALLER
080c9590       1 080e7350 libglib-2.0.so.0.4000.0`g_mutex_impl_new+0x23
------------------------------------------------------------------------
   Total       1 buffer, 32 bytes
>

Go figure. :(
Ideas?

Thanks!
-Ralph