IndexedSymbolInstance Crash

[sjg-wdw]

We are seeing a crash in a large app deployment that has only been reproduced in the field. So we only have the stack trace.
This is with an older version of MapLibre on Android. Best to see if it still applies.

#00 pc 0000000000714a58 /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/__string:703
std::__ndk1::char_traits<char16_t>::assign(char16_t&, char16_t const&)
#01 pc 00000000007148cc /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/__tree:2139
std::__ndk1::pair<std::__ndk1::__tree_iterator<std::__ndk1::__value_type<std::__ndk1::basic_string<char16_t,
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> >, 
std::__ndk1::vector<mbgl::IndexedSymbolInstance, std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > >, 
std::__ndk1::__tree_node<std::__ndk1::__value_type<std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> >, 
std::__ndk1::vector<mbgl::IndexedSymbolInstance, std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > >, void*>*, long>, bool> std::__ndk1::__tree<std::__ndk1::__value_type<std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> >, 
std::__ndk1::vector<mbgl::IndexedSymbolInstance, std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > >, 
std::__ndk1::__map_value_compare<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> >, std::__ndk1::__value_type<std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> >, 
std::__ndk1::vector<mbgl::IndexedSymbolInstance, std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > >, 
std::__ndk1::less<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> > >, true>, 
std::__ndk1::allocator<std::__ndk1::__value_type<std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> >, 
std::__ndk1::vector<mbgl::IndexedSymbolInstance, std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > > > >::__emplace_unique_key_args<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> >, std::__ndk1::piecewise_construct_t const&, 
std::__ndk1::tuple<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> > const&>, std::__ndk1::tuple<> >(std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> > const&, std::__ndk1::piecewise_construct_t const&, std::__ndk1::tuple<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> > const&>&&, std::__ndk1::tuple<>&&)
#02 pc 000000000004661c  /apex/com.android.runtime/lib64/bionic/libc.so
#03 pc 00000000007132c4  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/map:1519
std::__ndk1::map<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> >, std::__ndk1::vector<mbgl::IndexedSymbolInstance, 
std::__ndk1::allocator<mbgl::IndexedSymbolInstance> >, std::__ndk1::less<std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> > >, 
std::__ndk1::allocator<std::__ndk1::pair<std::__ndk1::basic_string<char16_t, std::__ndk1::char_traits<char16_t>, 
std::__ndk1::allocator<char16_t> > const, std::__ndk1::vector<mbgl::IndexedSymbolInstance, 
std::__ndk1::allocator<mbgl::IndexedSymbolInstance> > > > >::operator[](std::__ndk1::basic_string<char16_t, 
std::__ndk1::char_traits<char16_t>, std::__ndk1::allocator<char16_t> > const&)
#04 pc 00000000002fbbfd  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
#05 pc 0000000000715534  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/tuple:1401
pair<const mbgl::OverscaledTileID &, const mbgl::OverscaledTileID &, std::__ndk1::vector<mbgl::SymbolInstance, 
std::__ndk1::allocator<mbgl::SymbolInstance> > &, unsigned int &, const std::__ndk1::basic_string<char, 
std::__ndk1::char_traits<char>, std::__ndk1::allocator<char> > &, 0UL, 0UL, 1UL, 2UL, 3UL>
#06 pc 00000000007152e4  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/memory:1784
void std::__ndk1::allocator_traits<std::__ndk1::allocator<std::__ndk1::__tree_node<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, void*> > >::__construct<std::__ndk1::pair<mbgl::OverscaledTileID const, mbgl::TileLayerIndex>, 
std::__ndk1::piecewise_construct_t const&, std::__ndk1::tuple<mbgl::OverscaledTileID const&>, 
std::__ndk1::tuple<mbgl::OverscaledTileID const&, std::__ndk1::vector<mbgl::SymbolInstance, 
std::__ndk1::allocator<mbgl::SymbolInstance> >&, unsigned int&, std::__ndk1::basic_string<char, 
std::__ndk1::char_traits<char>, std::__ndk1::allocator<char> > const&> >(std::__ndk1::integral_constant<bool, true>, 
std::__ndk1::allocator<std::__ndk1::__tree_node<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, void*> >&, std::__ndk1::pair<mbgl::OverscaledTileID const, mbgl::TileLayerIndex>*, 
std::__ndk1::piecewise_construct_t const&, std::__ndk1::tuple<mbgl::OverscaledTileID const&>&&, 
std::__ndk1::tuple<mbgl::OverscaledTileID const&, std::__ndk1::vector<mbgl::SymbolInstance, 
std::__ndk1::allocator<mbgl::SymbolInstance> >&, unsigned int&, std::__ndk1::basic_string<char, 
std::__ndk1::char_traits<char>, std::__ndk1::allocator<char> > const&>&&)
#07 pc 00000000002fbbfd  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
#08 pc 0000000000714024  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
/usr/local/lib/android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/c++/v1/__tree:1189�std::__ndk1::pair<std::__ndk1::__tree_iterator<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, 
std::__ndk1::__tree_node<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, void*>*, long>, bool> std::__ndk1::__tree<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, 
std::__ndk1::__map_value_compare<mbgl::OverscaledTileID, std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex>, std::__ndk1::less<mbgl::OverscaledTileID>, true>, 
std::__ndk1::allocator<std::__ndk1::__value_type<mbgl::OverscaledTileID, mbgl::TileLayerIndex> > >::__emplace_unique<std::__ndk1::piecewise_construct_t const&, std::__ndk1::tuple<mbgl::OverscaledTileID const&>, 
std::__ndk1::tuple<mbgl::OverscaledTileID const&, std::__ndk1::vector<mbgl::SymbolInstance, 
std::__ndk1::allocator<mbgl::SymbolInstance> >&, unsigned int&, std::__ndk1::basic_string<char, 
std::__ndk1::char_traits<char>, std::__ndk1::allocator<char> > const&> >(std::__ndk1::piecewise_construct_t const&, 
std::__ndk1::tuple<mbgl::OverscaledTileID const&>&&, std::__ndk1::tuple<mbgl::OverscaledTileID const&, 
std::__ndk1::vector<mbgl::SymbolInstance, std::__ndk1::allocator<mbgl::SymbolInstance> >&, unsigned int&, 
std::__ndk1::basic_string<char, std::__ndk1::char_traits<char>, std::__ndk1::allocator<char> > const&>&&)
#09 pc 00000000000c96bc  /apex/com.android.runtime/lib64/bionic/libc.so
#10 pc 000000000056d7b0  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
[symbol_bucket.cpp:315 mbgl::SymbolBucket::registerAtCrossTileIndex(mbgl::CrossTileSymbolLayerIndex&, mbgl::RenderTile const&)
#11 pc 0000000000714564  /base.apk!/lib/arm64-v8a/libmapbox-gl.so /cross_tile_symbol_index.cpp:231
mbgl::CrossTileSymbolIndex::addLayer(mbgl::RenderLayer const&, float)
#12 pc 000000000091d9cc  /base.apk!/lib/arm64-v8a/libmapbox-gl.so (_Znwm+28) /buildbot/src/android/ndk-r25-release/toolchain/llvm-project/libcxx/src/new.cpp:67 operator new(unsigned long)
#13 pc 00000000005c5524  /base.apk!/lib/arm64-v8a/libmapbox-gl.so
[render_orchestrator.cpp:389 mbgl::RenderOrchestrator::createRenderTree(std::__ndk1::shared_ptr<mbgl::UpdateParameters> const&)

[sjg-wdw]

After looking at it with Tim this appears to be a corrupt string being used as a key in a map. Also this is in v10 and we'd need to reproduce it in v11 to have any idea what might be causing it. Putting into hibernation for the moment.

[sjg-wdw]

More broadly, it's pure memory corruption. The other members near the key can also get corrupted.
Tim is trying some guard bytes and logic with the user's deployment to see if that mitigates it. But we still don't have a cause.

[sjg-wdw]

Another update. We're integrating our guard code with MapLibre android11 to see if the newer version somehow fixes the problem. No idea yet.

[louwers]

@TimSylvester made a PR that detects this issue, maybe it will help us find the root cause.

#2744