From 0f9ee478249149f81797caff79e0e967ad07045a Mon Sep 17 00:00:00 2001 From: "daniel.korn" Date: Thu, 29 Oct 2026 21:34:02 +0100 Subject: [PATCH] perf: store generated points in an AppendVector Generated points were kept in a std::vector per thread on the malloc heap (121.8M points on a planet), with doubling slack and copies on every reallocation. Store them in an AppendVector instead: 8192-point chunks from mmap_allocator, which spill to the store files with --store. AppendVector gets a const operator[] for retrievePoint. Planet --store anon RSS peak 13.1 GB -> 11.2 GB, wall time unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- include/append_vector.h | 4 ++++ include/tile_data.h | 6 +++--- src/tile_data.cpp | 2 +- test/append_vector.test.cpp | 6 ++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/include/append_vector.h b/include/append_vector.h index 28b29a8b..8a66add9 100644 --- a/include/append_vector.h +++ b/include/append_vector.h @@ -162,6 +162,10 @@ namespace AppendVectorNS { return el; } + const T& operator [](int idx) const { + return vecs[idx / APPEND_VECTOR_SIZE][idx % APPEND_VECTOR_SIZE]; + } + Iterator begin() { return Iterator(*this, 0, 0); } diff --git a/include/tile_data.h b/include/tile_data.h index 5bc9fa67..c2763897 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -245,7 +245,7 @@ template void collectObjectsForTileTemplate( class TileDataSource { public: // Store for generated geometries - using point_store_t = std::vector; + using point_store_t = AppendVectorNS::AppendVector; using linestring_t = boost::geometry::model::linestring; using linestring_store_t = std::vector; @@ -391,8 +391,8 @@ class TileDataSource { const auto& shardId = getShard(id); const auto& shard = pointStores[shardId]; const auto offset = getId(id); - if (offset > shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); - return shard.at(offset); + if (offset >= shard.size()) throw std::out_of_range("Could not find generated node with id " + std::to_string(id) + ", shard " + std::to_string(shardId) + ", offset=" + std::to_string(offset)); + return shard[offset]; } NodeID storeLinestring(const Linestring& src); diff --git a/src/tile_data.cpp b/src/tile_data.cpp index 3fff009a..9a84db4f 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -643,7 +643,7 @@ NodeID TileDataSource::storePoint(const Point& input) { const auto& store = pointStore.get(this); NodeID offset = store.second->size(); - store.second->emplace_back(input); + store.second->push_back(input); NodeID rv = (store.first << (TILE_DATA_ID_SIZE - shardBits)) + offset; return rv; } diff --git a/test/append_vector.test.cpp b/test/append_vector.test.cpp index b3104428..f0d5d704 100644 --- a/test/append_vector.test.cpp +++ b/test/append_vector.test.cpp @@ -21,6 +21,12 @@ MU_TEST(test_append_vector) { mu_check(vec[25] == 25); + const AppendVector& constVec = vec; + mu_check(constVec[25] == 25); + mu_check(constVec[8191] == 8191); + mu_check(constVec[8192] == 8192); + mu_check(constVec[9999] == 9999); + const AppendVector::Iterator& it = vec.begin(); mu_check(*it == 0); mu_check(*(it + 1) == 1);