perf: 3.4x faster planet build (2h10 -> 39 min), less RAM, deterministic output

include/attribute_store.h

@@ -196,11 +196,14 @@ class AttributePairStore {
 		lookupsUncached(0)
 	{
 		// The "hot" shard has a capacity of 64K, the others are unbounded.
-		pairs.push_back(DequeMap<AttributePair>(1 << 16));
+		// reserve + emplace: DequeMap's move constructor is not noexcept, so
+		// vector growth would copy every shard.
+		pairs.reserve(ATTRIBUTE_SHARDS);
+		pairs.emplace_back(1 << 16);
 		// Reserve offset 0 as a sentinel
 		pairs[0].add(AttributePair(0, false, 0));
 		for (size_t i = 1; i < ATTRIBUTE_SHARDS; i++)
-			pairs.push_back(DequeMap<AttributePair>());
+			pairs.emplace_back();
 	}
 
 	void finalize() { finalized = true; }
@@ -345,6 +348,20 @@ struct AttributeSet {
 		return getValueAtIndex(actualIndex);
 	}
 
+	// Calls fn(pairIndex) for every pair, in the same order as getPair(0..numPairs()-1)
+	template<class F>
+	void forEachPair(F&& fn) const {
+		if (useVector) {
+			for (const uint32_t v : intValues)
+				fn(v);
+			return;
+		}
+
+		for (size_t i = 0; i < 8; i++)
+			if (isSet(i))
+				fn(getValueAtIndex(i));
+	}
+
 	AttributeSet(): useVector(false) {
 		for (int i = 0; i < sizeof(shortValues)/sizeof(shortValues[0]); i++)
 			shortValues[i] = 0;
@@ -402,6 +419,26 @@ struct AttributeSet {
 struct AttributeStore {
 	AttributeIndex add(AttributeSet &attributes);
 	std::vector<const AttributePair*> getUnsafe(AttributeIndex index) const;
+	// As getUnsafe, but calls fn(const AttributePair&) for each pair in order, without building a vector
+	template<class F>
+	void forEachPairUnsafe(AttributeIndex index, F&& fn) const {
+		forEachPairIndexUnsafe(index, [&](uint32_t pairIndex) {
+			fn(pairStore.getPairUnsafe(pairIndex));
+		});
+	}
+	// As forEachPairUnsafe, but calls fn(pairIndex) (for pairStore.getPairUnsafe)
+	template<class F>
+	void forEachPairIndexUnsafe(AttributeIndex index, F&& fn) const {
+		const AttributeSet* attrSet;
+		try {
+			uint32_t shard = index >> (32 - SHARD_BITS);
+			uint32_t offset = index & (~(~0u << (32 - SHARD_BITS)));
+			attrSet = &sets[shard].at(offset);
+		} catch (std::out_of_range &err) {
+			throw std::runtime_error("Failed to fetch attributes at index "+std::to_string(index));
+		}
+		attrSet->forEachPair(fn);
+	}
 	void reset(); // used for testing
 	size_t size() const;
 	void reportSize() const;
@@ -411,6 +448,11 @@ struct AttributeStore {
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, double v, char minzoom);
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, int v, char minzoom);
 	void addAttribute(AttributeSet& attributeSet, std::string const &key, bool v, char minzoom);
+	// As above, with the key already resolved by keyStore.key2index(key) (not overloads: kaguya::LuaRef keys convert to anything)
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, std::string const &key, const protozero::data_view v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, double v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, int v, char minzoom);
+	void addAttributeWithKeyIndex(AttributeSet& attributeSet, uint16_t keyIndex, bool v, char minzoom);
 
 	AttributeStore():
 		finalized(false),

include/clip_cache.h

@@ -5,75 +5,157 @@
 #include "coordinates_geom.h"
 #include "geom.h"
 #include <mutex>
+#include <unordered_map>
 
 class TileBbox;
 
+// Identifies the tile batch the current thread is working on. The clip cache
+// is per-thread and is reset whenever the batch changes, so a tile can only
+// reuse clips of its ancestors from the same batch. This makes output depend
+// only on the batch partition, not on thread count or timing.
+extern thread_local uint64_t clipCacheBatchId;
+// Set when the current batch can't reuse clips (a single tile), so add() is skipped.
+extern thread_local bool clipCacheNoStore;
+
 template <class T>
 class ClipCache {
 public:
 	ClipCache(size_t threadNum, unsigned int baseZoom):
-		baseZoom(baseZoom),
-		clipCache(threadNum * 16),
-		clipCacheMutex(threadNum * 16),
-		clipCacheSize(threadNum * 16) {
+		baseZoom(baseZoom) {
 	}
 
-	const std::shared_ptr<T> get(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const{
-		// Look for a previously clipped version at z-1, z-2, ...
-
-		std::lock_guard<std::mutex> lock(clipCacheMutex[objectID % clipCacheMutex.size()]);
-		while (zoom > 0) {
-			zoom--;
-			x /= 2;
-			y /= 2;
-			const auto& cache = clipCache[objectID % clipCache.size()];
-			const auto& rv = cache.find(std::make_tuple(zoom, TileCoordinates(x, y), objectID));
-			if (rv != cache.end()) {
-				return rv->second;
+	// An entry holds the clipped geometry and whether it is known to be valid
+	// (passed geom::is_valid). Callers that don't track validity get false.
+	struct Entry {
+		std::shared_ptr<T> geom;
+		bool valid = false;
+	};
+
+	const std::shared_ptr<T> get(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const {
+		return getEntry(zoom, x, y, objectID).geom;
+	}
+
+	Entry getEntry(uint zoom, TileCoordinate x, TileCoordinate y, NodeID objectID) const {
+		// Look for a previously clipped version at z-1, z-2, ... (nearest ancestor first)
+		LocalStore& s = local(zoom, TileCoordinates(x, y));
+		Entry rv;
+		const Key key{(const void*)this, objectID};
+		for (uint z = zoom < MAX_LEVELS ? zoom : MAX_LEVELS; z-- > 0; ) {
+			if (!(s.usedMask & (1u << z)))
+				continue;
+			const Level& level = s.levels[z];
+			const uint shift = zoom - z;
+			if (level.index.x != ((uint64_t)x >> shift) || level.index.y != ((uint64_t)y >> shift))
+				continue;
+			const auto it = level.m.find(key);
+			if (it != level.m.end()) {
+				rv = it->second;
+				break;
 			}
 		}
-
-		return nullptr;
+		return rv;
 	}
 
-	void add(const TileBbox& bbox, const NodeID objectID, const T& output) {
+	void add(const TileBbox& bbox, const NodeID objectID, const T& output, bool valid = false) {
 		// The point of caching is to reuse the clip, so caching at the terminal zoom is
 		// pointless.
-		if (bbox.zoom == baseZoom)
+		if (bbox.zoom == baseZoom || clipCacheNoStore)
+			return;
+
+		// Levels beyond MAX_LEVELS could never be looked up by a child tile anyway
+		// (tile zooms are far below this).
+		if (bbox.zoom >= MAX_LEVELS)
 			return;
 
 		std::shared_ptr<T> copy = std::make_shared<T>();
 		boost::geometry::assign(*copy, output);
 
-		size_t index = objectID % clipCacheMutex.size();
-		std::vector<std::shared_ptr<T>> objects;
-		std::lock_guard<std::mutex> lock(clipCacheMutex[index]);
-		auto& cache = clipCache[index];
-		// Reset the cache periodically so it doesn't grow without bound.
-		//
-		// I also tried boost's lru_cache -- but it seemed to perform worse, maybe
-		// due to the bookkeeping? We could try authoring a bounded map that
-		// evicts in FIFO order, which will have less bookkeeping.
-		clipCacheSize[index]++;
-		if (clipCacheSize[index] > 1024) {
-			clipCacheSize[index] = 0;
-			// Copy the map's contents to a vector so that calling .clear()
-			// and releasing the lock can happen separately from running the
-			// destructors of all of the objects.
-			objects.reserve(1025);
-			for (const auto& x : cache)
-				objects.push_back(x.second);
-			cache.clear();
-		}
+		LocalStore& s = local(bbox.zoom, bbox.index);
+		// Safety cap (pruning normally keeps the store small). Count-based, so
+		// the result stays deterministic.
+		if (s.total > (1 << 18))
+			s.clearFrom(0);
 
-		cache[std::make_tuple(bbox.zoom, bbox.index, objectID)] = copy;
+		Level& level = s.levels[bbox.zoom];
+		if ((s.usedMask & (1u << bbox.zoom)) && !(level.index == bbox.index)) {
+			// Can't happen: local() just dropped every level >= bbox.zoom unless
+			// bbox is the current tile. Kept so a level only ever holds one tile.
+			s.total -= level.m.size();
+			level.m.clear();
+		}
+		s.usedMask |= (1u << bbox.zoom);
+		level.index = bbox.index;
+		auto ins = level.m.emplace(Key{(const void*)this, objectID}, Entry{copy, valid});
+		if (ins.second)
+			s.total++;
+		else
+			ins.first->second = Entry{copy, valid};
 	}
 
 private:
+	static const uint MAX_LEVELS = 32;
+
+	// Entries of one ClipCache instance (`owner`) for one object.
+	struct Key {
+		const void* owner;
+		NodeID id;
+		bool operator==(const Key& o) const { return owner == o.owner && id == o.id; }
+	};
+	struct KeyHash {
+		size_t operator()(const Key& k) const {
+			uint64_t h = (uint64_t)k.id * 0x9E3779B97F4A7C15ull ^ (uint64_t)(uintptr_t)k.owner;
+			h ^= h >> 29;
+			h *= 0xBF58476D1CE4E5B9ull;
+			h ^= h >> 32;
+			return (size_t)h;
+		}
+	};
+	// All entries stored at one zoom. Only one tile per zoom can be live at a
+	// time (see below), so each level holds a single tile index.
+	struct Level {
+		TileCoordinates index = TileCoordinates(0, 0);
+		std::unordered_map<Key, Entry, KeyHash> m;
+	};
+
+	// Per-thread store, shared by all ClipCache instances on the thread (hence
+	// keyed on `this`), cleared when the thread starts a new batch.
+	// Tiles within a batch come in depth-first order, so when a new tile at
+	// zoom z starts, no entry at zoom >= z can be an ancestor of it or of any
+	// later tile: those are dropped, leaving only the current tile's ancestors.
+	struct LocalStore {
+		uint64_t gen = ~0ull;
+		uint zoom = 0;
+		TileCoordinates index = TileCoordinates(0, 0);
+		uint32_t usedMask = 0;
+		size_t total = 0;
+		Level levels[MAX_LEVELS];
+
+		// Drop every level >= zoom.
+		void clearFrom(uint zoom) {
+			for (uint z = zoom; z < MAX_LEVELS; z++) {
+				if (usedMask & (1u << z)) {
+					total -= levels[z].m.size();
+					levels[z].m.clear();
+				}
+			}
+			usedMask &= zoom >= MAX_LEVELS ? ~0u : ((1u << zoom) - 1);
+		}
+	};
+	static LocalStore& local(uint zoom, TileCoordinates index) {
+		static thread_local LocalStore s;
+		if (s.gen != clipCacheBatchId) {
+			s.clearFrom(0);
+			s.gen = clipCacheBatchId;
+		} else if (s.zoom == zoom && s.index == index) {
+			return s;
+		}
+		s.zoom = zoom;
+		s.index = index;
+		s.clearFrom(zoom);
+		return s;
+	}
+
 	unsigned int baseZoom;
-	std::vector<std::map<std::tuple<uint16_t, TileCoordinates, NodeID>, std::shared_ptr<T>>> clipCache;
-	mutable std::vector<std::mutex> clipCacheMutex;
-	std::vector<size_t> clipCacheSize;
 };
 
 #endif

include/coordinates.h

@@ -7,6 +7,7 @@
 //
 // Things that pull in boost::geometry should go in coordinates_geom.h
 
+#include <cstddef>
 #include <cstdint>
 #include <utility>
 #include <vector>
@@ -127,6 +128,10 @@ double meter2degp(double meter, double latp);
 
 // the range between smallest y and largest y is filled, for each x
 void fillCoveredTiles(std::unordered_set<TileCoordinates> &tileSet);
+// Same for a vector sorted by TileCoordinatesCompare without duplicates; the result stays sorted and unique.
+void fillCoveredTiles(std::vector<TileCoordinates> &tiles);
+// Sort (TileCoordinatesCompare) and dedupe tiles[start, end).
+void sortUniqueTiles(std::vector<TileCoordinates> &tiles, std::size_t start = 0);
 
 #endif //_COORDINATES_H
 

include/coordinates_geom.h

@@ -6,6 +6,9 @@
 
 void insertIntermediateTiles(Linestring const &points, uint baseZoom, std::unordered_set<TileCoordinates> &tileSet);
 void insertIntermediateTiles(Ring const &points, uint baseZoom, std::unordered_set<TileCoordinates> &tileSet);
+// Vector variants: append the tiles to `tiles`, then sort and dedupe the appended range [old size, end).
+void insertIntermediateTiles(Linestring const &points, uint baseZoom, std::vector<TileCoordinates> &tiles);
+void insertIntermediateTiles(Ring const &points, uint baseZoom, std::vector<TileCoordinates> &tiles);
 
 // ------------------------------------------------------
 // Helper class for dealing with spherical Mercator tiles

include/geom.h

@@ -82,14 +82,24 @@ void make_valid(MultiPolygon &mp);
 // make_valid first (preserves area), then a zero-width buffer as a last
 // resort. Returns true if mp is valid afterwards; on failure mp is left as the
 // best-effort input so callers never regress.
-bool repair_multi_polygon(MultiPolygon &mp);
+bool repair_multi_polygon(MultiPolygon &mp, bool knownInvalid = false);
 
 void union_many(std::vector<MultiPolygon> &mps);
 
 Point intersect_edge(Point const &a, Point const &b, char edge, Box const &bbox);
 char bit_code(Point const &p, Box const &bbox);
-void fast_clip(Ring &points, Box const &bbox);
-void fast_clip(MultiPolygon &mp, Box const &bbox);
+// fast_clip returns true if the geometry was modified (false means identity).
+bool fast_clip(Ring &points, Box const &bbox);
+bool fast_clip(MultiPolygon &mp, Box const &bbox);
+
+// Cheap conservative validity tests, run before geom::is_valid. They return
+// true only when the geometry is certainly valid (is_valid would return true
+// with no_failure); false means "unsure" and the caller must run is_valid.
+// Tolerances are relative to the coordinate magnitude.
+bool polygonDefinitelyValid(const Polygon &p);
+bool multiPolygonDefinitelyValid(const MultiPolygon &mp);
+
+
 
 #endif //_GEOM_TYPES_H
 

include/geometry_cache.h

@@ -9,16 +9,19 @@
 #include "coordinates.h"
 #include <memory>
 
-#define NUM_BUCKETS 256
-// Keep bucket size small so linear search is fast
-#define BUCKET_SIZE 32
-
+#include <vector>
 
 template <class T>
 class GeometryCache {
 public:
+	// 64 x 32 entries: the cache only lives for one tile batch (see clear()), so a
+	// smaller table loses few hits and pins much less heap in every tile thread.
+	// Keep bucket size small so linear search is fast.
+	enum : size_t { NUM_BUCKETS = 64, BUCKET_SIZE = 32 };
+
 	GeometryCache():
 		offsets(NUM_BUCKETS),
+		filled(NUM_BUCKETS),
 		ids(NUM_BUCKETS * BUCKET_SIZE),
 		geometries(NUM_BUCKETS * BUCKET_SIZE) {}
 
@@ -35,16 +38,37 @@ class GeometryCache {
 	void add(NodeID objectID, std::shared_ptr<T> geom) {
 		const size_t bucket = objectID % NUM_BUCKETS;
 		size_t offset = bucket * BUCKET_SIZE + offsets[bucket];
-		geometries[offset] = geom;
+		geometries[offset] = std::move(geom);
 		ids[offset] = objectID;
+		if (filled[bucket] == 0)
+			dirtyBuckets.push_back(bucket);
+		if (filled[bucket] < BUCKET_SIZE)
+			filled[bucket]++;
 		offsets[bucket]++;
 
 		if (offsets[bucket] == BUCKET_SIZE)
 			offsets[bucket] = 0;
 	}
 
+	// Drop all entries (and release their geometries). Only touches the
+	// buckets used since the last clear, so it is cheap on a lightly used cache.
+	void clear() {
+		for (size_t bucket : dirtyBuckets) {
+			const size_t start = bucket * BUCKET_SIZE;
+			for (size_t i = start; i < start + filled[bucket]; i++) {
+				ids[i] = 0;
+				geometries[i].reset();
+			}
+			filled[bucket] = 0;
+			offsets[bucket] = 0;
+		}
+		dirtyBuckets.clear();
+	}
+
 private:
 	std::vector<uint8_t> offsets;
+	std::vector<uint8_t> filled;          // used slots per bucket since clear()
+	std::vector<size_t> dirtyBuckets;     // buckets with filled > 0
 	std::vector<NodeID> ids;
 	// CONSIDER: we could use a raw pointer - getOrBuildLinestring controls the lifetime
 	std::vector<std::shared_ptr<Linestring>> geometries;