Valgrind notes

-r simulation --crash --logsize 1024MB -f ./foundationdb/tests/slow/WriteDuringReadAtomicRestore.toml -s 508459609 -b on

Nightly

(gdb) info threads Id Target Id Frame

• 1 Thread 0x7fd33f679d00 (LWP 32673) "fdbserver" 0x0000000003077b11 in rocksdb::ForwardRangeDelIterator::ShouldDelete(rocksdb::ParsedInternalKey const&) () 2 Thread 0x7fd33d4ff700 (LWP 32674) "rocksdb:low" 0x00007fd33eb47a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 3 Thread 0x7fd33c7ff700 (LWP 32675) "rocksdb:low" 0x00007fd33eb47a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 4 Thread 0x7fd33bbfe700 (LWP 32676) "rocksdb:low" 0x00007fd33eb47a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 5 Thread 0x7fd33a9ff700 (LWP 32677) "rocksdb:low" 0x00007fd33eb47a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 6 Thread 0x7fd339bfe700 (LWP 32678) "rocksdb:high" 0x00007fd33eb47a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 7 Thread 0x7fd32f9f1700 (LWP 32694) "fdbserver" 0x00007fd33eb47de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 (gdb) bt #0 0x0000000003077b11 in rocksdb::ForwardRangeDelIterator::ShouldDelete(rocksdb::ParsedInternalKey const&) () #1 0x0000000003079cd8 in rocksdb::RangeDelAggregator::StripeRep::ShouldDelete(rocksdb::ParsedInternalKey const&, rocksdb::RangeDelPositioningMode) () #2 0x000000000301b184 in rocksdb::DBIter::FindNextUserEntryInternal(bool, rocksdb::Slice const*) () #3 0x000000000301b753 in rocksdb::DBIter::FindNextUserEntry(bool, rocksdb::Slice const*) () #4 0x000000000301c3e2 in rocksdb::DBIter::Seek(rocksdb::Slice const&) () #5 0x00000000014bbd84 in (anonymous namespace)::RocksDBKeyValueStore::Reader::action (this=0x7fd33863c7c0, a=...) at /root/src/foundationdb/flow/Arena.h:465 #6 0x00000000014bcc17 in TypedAction<(anonymous namespace)::RocksDBKeyValueStore::Reader, (anonymous namespace)::RocksDBKeyValueStore::Reader::ReadRangeAction>::operator() (this=0x7fd32e2e25e0, p=) at /root/src/foundationdb/flow/IThreadPool.h:74

cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.27.3 -D USE_VALGRIND=ON -G Ninja && ninja -C ${HOME}/build_output -j 80 fdbserver

valgrind --track-origins=yes ./build_output/bin/fdbserver -r simulation --crash --logsize 1024MB -f ${HOME}/src/foundationdb/tests/fast/FuzzApiCorrectness.toml -s 66642079 -b on

Add storageEngineType = 4 to FuzzApiCorrectness.toml.

Check Valgrind APIs in memcheck.h

Annotate locks

	ThreadSpinLock() {
#if VALGRIND
		ANNOTATE_RWLOCK_CREATE(this);
#endif
	}
	~ThreadSpinLock() {
#if VALGRIND
		ANNOTATE_RWLOCK_DESTROY(this);
#endif
	}

Happen before/after

	// Pushes n at the end of the queue and returns the node immediately before it
	BaseNode* pushNode(BaseNode* n) {
#if VALGRIND
		ANNOTATE_HAPPENS_AFTER(&head);
#endif
		BaseNode* prev = head.exchange(n);
#if VALGRIND
		ANNOTATE_HAPPENS_BEFORE(&head);
		ANNOTATE_HAPPENS_AFTER(&prev->next);
#endif
		prev->next.store(n);
		return prev;
	}

FDB_VALGRIND_PRECISE environmental variable

Currently valgrindPrecise replaces FastAllocator::allocate with malloc, and FastAllocator::release with free. This improves diagnostics for fast-allocated memory. The main thing it improves is the case where you free a buffer and then allocate a buffer again - with FastAllocator you'll get the same buffer back, and so uses of the freed pointer either won't be noticed or will be counted as use of uninitialized memory instead of use after free.

// valgrindPrecise also enables extra instrumentation for Arenas, so you can // catch things like buffer overflows in arena-allocated memory more easily // (valgrind otherwise wouldn't know that memory used for Arena bookkeeping // should only be accessed within certain Arena routines.) Unfortunately the // current Arena contract requires some allocations to be adjacent, so we can't // insert redzones between arena allocations, but we can at least catch buffer // overflows if it's the most recently allocated memory from an Arena.

VALGRIND_MAKE_MEM_DEFINED and VALGRIND_CHECK_MEM_IS_DEFINED

	void* p = freelist;
	if (!p)
		getMagazine();
#if VALGRIND
	VALGRIND_MAKE_MEM_DEFINED(p, sizeof(void*));
#endif

#if VALGRIND
		VALGRIND_CHECK_MEM_IS_DEFINED(p, packetLen);
#endif

#if VALGRIND
			// It's possible we'll read or write a page where not all of the data is defined, but the checksum of the
			// page is still valid
			VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE(&checksum, sizeof(uint32_t));
#endif