seems memory leak happens when using cgo

[AaronLin98]

version: gorocksdb@v1.6.36

function: write_batch.go

func (wb *WriteBatch) Put(key, value []byte) {
	cKey := byteToChar(key)
	cValue := byteToChar(value)
	C.rocksdb_writebatch_put(wb.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)))
}

use case:

func writeLargeObjects(db *grocksdb.DB) {
	for i := 0; i < 1000; i += 10 {
		writeLargeObjectsCore(db, i, i+10)
	}
}
func writeLargeObjectsCore(db *grocksdb.DB, l, r int) {
	largeValue := make([]byte, 1*1024*1024) // 1MB bytes
	for i := range largeValue {
		largeValue[i] = 'x'
	}

	batch := grocksdb.NewWriteBatch()
	for i := l; i < r; i++ {
		key := fmt.Sprintf("large_key_%d", i)
		batch.Put([]byte(key), largeValue)
	}

	writeOpts := grocksdb.NewDefaultWriteOptions()
	err := db.Write(writeOpts, batch)
	if err != nil {
		log.Fatalf("Write failed: %v", err)
	}
	writeOpts.Destroy()
	batch.Destroy()
}

results:
By this use case, memory leak will happen.
The VmRSS is much larger than Go sys memory and rocksdb-memtable and rocksdb-block-cache.

[linxGnu]

@AaronLin98

v1.6.36 is outdated. I highly recommend upgrading both RocksDB & grocksdb to newer version and give it another try.

[AaronLin98]

I have tried the new version, it seems like cgo has some problem for memory leak

[linxGnu]

@AaronLin98

Sorry for late reply due to personal matter. Have you tried:

• Upgrade to latest version
• Calling writebatch.Clear() before writebatch.Destroy()

[jseparator]

In compaction_filter.go line 73:

func registerCompactionFilter(filter CompactionFilter) int {
	return compactionFilters.Append(compactionFilterWrapper{C.CString(filter.Name()), filter})
}

The memory of C.CString(filter.Name()) is not freed, Also compactionFilters only Append never Remove from COWList

[linxGnu]

@jseparator

Yes, it's on purpose. Register a compaction filter once and keep references to registered ones.

[jseparator]

void rocksdb_multi_get(rocksdb_t* db, const rocksdb_readoptions_t* options,
                       size_t num_keys, const char* const* keys_list,
                       const size_t* keys_list_sizes, char** values_list,
                       size_t* values_list_sizes, char** errs) {
  std::vector<Slice> keys(num_keys);
  for (size_t i = 0; i < num_keys; i++) {
    keys[i] = Slice(keys_list[i], keys_list_sizes[i]);
  }
  std::vector<std::string> values(num_keys);
  std::vector<Status> statuses = db->rep->MultiGet(options->rep, keys, &values);
  for (size_t i = 0; i < num_keys; i++) {
    if (statuses[i].ok()) {
      values_list[i] = CopyString(values[i]);
      values_list_sizes[i] = values[i].size();
      errs[i] = nullptr;
    } else {
      values_list[i] = nullptr;
      values_list_sizes[i] = 0;
      if (!statuses[i].IsNotFound()) {
        errs[i] = strdup(statuses[i].ToString().c_str());
      } else {
        errs[i] = nullptr;
      }
    }
  }
}

// MultiGet returns the data associated with the passed keys from the database
func (db *DB) MultiGet(opts *ReadOptions, keys ...[]byte) (Slices, error) {
	// will destroy `cKeys` before return
	cKeys, cKeySizes := byteSlicesToCSlices(keys)

	vals := make(charsSlice, len(keys))
	valSizes := make(sizeTSlice, len(keys))
	rocksErrs := make(charsSlice, len(keys))

	C.rocksdb_multi_get(
		db.c,
		opts.c,
		C.size_t(len(keys)),
		cKeys.c(),
		cKeySizes.c(),
		vals.c(),
		valSizes.c(),
		rocksErrs.c(),
	)

	var errs []error

	for i, rocksErr := range rocksErrs {
		if err := fromCError(rocksErr); err != nil {
			errs = append(errs, fmt.Errorf("getting %q failed: %v", string(keys[i]), err.Error()))
		}
	}

	if len(errs) > 0 {
		cKeys.Destroy()
		return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0])
	}

	slices := make(Slices, len(keys))
	for i, val := range vals {
		slices[i] = NewSlice(val, valSizes[i])
	}

	cKeys.Destroy()
	return slices, nil
}

if len(errs) > 0 // vals memory is not freed.

if len(errs) > 0 {
        cKeys.Destroy()
	for i, val := range vals {
		if valSizes[i] > 0 {
			C.rocksdb_free(unsafe.Pointer(val))
		}
	}
	return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0])
}

[linxGnu]

That makes sense.
Could you please help by making PR @jseparator, I am very happy to review it. Thank you

[linxGnu]

Fixed by @jseparator : #186