postgres: some perf work

datastore/postgres/bench_test.go

@@ -0,0 +1,108 @@
+package postgres
+
+import (
+	"bytes"
+	"crypto/md5"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/internal/wart"
+	"github.com/quay/claircore/test"
+)
+
+// This is a copy of the current implementation as of the addition of
+// [BenchmarkMD5Vuln] to serve as a comparison.
+func md5Baseline(v *claircore.Vulnerability) (string, []byte) {
+	var b bytes.Buffer
+	b.WriteString(v.Name)
+	b.WriteString(v.Description)
+	b.WriteString(v.Issued.String())
+	b.WriteString(v.Links)
+	b.WriteString(v.Severity)
+	if v.Package != nil {
+		b.WriteString(v.Package.Name)
+		b.WriteString(v.Package.Version)
+		b.WriteString(v.Package.Module)
+		b.WriteString(v.Package.Arch)
+		b.WriteString(wart.StringFromPackageKind(v.Package.Kind))
+	}
+	if v.Dist != nil {
+		b.WriteString(v.Dist.DID)
+		b.WriteString(v.Dist.Name)
+		b.WriteString(v.Dist.Version)
+		b.WriteString(v.Dist.VersionCodeName)
+		b.WriteString(v.Dist.VersionID)
+		b.WriteString(v.Dist.Arch)
+		b.WriteString(v.Dist.CPE.BindFS())
+		b.WriteString(v.Dist.PrettyName)
+	}
+	if v.Repo != nil {
+		b.WriteString(v.Repo.Name)
+		b.WriteString(v.Repo.Key)
+		b.WriteString(v.Repo.URI)
+	}
+	b.WriteString(v.ArchOperation.String())
+	b.WriteString(v.FixedInVersion)
+	if k, l, u := rangefmt(v.Range); k != nil {
+		b.WriteString(*k)
+		b.WriteString(l)
+		b.WriteString(u)
+	}
+	s := md5.Sum(b.Bytes())
+	return "md5", s[:]
+}
+
+func TestHashEquivalent(t *testing.T) {
+	cfg := &quick.Config{
+		MaxCount: 1000,
+		Values: func(vs []reflect.Value, _ *rand.Rand) {
+			gen := test.GenUniqueVulnerabilities(1, "test")
+			vs[0] = reflect.ValueOf(gen[0])
+		},
+	}
+	if err := quick.CheckEqual(md5Baseline, md5Vuln, cfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkMD5Vuln(b *testing.B) {
+	vs := test.GenUniqueVulnerabilities(1, "test")
+	run := func(f func(*claircore.Vulnerability) (string, []byte)) func(*testing.B) {
+		return func(b *testing.B) {
+			b.ReportAllocs()
+			for b.Loop() {
+				kind, hash := f(vs[0])
+				if kind != `md5` || len(hash) != 16 {
+					b.Fatal("???")
+				}
+			}
+		}
+	}
+	b.Run("Baseline", run(md5Baseline))
+	b.Run("Current", run(md5Vuln))
+}
+
+func BenchmarkVulnerabilityHash(b *testing.B) {
+	vs := test.GenUniqueVulnerabilities(1, "test")
+	b.Run("MD5", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			kind, hash := md5Vuln(vs[0])
+			if kind != hashMD5 || len(hash) != 16 {
+				b.Fatal("???")
+			}
+		}
+	})
+	b.Run("XXH64", func(b *testing.B) {
+		b.ReportAllocs()
+		for b.Loop() {
+			kind, hash := xxhVuln(vs[0])
+			if kind != hashXXH64 || len(hash) != 8 {
+				b.Fatal("???")
+			}
+		}
+	})
+}

datastore/postgres/hash.go

@@ -0,0 +1,118 @@
+package postgres
+
+import (
+	"crypto/md5"
+	"hash"
+	"io"
+	"sync"
+	"unsafe"
+
+	"github.com/cespare/xxhash/v2"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/internal/wart"
+)
+
+func doHash[H hash.Hash](h H, v *claircore.Vulnerability) {
+	tmp := make([]byte, 0, 64)
+	writeString(h, v.Name)
+	writeString(h, v.Description)
+	// BUG(hank) This codified the naive string representation of a timestamp.
+	// Changing this to use a "normal" [(time.Time).AppendText] call is a
+	// breaking change.
+	h.Write(v.Issued.AppendFormat(tmp, "2006-01-02 15:04:05.999999999 -0700 MST"))
+	writeString(h, v.Links)
+	writeString(h, v.Severity)
+	if v.Package != nil {
+		writeString(h, v.Package.Name)
+		writeString(h, v.Package.Version)
+		writeString(h, v.Package.Module)
+		writeString(h, v.Package.Arch)
+		writeString(h, wart.StringFromPackageKind(v.Package.Kind))
+	}
+	if v.Dist != nil {
+		writeString(h, v.Dist.DID)
+		writeString(h, v.Dist.Name)
+		writeString(h, v.Dist.Version)
+		writeString(h, v.Dist.VersionCodeName)
+		writeString(h, v.Dist.VersionID)
+		writeString(h, v.Dist.Arch)
+		writeString(h, v.Dist.CPE.BindFS())
+		writeString(h, v.Dist.PrettyName)
+	}
+	if v.Repo != nil {
+		writeString(h, v.Repo.Name)
+		writeString(h, v.Repo.Key)
+		writeString(h, v.Repo.URI)
+	}
+	writeString(h, v.ArchOperation.String())
+	writeString(h, v.FixedInVersion)
+	if k, l, u := rangefmt(v.Range); k != nil {
+		writeString(h, *k)
+		writeString(h, l)
+		writeString(h, u)
+	}
+}
+
+// WriteString is effectively [io.WriteString], except with an extra hack to
+// avoid an allocation when the writer is not also an [io.StringWriter].
+func writeString(w io.Writer, s string) (int, error) {
+	if sw, ok := w.(io.StringWriter); ok {
+		return sw.WriteString(s)
+	}
+	// SAFETY: Mutating the returned slice's data breaks Go's invariant that strings
+	// are immutable. Don't do it!
+	b := unsafe.Slice(unsafe.StringData(s), len(s))
+	return w.Write(b)
+}
+
+const (
+	hashMD5   = `md5`
+	hashXXH64 = `xxh64`
+)
+
+var (
+	md5Pool sync.Pool
+	xxhPool sync.Pool
+)
+
+func getMD5() hash.Hash {
+	if v := md5Pool.Get(); v != nil {
+		return v.(hash.Hash)
+	}
+	return md5.New()
+}
+
+func putMD5(h hash.Hash) {
+	h.Reset()
+	md5Pool.Put(h)
+}
+
+// Md5Vuln creates an md5 hash from the members of the passed-in Vulnerability,
+// giving us a stable, context-free identifier for this revision of the
+// Vulnerability.
+func md5Vuln(v *claircore.Vulnerability) (string, []byte) {
+	h := getMD5()
+	defer putMD5(h)
+	doHash(h, v)
+	return hashMD5, h.Sum(nil)
+}
+
+func getXXH() *xxhash.Digest {
+	if v := xxhPool.Get(); v != nil {
+		return v.(*xxhash.Digest)
+	}
+	return xxhash.New()
+}
+
+func putXXH(d *xxhash.Digest) {
+	d.Reset()
+	xxhPool.Put(d)
+}
+
+func xxhVuln(v *claircore.Vulnerability) (string, []byte) {
+	h := getXXH()
+	defer putXXH(h)
+	doHash(h, v)
+	return hashXXH64, h.Sum(nil)
+}

datastore/postgres/updatevulnerabilities.go

@@ -1,9 +1,7 @@
 package postgres
 
 import (
-	"bytes"
 	"context"
-	"crypto/md5"
 	"fmt"
 	"log/slog"
 	"strconv"
@@ -18,7 +16,6 @@ import (
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/datastore"
-	"github.com/quay/claircore/internal/wart"
 	"github.com/quay/claircore/libvuln/driver"
 )
 
@@ -447,49 +444,6 @@ func skipVulnerability(v *claircore.Vulnerability) bool {
 	return v.Package == nil || v.Package.Name == ""
 }
 
-// Md5Vuln creates an md5 hash from the members of the passed-in Vulnerability,
-// giving us a stable, context-free identifier for this revision of the
-// Vulnerability.
-func md5Vuln(v *claircore.Vulnerability) (string, []byte) {
-	var b bytes.Buffer
-	b.WriteString(v.Name)
-	b.WriteString(v.Description)
-	b.WriteString(v.Issued.String())
-	b.WriteString(v.Links)
-	b.WriteString(v.Severity)
-	if v.Package != nil {
-		b.WriteString(v.Package.Name)
-		b.WriteString(v.Package.Version)
-		b.WriteString(v.Package.Module)
-		b.WriteString(v.Package.Arch)
-		b.WriteString(wart.StringFromPackageKind(v.Package.Kind))
-	}
-	if v.Dist != nil {
-		b.WriteString(v.Dist.DID)
-		b.WriteString(v.Dist.Name)
-		b.WriteString(v.Dist.Version)
-		b.WriteString(v.Dist.VersionCodeName)
-		b.WriteString(v.Dist.VersionID)
-		b.WriteString(v.Dist.Arch)
-		b.WriteString(v.Dist.CPE.BindFS())
-		b.WriteString(v.Dist.PrettyName)
-	}
-	if v.Repo != nil {
-		b.WriteString(v.Repo.Name)
-		b.WriteString(v.Repo.Key)
-		b.WriteString(v.Repo.URI)
-	}
-	b.WriteString(v.ArchOperation.String())
-	b.WriteString(v.FixedInVersion)
-	if k, l, u := rangefmt(v.Range); k != nil {
-		b.WriteString(*k)
-		b.WriteString(l)
-		b.WriteString(u)
-	}
-	s := md5.Sum(b.Bytes())
-	return "md5", s[:]
-}
-
 func rangefmt(r *claircore.Range) (kind *string, lower, upper string) {
 	lower, upper = "{}", "{}"
 	if r == nil || r.Lower.Kind != r.Upper.Kind {