diff --git a/datastore/postgres/bench_test.go b/datastore/postgres/bench_test.go new file mode 100644 index 000000000..ad8ea3639 --- /dev/null +++ b/datastore/postgres/bench_test.go @@ -0,0 +1,108 @@ +package postgres + +import ( + "bytes" + "crypto/md5" + "math/rand" + "reflect" + "testing" + "testing/quick" + + "github.com/quay/claircore" + "github.com/quay/claircore/internal/wart" + "github.com/quay/claircore/test" +) + +// This is a copy of the current implementation as of the addition of +// [BenchmarkMD5Vuln] to serve as a comparison. +func md5Baseline(v *claircore.Vulnerability) (string, []byte) { + var b bytes.Buffer + b.WriteString(v.Name) + b.WriteString(v.Description) + b.WriteString(v.Issued.String()) + b.WriteString(v.Links) + b.WriteString(v.Severity) + if v.Package != nil { + b.WriteString(v.Package.Name) + b.WriteString(v.Package.Version) + b.WriteString(v.Package.Module) + b.WriteString(v.Package.Arch) + b.WriteString(wart.StringFromPackageKind(v.Package.Kind)) + } + if v.Dist != nil { + b.WriteString(v.Dist.DID) + b.WriteString(v.Dist.Name) + b.WriteString(v.Dist.Version) + b.WriteString(v.Dist.VersionCodeName) + b.WriteString(v.Dist.VersionID) + b.WriteString(v.Dist.Arch) + b.WriteString(v.Dist.CPE.BindFS()) + b.WriteString(v.Dist.PrettyName) + } + if v.Repo != nil { + b.WriteString(v.Repo.Name) + b.WriteString(v.Repo.Key) + b.WriteString(v.Repo.URI) + } + b.WriteString(v.ArchOperation.String()) + b.WriteString(v.FixedInVersion) + if k, l, u := rangefmt(v.Range); k != nil { + b.WriteString(*k) + b.WriteString(l) + b.WriteString(u) + } + s := md5.Sum(b.Bytes()) + return "md5", s[:] +} + +func TestHashEquivalent(t *testing.T) { + cfg := &quick.Config{ + MaxCount: 1000, + Values: func(vs []reflect.Value, _ *rand.Rand) { + gen := test.GenUniqueVulnerabilities(1, "test") + vs[0] = reflect.ValueOf(gen[0]) + }, + } + if err := quick.CheckEqual(md5Baseline, md5Vuln, cfg); err != nil { + t.Error(err) + } +} + +func BenchmarkMD5Vuln(b *testing.B) { + vs := test.GenUniqueVulnerabilities(1, "test") + run := func(f func(*claircore.Vulnerability) (string, []byte)) func(*testing.B) { + return func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + kind, hash := f(vs[0]) + if kind != `md5` || len(hash) != 16 { + b.Fatal("???") + } + } + } + } + b.Run("Baseline", run(md5Baseline)) + b.Run("Current", run(md5Vuln)) +} + +func BenchmarkVulnerabilityHash(b *testing.B) { + vs := test.GenUniqueVulnerabilities(1, "test") + b.Run("MD5", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + kind, hash := md5Vuln(vs[0]) + if kind != hashMD5 || len(hash) != 16 { + b.Fatal("???") + } + } + }) + b.Run("XXH64", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + kind, hash := xxhVuln(vs[0]) + if kind != hashXXH64 || len(hash) != 8 { + b.Fatal("???") + } + } + }) +} diff --git a/datastore/postgres/hash.go b/datastore/postgres/hash.go new file mode 100644 index 000000000..a15820388 --- /dev/null +++ b/datastore/postgres/hash.go @@ -0,0 +1,118 @@ +package postgres + +import ( + "crypto/md5" + "hash" + "io" + "sync" + "unsafe" + + "github.com/cespare/xxhash/v2" + + "github.com/quay/claircore" + "github.com/quay/claircore/internal/wart" +) + +func doHash[H hash.Hash](h H, v *claircore.Vulnerability) { + tmp := make([]byte, 0, 64) + writeString(h, v.Name) + writeString(h, v.Description) + // BUG(hank) This codified the naive string representation of a timestamp. + // Changing this to use a "normal" [(time.Time).AppendText] call is a + // breaking change. + h.Write(v.Issued.AppendFormat(tmp, "2006-01-02 15:04:05.999999999 -0700 MST")) + writeString(h, v.Links) + writeString(h, v.Severity) + if v.Package != nil { + writeString(h, v.Package.Name) + writeString(h, v.Package.Version) + writeString(h, v.Package.Module) + writeString(h, v.Package.Arch) + writeString(h, wart.StringFromPackageKind(v.Package.Kind)) + } + if v.Dist != nil { + writeString(h, v.Dist.DID) + writeString(h, v.Dist.Name) + writeString(h, v.Dist.Version) + writeString(h, v.Dist.VersionCodeName) + writeString(h, v.Dist.VersionID) + writeString(h, v.Dist.Arch) + writeString(h, v.Dist.CPE.BindFS()) + writeString(h, v.Dist.PrettyName) + } + if v.Repo != nil { + writeString(h, v.Repo.Name) + writeString(h, v.Repo.Key) + writeString(h, v.Repo.URI) + } + writeString(h, v.ArchOperation.String()) + writeString(h, v.FixedInVersion) + if k, l, u := rangefmt(v.Range); k != nil { + writeString(h, *k) + writeString(h, l) + writeString(h, u) + } +} + +// WriteString is effectively [io.WriteString], except with an extra hack to +// avoid an allocation when the writer is not also an [io.StringWriter]. +func writeString(w io.Writer, s string) (int, error) { + if sw, ok := w.(io.StringWriter); ok { + return sw.WriteString(s) + } + // SAFETY: Mutating the returned slice's data breaks Go's invariant that strings + // are immutable. Don't do it! + b := unsafe.Slice(unsafe.StringData(s), len(s)) + return w.Write(b) +} + +const ( + hashMD5 = `md5` + hashXXH64 = `xxh64` +) + +var ( + md5Pool sync.Pool + xxhPool sync.Pool +) + +func getMD5() hash.Hash { + if v := md5Pool.Get(); v != nil { + return v.(hash.Hash) + } + return md5.New() +} + +func putMD5(h hash.Hash) { + h.Reset() + md5Pool.Put(h) +} + +// Md5Vuln creates an md5 hash from the members of the passed-in Vulnerability, +// giving us a stable, context-free identifier for this revision of the +// Vulnerability. +func md5Vuln(v *claircore.Vulnerability) (string, []byte) { + h := getMD5() + defer putMD5(h) + doHash(h, v) + return hashMD5, h.Sum(nil) +} + +func getXXH() *xxhash.Digest { + if v := xxhPool.Get(); v != nil { + return v.(*xxhash.Digest) + } + return xxhash.New() +} + +func putXXH(d *xxhash.Digest) { + d.Reset() + xxhPool.Put(d) +} + +func xxhVuln(v *claircore.Vulnerability) (string, []byte) { + h := getXXH() + defer putXXH(h) + doHash(h, v) + return hashXXH64, h.Sum(nil) +} diff --git a/datastore/postgres/updatevulnerabilities.go b/datastore/postgres/updatevulnerabilities.go index 6c89f1e8b..3009e25fd 100644 --- a/datastore/postgres/updatevulnerabilities.go +++ b/datastore/postgres/updatevulnerabilities.go @@ -1,9 +1,7 @@ package postgres import ( - "bytes" "context" - "crypto/md5" "fmt" "log/slog" "strconv" @@ -18,7 +16,6 @@ import ( "github.com/quay/claircore" "github.com/quay/claircore/datastore" - "github.com/quay/claircore/internal/wart" "github.com/quay/claircore/libvuln/driver" ) @@ -447,49 +444,6 @@ func skipVulnerability(v *claircore.Vulnerability) bool { return v.Package == nil || v.Package.Name == "" } -// Md5Vuln creates an md5 hash from the members of the passed-in Vulnerability, -// giving us a stable, context-free identifier for this revision of the -// Vulnerability. -func md5Vuln(v *claircore.Vulnerability) (string, []byte) { - var b bytes.Buffer - b.WriteString(v.Name) - b.WriteString(v.Description) - b.WriteString(v.Issued.String()) - b.WriteString(v.Links) - b.WriteString(v.Severity) - if v.Package != nil { - b.WriteString(v.Package.Name) - b.WriteString(v.Package.Version) - b.WriteString(v.Package.Module) - b.WriteString(v.Package.Arch) - b.WriteString(wart.StringFromPackageKind(v.Package.Kind)) - } - if v.Dist != nil { - b.WriteString(v.Dist.DID) - b.WriteString(v.Dist.Name) - b.WriteString(v.Dist.Version) - b.WriteString(v.Dist.VersionCodeName) - b.WriteString(v.Dist.VersionID) - b.WriteString(v.Dist.Arch) - b.WriteString(v.Dist.CPE.BindFS()) - b.WriteString(v.Dist.PrettyName) - } - if v.Repo != nil { - b.WriteString(v.Repo.Name) - b.WriteString(v.Repo.Key) - b.WriteString(v.Repo.URI) - } - b.WriteString(v.ArchOperation.String()) - b.WriteString(v.FixedInVersion) - if k, l, u := rangefmt(v.Range); k != nil { - b.WriteString(*k) - b.WriteString(l) - b.WriteString(u) - } - s := md5.Sum(b.Bytes()) - return "md5", s[:] -} - func rangefmt(r *claircore.Range) (kind *string, lower, upper string) { lower, upper = "{}", "{}" if r == nil || r.Lower.Kind != r.Upper.Kind {