wip: embeddings and vector db rudimentals

Author: AstraBert

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.24.5-bookworm
+FROM golang:1.25.1-alpine
 
 WORKDIR /app/
 

embeddings/embeddings.go

@@ -0,0 +1,25 @@
+package embeddings
+
+import (
+	"context"
+
+	"github.com/openai/openai-go/v2"
+	"github.com/openai/openai-go/v2/packages/param"
+)
+
+func EmbedText(text string) ([]float64, error) {
+	client := openai.NewClient()
+	embd, err := client.Embeddings.New(context.TODO(), openai.EmbeddingNewParams{
+		Input:      openai.EmbeddingNewParamsInputUnion{OfString: param.Opt[string]{Value: text}},
+		Model:      openai.EmbeddingModelTextEmbedding3Small,
+		Dimensions: param.Opt[int64]{Value: 768},
+	})
+	if err != nil {
+		return nil, err
+	}
+	embds := [][]float64{}
+	for _, item := range embd.Data {
+		embds = append(embds, item.Embedding)
+	}
+	return embds[0], nil
+}

embeddings/embeddings_test.go

@@ -0,0 +1,20 @@
+package embeddings
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEmbedText(t *testing.T) {
+	_, ok := os.LookupEnv("OPENAI_API_KEY")
+	if !ok {
+		t.Skip("Skipping test because OPENAI_API_KEY is not in the environment")
+	}
+	embd, err := EmbedText("hello world!")
+	if err != nil {
+		t.Errorf("Expecting no error, got %s", err.Error())
+	}
+	if len(embd) != 768 {
+		t.Errorf("Expecting the embedding to be of length 768, got %d", len(embd))
+	}
+}

go.mod

@@ -1,11 +1,13 @@
 module github.com/AstraBert/git-push-blog
 
-go 1.24.5
+go 1.25.1
 
 require (
 	github.com/a-h/templ v0.3.943
 	github.com/blevesearch/bleve v1.0.14
 	github.com/gomarkdown/markdown v0.0.0-20250810172220-2e2c11897d1a
+	github.com/openai/openai-go/v2 v2.6.0
+	github.com/takara-ai/serverlessVector v1.0.0
 )
 
 require (
@@ -26,6 +28,10 @@ require (
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/steveyen/gtreap v0.1.0 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/willf/bitset v1.1.10 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect

go.sum

@@ -80,6 +80,8 @@ github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOl
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/openai/openai-go/v2 v2.6.0 h1:0t3e5AUr5fsgb9TotDJNTdpGqf/SSSfMX4pr8QrV9OY=
+github.com/openai/openai-go/v2 v2.6.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -103,9 +105,21 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/takara-ai/serverlessVector v1.0.0 h1:BGXb20ZgNlgJ/4YnqBwzg9414zD4189vy2S1vfiaO/Y=
+github.com/takara-ai/serverlessVector v1.0.0/go.mod h1:fCkqNXUgsyg91gqayAoP/2vLXMY1FwU94S5OAVE6pDY=
 github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=

vectordb/vectordb.go

@@ -0,0 +1,10 @@
+package vectordb
+
+import (
+	"github.com/takara-ai/serverlessVector"
+)
+
+func CreateDB() *serverlessVector.VectorDB {
+	db := serverlessVector.NewVectorDB(384)
+	return db
+}

vectordb/vectordb_test.go

@@ -0,0 +1,21 @@
+package vectordb
+
+import "testing"
+
+func TestCreateVectorDB(t *testing.T) {
+	db := CreateDB()
+	if db.Size() > 0 {
+		t.Errorf("Expecting new DB to have size zero, got %d", db.Size())
+	}
+	if dims, ok := db.GetStats()["dimension"]; ok {
+		if dimsInt, isInt := dims.(int); isInt {
+			if dimsInt != 384 {
+				t.Errorf("Expecting 'dimension' to be 384, but got %d", dimsInt)
+			}
+		} else {
+			t.Error("Expecting 'dimension' to be an integer, but they are not.")
+		}
+	} else {
+		t.Error("Expecting 'dimension' to be found in stats, but it is not.")
+	}
+}