fixed:bloom filter doesn't work on Chinese

Author: mkitsdts

.gitignore

@@ -6,4 +6,7 @@
 /static/server.crt
 /plugin/*.vsix
 /.vscode/
-/config.yaml
+/config.yaml
+/mkblog.pid
+/nohup.out
+/test_add_article.sh

Makefile

@@ -16,7 +16,7 @@ copy:
 	@echo "Copying frontend build to static/ and static/assets/..."
 	mkdir -p static static/assets
 	@# detect build output dir (common: dist or build)
-	if [ -d frontend/dist ]; then DIST=frontend/dist; elif [ -d frontend/build ]; then DIST=frontend/build; else echo "No frontend build dir (frontend/dist or frontend/build). Run 'make frontend-build' first." && exit 1; fi; \
+	if [ -d frontend/dist ]; then DIST=frontend/dist; elif [ -d frontend/build ]; then DIST=frontend/build; else echo "No frontend build dir (frontend/dist or frontend/build). Run 'make fbuild' first." && exit 1; fi; \
 	cp -f "$$DIST/index.html" static/; \
 	find "$$DIST" -type f \( -name '*.css' -o -name '*.js' \) -exec cp {} static/assets/ \;
 

config/config.go

@@ -126,7 +126,6 @@ func init() {
 	if !Cfg.TLS.Enabled {
 		slog.Info("loaded configuration successfully", "mysql", Cfg.MySQL, "tls", Cfg.TLS, "auth_enabled", Cfg.Auth.Enabled)
 		slog.Warn("TLS is disabled, consider enabling it in production environments")
-		panic("TLS is disabled, please enable it for better security")
 	}
 
 	if cert := os.Getenv("TLS_CERT"); cert != "" {

pkg/bloom/bloom.go

@@ -8,7 +8,7 @@ import (
 type BloomFilter struct {
 	bitset   []byte
 	bitcount []uint8
-	hashs    []func(data []byte) uint8
+	hashs    []func(data []rune) uint8
 }
 
 const (
@@ -19,30 +19,26 @@ const (
 
 var bf *BloomFilter
 
-func GetBloomFilter() *BloomFilter {
-	return bf
-}
-
 func init() {
 	bf = &BloomFilter{
 		bitset:   make([]byte, byteSize),
 		bitcount: make([]uint8, byteSize),
-		hashs: []func(data []byte) uint8{
-			func(data []byte) uint8 {
+		hashs: []func(data []rune) uint8{
+			func(data []rune) uint8 {
 				var hash uint32 = 0
 				for _, b := range data {
 					hash += uint32(b)
 				}
 				return uint8(hash % uint32(bitSize))
 			},
-			func(data []byte) uint8 {
+			func(data []rune) uint8 {
 				var hash uint32 = 0
 				for i, b := range data {
 					hash += uint32(b) * uint32(i+1)
 				}
 				return uint8(hash % uint32(byteSize))
 			},
-			func(data []byte) uint8 {
+			func(data []rune) uint8 {
 				var hash uint32 = 0
 				for i, b := range data {
 					hash += uint32(b) * uint32((i+1)*(i+1))
@@ -54,11 +50,15 @@ func init() {
 	var titles []string
 	database.GetDatabase().Model(models.ArticleDetail{}).Select("title").Find(&titles)
 	for _, title := range titles {
-		bf.Add([]byte(title))
+		bf.Add([]rune(title))
 	}
 }
 
-func (bf *BloomFilter) Add(data []byte) {
+func GetBloomFilter() *BloomFilter {
+	return bf
+}
+
+func (bf *BloomFilter) Add(data []rune) {
 	for _, h := range bf.hashs {
 		pos := h(data)
 		bytePos := pos / 8
@@ -70,7 +70,7 @@ func (bf *BloomFilter) Add(data []byte) {
 	}
 }
 
-func (bf *BloomFilter) Exists(data []byte) bool {
+func (bf *BloomFilter) Exists(data []rune) bool {
 	for _, h := range bf.hashs {
 		pos := h(data)
 		bytePos := pos / 8
@@ -82,7 +82,7 @@ func (bf *BloomFilter) Exists(data []byte) bool {
 	return true
 }
 
-func (bf *BloomFilter) Remove(data []byte) {
+func (bf *BloomFilter) Remove(data []rune) {
 	for _, h := range bf.hashs {
 		pos := h(data)
 		bytePos := pos / 8

pkg/bloom/bloom_test.go

@@ -0,0 +1,83 @@
+package bloom
+
+import "testing"
+
+func TestBloomFilter(t *testing.T) {
+	bf := GetBloomFilter()
+	test_data := [][]rune{
+		[]rune("test1"),
+		[]rune("test2"),
+		[]rune("test3"),
+	}
+	for _, data := range test_data {
+		bf.Add(data)
+		if !bf.Exists(data) {
+			t.Errorf("Expected %s to be in the bloom filter", string(data))
+		}
+		if bf.Exists([]rune("not_in_filter")) {
+			t.Error("Expected 'not_in_filter' to not be in the bloom filter")
+		}
+		if bf.Exists([]rune("1test")) {
+			t.Error("Expected '1test' to not be in the bloom filter")
+		}
+		if bf.Exists([]rune("2test")) {
+			t.Error("Expected '2test' to not be in the bloom filter")
+		}
+		if bf.Exists([]rune("3test")) {
+			t.Error("Expected '3test' to not be in the bloom filter")
+		}
+	}
+	for _, data := range test_data {
+		bf.Remove(data)
+		if bf.Exists(data) {
+			t.Errorf("Expected %s to be removed from the bloom filter", string(data))
+		}
+	}
+	if bf.Exists([]rune("test1")) || bf.Exists([]rune("test2")) || bf.Exists([]rune("test3")) {
+		t.Error("Expected all test data to be removed from the bloom filter")
+	}
+	if bf.Exists([]rune("not_in_filter")) {
+		t.Error("Expected 'not_in_filter' to not be in the bloom filter after removal")
+	}
+	if bf.Exists([]rune("1test")) {
+		t.Error("Expected '1test' to not be in the bloom filter after removal")
+	}
+	if bf.Exists([]rune("2test")) {
+		t.Error("Expected '2test' to not be in the bloom filter after removal")
+	}
+	if bf.Exists([]rune("3test")) {
+		t.Error("Expected '3test' to not be in the bloom filter after removal")
+	}
+	t.Log("Bloom filter test passed")
+}
+
+func TestBloomFilterChinese(t *testing.T) {
+	bf := GetBloomFilter()
+	test_data := [][]rune{
+		[]rune("测试1"),
+		[]rune("测试2"),
+		[]rune("测试3"),
+	}
+	for _, data := range test_data {
+		bf.Add(data)
+		if !bf.Exists(data) {
+			t.Errorf("Expected %s to be in the bloom filter", string(data))
+		}
+		if bf.Exists([]rune("不在过滤器中")) {
+			t.Error("Expected '不在过滤器中' to not be in the bloom filter")
+		}
+	}
+	for _, data := range test_data {
+		bf.Remove(data)
+		if bf.Exists(data) {
+			t.Errorf("Expected %s to be removed from the bloom filter", string(data))
+		}
+	}
+	if bf.Exists([]rune("测试1")) || bf.Exists([]rune("测试2")) || bf.Exists([]rune("测试3")) {
+		t.Error("Expected all test data to be removed from the bloom filter")
+	}
+	if bf.Exists([]rune("不在过滤器中")) {
+		t.Error("Expected '不在过滤器中' to not be in the bloom filter after removal")
+	}
+	t.Log("Bloom filter Chinese test passed")
+}

service/api/article.go

@@ -80,7 +80,7 @@ func UploadArticle(c *gin.Context) {
 		slog.Error("insert article summary failed ", "error: ", err)
 	}
 
-	bloom.GetBloomFilter().Add([]byte(article.Title))
+	bloom.GetBloomFilter().Add([]rune(article.Title))
 	c.JSON(200, gin.H{"msg": "successfully added article"})
 }
 
@@ -93,7 +93,7 @@ func GetArticleDetail(c *gin.Context) {
 	}
 
 	// 先检查布隆过滤器，快速判断文章是否存在，减少数据库压力
-	if !bloom.GetBloomFilter().Exists([]byte(title)) {
+	if !bloom.GetBloomFilter().Exists([]rune(title)) {
 		c.JSON(404, gin.H{"msg": "article not found"})
 		return
 	}
@@ -272,7 +272,7 @@ func DeleteArticle(c *gin.Context) {
 	}
 
 	// 先检查布隆过滤器，快速判断文章是否存在，减少数据库压力
-	if !bloom.GetBloomFilter().Exists([]byte(title)) {
+	if !bloom.GetBloomFilter().Exists([]rune(title)) {
 		c.JSON(404, gin.H{"msg": "article not found"})
 		return
 	}
@@ -289,6 +289,6 @@ func DeleteArticle(c *gin.Context) {
 		}
 		time.Sleep(10 << i * time.Millisecond)
 	}
-	bloom.GetBloomFilter().Remove([]byte(title))
+	bloom.GetBloomFilter().Remove([]rune(title))
 	c.JSON(200, gin.H{"msg": "successfully deleted article"})
 }