From e24047cdc7e67a31b31dc56b87f7576a2acb5a90 Mon Sep 17 00:00:00 2001 From: KirCute Date: Wed, 14 Jan 2026 19:32:42 +0800 Subject: [PATCH 01/11] feat(drivers): add autoindex driver --- drivers/all.go | 1 + drivers/autoindex/driver.go | 134 ++++++++++++++++++++++++++++++++++++ drivers/autoindex/meta.go | 28 ++++++++ go.mod | 3 + go.sum | 6 ++ 5 files changed, 172 insertions(+) create mode 100644 drivers/autoindex/driver.go create mode 100644 drivers/autoindex/meta.go diff --git a/drivers/all.go b/drivers/all.go index 15fbf2b96..7e1c24bba 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -17,6 +17,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex" _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go new file mode 100644 index 000000000..4247f6e4b --- /dev/null +++ b/drivers/autoindex/driver.go @@ -0,0 +1,134 @@ +package autoindex + +import ( + "context" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/antchfx/htmlquery" + "github.com/antchfx/xpath" + "github.com/pkg/errors" +) + +type Autoindex struct { + model.Storage + Addition + itemXPath *xpath.Expr + nameXPath *xpath.Expr + modifiedXPath *xpath.Expr + sizeXPath *xpath.Expr + ignores map[string]any +} + +func (d *Autoindex) Config() driver.Config { + return config +} + +func (d *Autoindex) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Autoindex) Init(ctx context.Context) error { + var err error + d.itemXPath, err = xpath.Compile(d.ItemXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Item XPath") + } + d.nameXPath, err = xpath.Compile(d.NameXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Name XPath") + } + d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Modified XPath") + } + d.sizeXPath, err = xpath.Compile(d.SizeXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Size XPath") + } + ignores := strings.Split(d.IgnoreFileNames, "\n") + for _, i := range ignores { + d.ignores[i] = struct{}{} + } + hasScheme := strings.Contains(d.URL, "://") + hasSuffix := strings.HasSuffix(d.URL, "/") + if !hasScheme || !hasSuffix { + if !hasSuffix { + d.URL = d.URL + "/" + } + if !hasScheme { + d.URL = "https://" + d.URL + } + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *Autoindex) Drop(ctx context.Context) error { + return nil +} + +func (d *Autoindex) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + Name: op.RootName, + Path: d.URL, + Modified: d.Modified, + Mask: model.Locked, + IsFolder: true, + }, nil +} + +func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + res, err := http.Get(dir.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) + } + defer res.Body.Close() + + doc, err := htmlquery.Parse(res.Body) + if err != nil { + return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) + } + items := htmlquery.QuerySelectorAll(doc, d.itemXPath) + objs := make([]model.Obj, 0, len(items)) + for _, item := range items { + nameElem := htmlquery.QuerySelector(item, d.nameXPath) + if nameElem == nil { + continue + } + nameFull := htmlquery.InnerText(nameElem) + name, isDir := strings.CutSuffix(nameFull, "/") + if _, ok := d.ignores[name]; ok { + continue + } + var size int64 = 0 + if sizeElem := htmlquery.QuerySelector(item, d.sizeXPath); sizeElem != nil { + size, _ = strconv.ParseInt(htmlquery.InnerText(sizeElem), 10, 64) + } + var modified time.Time + if modifiedElem := htmlquery.QuerySelector(item, d.modifiedXPath); modifiedElem != nil { + modified, _ = time.Parse(d.ModifiedTimeFormat, htmlquery.InnerText(modifiedElem)) + } + objs = append(objs, &model.Object{ + Name: name, + IsFolder: isDir, + Path: dir.GetPath() + nameFull, + Modified: modified, + Size: size, + }) + } + return objs, nil +} + +func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return &model.Link{ + URL: file.GetPath(), + }, nil +} + +var _ driver.Driver = (*Autoindex)(nil) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go new file mode 100644 index 000000000..bd5b56ebf --- /dev/null +++ b/drivers/autoindex/meta.go @@ -0,0 +1,28 @@ +package autoindex + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + URL string `json:"url" required:"true"` + ItemXPath string `json:"item_xpath" required:"true"` + NameXPath string `json:"name_xpath" required:"true"` + ModifiedXPath string `json:"modified_xpath" required:"true"` + SizeXPath string `json:"size_xpath" required:"true"` + IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory"` + ModifiedTimeFormat string `json:"modified_time_format" default:"2006-01-02 15:04"` +} + +var config = driver.Config{ + Name: "Autoindex", + LocalSort: true, + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Autoindex{} + }) +} diff --git a/go.mod b/go.mod index 82a778d02..dfc39e34b 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,8 @@ require ( github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/antchfx/htmlquery v1.3.5 // indirect + github.com/antchfx/xpath v1.3.5 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect @@ -104,6 +106,7 @@ require ( github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 029e5d514..17c5ec462 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,10 @@ github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9 github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= +github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= +github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= +github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -347,6 +351,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= From 4074a420047324baa9385d8721c0963b9d680e7d Mon Sep 17 00:00:00 2001 From: KirCute Date: Thu, 15 Jan 2026 15:59:36 +0800 Subject: [PATCH 02/11] fix --- drivers/autoindex/driver.go | 20 +++++++++---- drivers/autoindex/util.go | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 drivers/autoindex/util.go diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 4247f6e4b..933a4b4d4 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -3,7 +3,6 @@ package autoindex import ( "context" "net/http" - "strconv" "strings" "time" @@ -52,7 +51,12 @@ func (d *Autoindex) Init(ctx context.Context) error { return errors.WithMessage(err, "failed to compile Size XPath") } ignores := strings.Split(d.IgnoreFileNames, "\n") + d.ignores = make(map[string]any, len(ignores)) for _, i := range ignores { + i = strings.TrimSpace(i) + if len(i) == 0 { + continue + } d.ignores[i] = struct{}{} } hasScheme := strings.Contains(d.URL, "://") @@ -101,18 +105,18 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if nameElem == nil { continue } - nameFull := htmlquery.InnerText(nameElem) + nameFull := strings.TrimSpace(htmlquery.InnerText(nameElem)) name, isDir := strings.CutSuffix(nameFull, "/") if _, ok := d.ignores[name]; ok { continue } var size int64 = 0 if sizeElem := htmlquery.QuerySelector(item, d.sizeXPath); sizeElem != nil { - size, _ = strconv.ParseInt(htmlquery.InnerText(sizeElem), 10, 64) + size = ParseSize(htmlquery.InnerText(sizeElem)) } var modified time.Time if modifiedElem := htmlquery.QuerySelector(item, d.modifiedXPath); modifiedElem != nil { - modified, _ = time.Parse(d.ModifiedTimeFormat, htmlquery.InnerText(modifiedElem)) + modified, _ = time.Parse(d.ModifiedTimeFormat, strings.TrimSpace(htmlquery.InnerText(modifiedElem))) } objs = append(objs, &model.Object{ Name: name, @@ -126,8 +130,14 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + res, err := http.Head(file.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) + } + _ = res.Body.Close() return &model.Link{ - URL: file.GetPath(), + URL: file.GetPath(), + ContentLength: res.ContentLength, }, nil } diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go new file mode 100644 index 000000000..2d0a5754e --- /dev/null +++ b/drivers/autoindex/util.go @@ -0,0 +1,60 @@ +package autoindex + +import ( + "strconv" + "strings" +) + +var units = map[string]int64{ + "": 1, + "b": 1, + "byte": 1, + "bytes": 1, + "k": 1 << 10, + "kb": 1 << 10, + "kib": 1 << 10, + "m": 1 << 20, + "mb": 1 << 20, + "mib": 1 << 20, + "g": 1 << 30, + "gb": 1 << 30, + "gib": 1 << 30, + "t": 1 << 40, + "tb": 1 << 40, + "tib": 1 << 40, + "p": 1 << 50, + "pb": 1 << 50, + "pib": 1 << 50, +} + +func splitUnit(s string) (string, string) { + for i := len(s) - 1; i >= 0; i-- { + if s[i] >= '0' && s[i] <= '9' { + return s[:i+1], s[i+1:] + } + } + return "", s +} + +func ParseSize(s string) int64 { + s = strings.TrimSpace(s) + if s == "-" { + return 0 + } + nbs, unit := splitUnit(s) + mul, ok := units[strings.ToLower(unit)] + if !ok { + mul = 1 + } + nb, err := strconv.ParseInt(nbs, 10, 64) + if err != nil { + fnb, err := strconv.ParseFloat(nbs, 64) + if err != nil { + return 0 + } + nb = int64(fnb * float64(mul)) + } else { + nb = nb * mul + } + return nb +} From 5736e156367b3c0d823b787bb4d7251a1941aafd Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 15 Jan 2026 16:43:19 +0800 Subject: [PATCH 03/11] add NoUpload Signed-off-by: MadDogOwner --- drivers/autoindex/meta.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go index bd5b56ebf..ed4c9ff4a 100644 --- a/drivers/autoindex/meta.go +++ b/drivers/autoindex/meta.go @@ -19,6 +19,7 @@ var config = driver.Config{ Name: "Autoindex", LocalSort: true, CheckStatus: true, + NoUpload: true, } func init() { From 6f2b439affeeea2024e02bba6cac83159895a7a7 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 15 Jan 2026 16:47:54 +0800 Subject: [PATCH 04/11] add TestParseSize Signed-off-by: MadDogOwner --- drivers/autoindex/util_test.go | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 drivers/autoindex/util_test.go diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go new file mode 100644 index 000000000..a5390a9ac --- /dev/null +++ b/drivers/autoindex/util_test.go @@ -0,0 +1,43 @@ +package autoindex + +import ( + "testing" +) + +func TestParseSize(t *testing.T) { + tests := []struct { + input string + want int64 + }{ + {"100", 100}, + {"1k", 1024}, + {"1kb", 1024}, + {"1K", 1024}, // case insensitive + {"1.5m", 1572864}, // 1.5 * 1024^2 + {"500 bytes", 500}, + {"-", 0}, + {"", 0}, + {"abc", 0}, + {"1.5GB", 1610612736}, // 1.5 * 1024^3 + {"2t", 2199023255552}, // 2 * 1024^4 + {"1p", 1125899906842624}, // 1 * 1024^5 + {"0", 0}, + {" 100 ", 100}, // trimmed + {"100b", 100}, + {"1gib", 1073741824}, // 1024^3 + {"1z", 1}, // invalid unit, mul=1 + {"1.5", 1}, // float without unit, truncated + {"2.7k", 2764}, // 2.7 * 1024 truncated + {"1.0g", 1073741824}, // 1.0 * 1024^3 + {"invalid", 0}, + {"123xyz", 123}, // unit not found, mul=1 + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got := ParseSize(tt.input) + if got != tt.want { + t.Errorf("ParseSize(%q) = %d, want %d", tt.input, got, tt.want) + } + }) + } +} From 0389e917a42cf9b0bcf379488bf561ed65e06b1d Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 15 Jan 2026 18:16:40 +0800 Subject: [PATCH 05/11] go mod tidy Signed-off-by: MadDogOwner --- go.mod | 4 ++-- go.sum | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index dfc39e34b..e751bcfe0 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,8 @@ require ( github.com/ProtonMail/gopenpgp/v2 v2.9.0 github.com/SheltonZhu/115driver v1.1.1 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/antchfx/htmlquery v1.3.5 + github.com/antchfx/xpath v1.3.5 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.7 github.com/blevesearch/bleve/v2 v2.5.2 @@ -94,8 +96,6 @@ require ( github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect - github.com/antchfx/htmlquery v1.3.5 // indirect - github.com/antchfx/xpath v1.3.5 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect diff --git a/go.sum b/go.sum index 17c5ec462..57964be4c 100644 --- a/go.sum +++ b/go.sum @@ -404,8 +404,6 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499 h1:4ovnBdiGDFi8putQGxhipuuhXItAgh4/YnzufPYkZkQ= -github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= From 977644567c4751276ca6b9d5a8507bc6e4897df9 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 15 Jan 2026 18:51:44 +0800 Subject: [PATCH 06/11] use base.RestyClient Signed-off-by: MadDogOwner --- drivers/autoindex/driver.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 933a4b4d4..f5c4b97be 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -2,10 +2,10 @@ package autoindex import ( "context" - "net/http" "strings" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -88,13 +88,15 @@ func (d *Autoindex) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - res, err := http.Get(dir.GetPath()) + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Get(dir.GetPath()) if err != nil { return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) } - defer res.Body.Close() - - doc, err := htmlquery.Parse(res.Body) + defer res.RawResponse.Body.Close() + doc, err := htmlquery.Parse(res.RawBody()) if err != nil { return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) } @@ -130,14 +132,17 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - res, err := http.Head(file.GetPath()) + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Head(file.GetPath()) if err != nil { return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) } - _ = res.Body.Close() + _ = res.RawResponse.Body.Close() return &model.Link{ URL: file.GetPath(), - ContentLength: res.ContentLength, + ContentLength: res.RawResponse.ContentLength, }, nil } From 31dab61bb62018b4398ff5edf32c98ba5b85384d Mon Sep 17 00:00:00 2001 From: KirCute Date: Thu, 15 Jan 2026 19:58:22 +0800 Subject: [PATCH 07/11] fix: support evaluate size and modified time --- drivers/autoindex/driver.go | 27 +++++++++-------- drivers/autoindex/util.go | 54 +++++++++++++++++++++++++++++++--- drivers/autoindex/util_test.go | 2 +- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index f5c4b97be..9bee6e7ae 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -3,7 +3,6 @@ package autoindex import ( "context" "strings" - "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -12,6 +11,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/antchfx/xpath" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type Autoindex struct { @@ -100,25 +100,26 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if err != nil { return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) } - items := htmlquery.QuerySelectorAll(doc, d.itemXPath) - objs := make([]model.Obj, 0, len(items)) - for _, item := range items { - nameElem := htmlquery.QuerySelector(item, d.nameXPath) - if nameElem == nil { + itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) + var objs []model.Obj + for itemsIter.MoveNext() { + nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current())) + if err != nil { + log.Warnf("skip invalid name evaluating result: %v", err) continue } - nameFull := strings.TrimSpace(htmlquery.InnerText(nameElem)) + nameFull = strings.TrimSpace(nameFull) name, isDir := strings.CutSuffix(nameFull, "/") if _, ok := d.ignores[name]; ok { continue } - var size int64 = 0 - if sizeElem := htmlquery.QuerySelector(item, d.sizeXPath); sizeElem != nil { - size = ParseSize(htmlquery.InnerText(sizeElem)) + size, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current())) + if err != nil { + log.Errorf("failed to parse size of %s: %v", name, err) } - var modified time.Time - if modifiedElem := htmlquery.QuerySelector(item, d.modifiedXPath); modifiedElem != nil { - modified, _ = time.Parse(d.ModifiedTimeFormat, strings.TrimSpace(htmlquery.InnerText(modifiedElem))) + modified, err := parseTime(d.modifiedXPath.Evaluate(itemsIter.Current()), d.ModifiedTimeFormat) + if err != nil { + log.Errorf("failed to parse modified time of %s: %v", name, err) } objs = append(objs, &model.Object{ Name: name, diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go index 2d0a5754e..a2a7d3e13 100644 --- a/drivers/autoindex/util.go +++ b/drivers/autoindex/util.go @@ -1,8 +1,13 @@ package autoindex import ( + "fmt" "strconv" "strings" + "time" + + "github.com/antchfx/xpath" + "github.com/pkg/errors" ) var units = map[string]int64{ @@ -36,10 +41,17 @@ func splitUnit(s string) (string, string) { return "", s } -func ParseSize(s string) int64 { +func parseSize(a any) (int64, error) { + if f, ok := a.(float64); ok { + return int64(f), nil + } + s, err := parseString(a) + if err != nil { + return 0, err + } s = strings.TrimSpace(s) if s == "-" { - return 0 + return 0, nil } nbs, unit := splitUnit(s) mul, ok := units[strings.ToLower(unit)] @@ -50,11 +62,45 @@ func ParseSize(s string) int64 { if err != nil { fnb, err := strconv.ParseFloat(nbs, 64) if err != nil { - return 0 + return 0, fmt.Errorf("failed to convert %s to number", nbs) } nb = int64(fnb * float64(mul)) } else { nb = nb * mul } - return nb + return nb, nil +} + +func parseString(res any) (string, error) { + if r, ok := res.(string); ok { + if len(r) == 0 { + return "", fmt.Errorf("empty result") + } + return r, nil + } + n, ok := res.(*xpath.NodeIterator) + if !ok { + return "", fmt.Errorf("unsupported evaluating result") + } + if !n.MoveNext() { + return "", fmt.Errorf("no matched nodes") + } + ns := n.Current().Value() + if len(ns) == 0 { + return "", fmt.Errorf("empty result") + } + return ns, nil +} + +func parseTime(res any, format string) (time.Time, error) { + s, err := parseString(res) + if err != nil { + return time.Now(), err + } + s = strings.TrimSpace(s) + t, err := time.Parse(format, s) + if err != nil { + return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s) + } + return t, nil } diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go index a5390a9ac..c492552d1 100644 --- a/drivers/autoindex/util_test.go +++ b/drivers/autoindex/util_test.go @@ -34,7 +34,7 @@ func TestParseSize(t *testing.T) { } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got := ParseSize(tt.input) + got, _ := parseSize(tt.input) if got != tt.want { t.Errorf("ParseSize(%q) = %d, want %d", tt.input, got, tt.want) } From efa2ee49d540e08a4a667619657ac3268fc11384 Mon Sep 17 00:00:00 2001 From: KirCute Date: Thu, 15 Jan 2026 20:11:09 +0800 Subject: [PATCH 08/11] fix apache --- drivers/autoindex/driver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 9bee6e7ae..5f7f9bacb 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -103,7 +103,7 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) var objs []model.Obj for itemsIter.MoveNext() { - nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current())) + nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy())) if err != nil { log.Warnf("skip invalid name evaluating result: %v", err) continue @@ -113,11 +113,11 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if _, ok := d.ignores[name]; ok { continue } - size, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current())) + size, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) if err != nil { log.Errorf("failed to parse size of %s: %v", name, err) } - modified, err := parseTime(d.modifiedXPath.Evaluate(itemsIter.Current()), d.ModifiedTimeFormat) + modified, err := parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) if err != nil { log.Errorf("failed to parse modified time of %s: %v", name, err) } From 6a9ea3210c7f10f6a70b9b040762c4cec78c7422 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 15 Jan 2026 21:27:14 +0800 Subject: [PATCH 09/11] perf --- drivers/autoindex/driver.go | 13 ++++++-- drivers/autoindex/types.go | 13 ++++++++ drivers/autoindex/util.go | 28 ++++++++++------ drivers/autoindex/util_test.go | 58 +++++++++++++++++++--------------- 4 files changed, 74 insertions(+), 38 deletions(-) create mode 100644 drivers/autoindex/types.go diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 5f7f9bacb..6da34420a 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -113,7 +113,7 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if _, ok := d.ignores[name]; ok { continue } - size, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) + size, exact, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) if err != nil { log.Errorf("failed to parse size of %s: %v", name, err) } @@ -121,18 +121,25 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if err != nil { log.Errorf("failed to parse modified time of %s: %v", name, err) } - objs = append(objs, &model.Object{ + var o model.Obj = &model.Object{ Name: name, IsFolder: isDir, Path: dir.GetPath() + nameFull, Modified: modified, Size: size, - }) + } + if exact { + o = &exactSizeObj{Obj: o} + } + objs = append(objs, o) } return objs, nil } func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if _, ok := file.(*exactSizeObj); ok || args.Redirect { + return &model.Link{URL: file.GetPath()}, nil + } res, err := base.RestyClient.R(). SetContext(ctx). SetDoNotParseResponse(true). diff --git a/drivers/autoindex/types.go b/drivers/autoindex/types.go new file mode 100644 index 000000000..48ec396f5 --- /dev/null +++ b/drivers/autoindex/types.go @@ -0,0 +1,13 @@ +package autoindex + +import ( + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var ( + errEmptyEvaluateResult = fmt.Errorf("empty result") +) + +type exactSizeObj struct{ model.Obj } diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go index a2a7d3e13..f5f36a736 100644 --- a/drivers/autoindex/util.go +++ b/drivers/autoindex/util.go @@ -35,46 +35,56 @@ var units = map[string]int64{ func splitUnit(s string) (string, string) { for i := len(s) - 1; i >= 0; i-- { if s[i] >= '0' && s[i] <= '9' { - return s[:i+1], s[i+1:] + return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:]) } } return "", s } -func parseSize(a any) (int64, error) { +func parseSize(a any) (int64, bool, error) { + // 第二个返回值exact表示大小是否精确 if f, ok := a.(float64); ok { - return int64(f), nil + return int64(f), false, nil } s, err := parseString(a) + if errors.Is(err, errEmptyEvaluateResult) { + // 可能是错误，也可能确实大小为0 + // 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失 + // 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好 + return 0, false, nil + } if err != nil { - return 0, err + return 0, false, err } s = strings.TrimSpace(s) if s == "-" { - return 0, nil + return 0, false, nil } nbs, unit := splitUnit(s) mul, ok := units[strings.ToLower(unit)] + exact := mul == 1 if !ok { mul = 1 + // 推测无单位，exact应为false } nb, err := strconv.ParseInt(nbs, 10, 64) if err != nil { fnb, err := strconv.ParseFloat(nbs, 64) if err != nil { - return 0, fmt.Errorf("failed to convert %s to number", nbs) + return 0, false, fmt.Errorf("failed to convert %s to number", nbs) } nb = int64(fnb * float64(mul)) + exact = false } else { nb = nb * mul } - return nb, nil + return nb, exact, nil } func parseString(res any) (string, error) { if r, ok := res.(string); ok { if len(r) == 0 { - return "", fmt.Errorf("empty result") + return "", errEmptyEvaluateResult } return r, nil } @@ -87,7 +97,7 @@ func parseString(res any) (string, error) { } ns := n.Current().Value() if len(ns) == 0 { - return "", fmt.Errorf("empty result") + return "", errEmptyEvaluateResult } return ns, nil } diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go index c492552d1..ba743b943 100644 --- a/drivers/autoindex/util_test.go +++ b/drivers/autoindex/util_test.go @@ -4,39 +4,45 @@ import ( "testing" ) +type wantType struct { + v int64 + exact bool + error bool +} + func TestParseSize(t *testing.T) { tests := []struct { input string - want int64 + want wantType }{ - {"100", 100}, - {"1k", 1024}, - {"1kb", 1024}, - {"1K", 1024}, // case insensitive - {"1.5m", 1572864}, // 1.5 * 1024^2 - {"500 bytes", 500}, - {"-", 0}, - {"", 0}, - {"abc", 0}, - {"1.5GB", 1610612736}, // 1.5 * 1024^3 - {"2t", 2199023255552}, // 2 * 1024^4 - {"1p", 1125899906842624}, // 1 * 1024^5 - {"0", 0}, - {" 100 ", 100}, // trimmed - {"100b", 100}, - {"1gib", 1073741824}, // 1024^3 - {"1z", 1}, // invalid unit, mul=1 - {"1.5", 1}, // float without unit, truncated - {"2.7k", 2764}, // 2.7 * 1024 truncated - {"1.0g", 1073741824}, // 1.0 * 1024^3 - {"invalid", 0}, - {"123xyz", 123}, // unit not found, mul=1 + {"100", wantType{100, true, false}}, + {"1k", wantType{1024, false, false}}, + {"1kb", wantType{1024, false, false}}, + {"1K", wantType{1024, false, false}}, // case insensitive + {"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2 + {"500 bytes", wantType{500, true, false}}, + {"-", wantType{0, false, false}}, + {"", wantType{0, false, false}}, + {"abc", wantType{0, false, true}}, + {"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3 + {"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4 + {"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5 + {"0", wantType{0, true, false}}, + {" 100 ", wantType{100, true, false}}, // trimmed + {"100b", wantType{100, true, false}}, + {"1gib", wantType{1073741824, false, false}}, // 1024^3 + {"1z", wantType{1, false, false}}, // invalid unit, mul=1 + {"1.5", wantType{1, false, false}}, // float without unit, truncated + {"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated + {"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3 + {"invalid", wantType{0, false, true}}, + {"123xyz", wantType{123, false, false}}, // unit not found, mul=1 } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got, _ := parseSize(tt.input) - if got != tt.want { - t.Errorf("ParseSize(%q) = %d, want %d", tt.input, got, tt.want) + got, exact, err := parseSize(tt.input) + if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error { + t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error) } }) } From e50c08dd765c2355f7627036d433990a6f459dec Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Fri, 16 Jan 2026 00:17:54 +0800 Subject: [PATCH 10/11] feat: support ignore size and modified time --- drivers/autoindex/driver.go | 36 ++++++++++++++++++++++++------------ drivers/autoindex/meta.go | 8 ++++---- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 6da34420a..5ff45775e 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -3,6 +3,7 @@ package autoindex import ( "context" "strings" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -42,13 +43,17 @@ func (d *Autoindex) Init(ctx context.Context) error { if err != nil { return errors.WithMessage(err, "failed to compile Name XPath") } - d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) - if err != nil { - return errors.WithMessage(err, "failed to compile Modified XPath") + if len(d.ModifiedXPath) > 0 { + d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Modified XPath") + } } - d.sizeXPath, err = xpath.Compile(d.SizeXPath) - if err != nil { - return errors.WithMessage(err, "failed to compile Size XPath") + if len(d.SizeXPath) > 0 { + d.sizeXPath, err = xpath.Compile(d.SizeXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Size XPath") + } } ignores := strings.Split(d.IgnoreFileNames, "\n") d.ignores = make(map[string]any, len(ignores)) @@ -113,13 +118,20 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs if _, ok := d.ignores[name]; ok { continue } - size, exact, err := parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) - if err != nil { - log.Errorf("failed to parse size of %s: %v", name, err) + var size int64 = 0 + exact := false + modified := time.Now() + if d.sizeXPath != nil { + size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Errorf("failed to parse size of %s: %v", name, err) + } } - modified, err := parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) - if err != nil { - log.Errorf("failed to parse modified time of %s: %v", name, err) + if d.modifiedXPath != nil { + modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) + if err != nil { + log.Errorf("failed to parse modified time of %s: %v", name, err) + } } var o model.Obj = &model.Object{ Name: name, diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go index ed4c9ff4a..7b2dde318 100644 --- a/drivers/autoindex/meta.go +++ b/drivers/autoindex/meta.go @@ -9,10 +9,10 @@ type Addition struct { URL string `json:"url" required:"true"` ItemXPath string `json:"item_xpath" required:"true"` NameXPath string `json:"name_xpath" required:"true"` - ModifiedXPath string `json:"modified_xpath" required:"true"` - SizeXPath string `json:"size_xpath" required:"true"` - IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory"` - ModifiedTimeFormat string `json:"modified_time_format" default:"2006-01-02 15:04"` + ModifiedXPath string `json:"modified_xpath"` + SizeXPath string `json:"size_xpath"` + IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` + ModifiedTimeFormat string `json:"modified_time_format" default:"2006-01-02 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` } var config = driver.Config{ From 4d272f185abde845546de377e7979705db32784d Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Fri, 16 Jan 2026 16:29:27 +0800 Subject: [PATCH 11/11] rename driver --- drivers/autoindex/driver.go | 18 +++++++++--------- drivers/autoindex/meta.go | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go index 5ff45775e..5758ec6cd 100644 --- a/drivers/autoindex/driver.go +++ b/drivers/autoindex/driver.go @@ -15,7 +15,7 @@ import ( log "github.com/sirupsen/logrus" ) -type Autoindex struct { +type AutoIndex struct { model.Storage Addition itemXPath *xpath.Expr @@ -25,15 +25,15 @@ type Autoindex struct { ignores map[string]any } -func (d *Autoindex) Config() driver.Config { +func (d *AutoIndex) Config() driver.Config { return config } -func (d *Autoindex) GetAddition() driver.Additional { +func (d *AutoIndex) GetAddition() driver.Additional { return &d.Addition } -func (d *Autoindex) Init(ctx context.Context) error { +func (d *AutoIndex) Init(ctx context.Context) error { var err error d.itemXPath, err = xpath.Compile(d.ItemXPath) if err != nil { @@ -78,11 +78,11 @@ func (d *Autoindex) Init(ctx context.Context) error { return nil } -func (d *Autoindex) Drop(ctx context.Context) error { +func (d *AutoIndex) Drop(ctx context.Context) error { return nil } -func (d *Autoindex) GetRoot(ctx context.Context) (model.Obj, error) { +func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) { return &model.Object{ Name: op.RootName, Path: d.URL, @@ -92,7 +92,7 @@ func (d *Autoindex) GetRoot(ctx context.Context) (model.Obj, error) { }, nil } -func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { +func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { res, err := base.RestyClient.R(). SetContext(ctx). SetDoNotParseResponse(true). @@ -148,7 +148,7 @@ func (d *Autoindex) List(ctx context.Context, dir model.Obj, args model.ListArgs return objs, nil } -func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if _, ok := file.(*exactSizeObj); ok || args.Redirect { return &model.Link{URL: file.GetPath()}, nil } @@ -166,4 +166,4 @@ func (d *Autoindex) Link(ctx context.Context, file model.Obj, args model.LinkArg }, nil } -var _ driver.Driver = (*Autoindex)(nil) +var _ driver.Driver = (*AutoIndex)(nil) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go index 7b2dde318..8ebae5924 100644 --- a/drivers/autoindex/meta.go +++ b/drivers/autoindex/meta.go @@ -12,11 +12,11 @@ type Addition struct { ModifiedXPath string `json:"modified_xpath"` SizeXPath string `json:"size_xpath"` IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` - ModifiedTimeFormat string `json:"modified_time_format" default:"2006-01-02 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` + ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` } var config = driver.Config{ - Name: "Autoindex", + Name: "AutoIndex", LocalSort: true, CheckStatus: true, NoUpload: true, @@ -24,6 +24,6 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &Autoindex{} + return &AutoIndex{} }) }