Skip to content

Commit 7b3b8a3

Browse files
motoki317motoki317
committed
feat: Implement --include file flag
1 parent 487f550 commit 7b3b8a3

8 files changed

Lines changed: 141 additions & 49 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ Flags:
8888
If specifying both file paths and contents ignore regexp, split them by ':'.
8989
Example (ignore dist directory): --ignore '^dist/'
9090
Example (ignore import statements in js files): --ignore '\.m?[jt]s$:^import'
91+
--include stringArray Regexp of file paths (and its contents) to include.
92+
If specified, only matching files will be considered.
93+
Example (include only src directory): --include '^src/'
9194
--log-level string Log level (debug, info, warn, error)
9295
--micro Splits query to detect micro-clones (has performance implications!)
9396
-r, --repo string Source git directory (supports bare)

‎cmd/root.go‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@ package cmd
33
import (
44
"context"
55
"fmt"
6-
"github.com/spf13/pflag"
7-
"github.com/spf13/viper"
8-
"golang.org/x/term"
96
"log/slog"
107
"os"
118
"path/filepath"
129
"strings"
1310
"time"
1411

12+
"github.com/spf13/pflag"
13+
"github.com/spf13/viper"
14+
"golang.org/x/term"
15+
1516
"github.com/go-git/go-git/v5"
1617
"github.com/go-git/go-git/v5/plumbing"
1718
"github.com/pkg/errors"
@@ -114,12 +115,15 @@ var (
114115
)
115116

116117
var (
117-
logLevel string
118-
algorithm string
119-
algorithmParam []string
118+
logLevel string
119+
algorithm string
120+
algorithmParam []string
121+
120122
ignoreCLIOptions []string
121123
disableDefaultIgnore bool
122-
detectMicro bool
124+
includeCLIOptions []string
125+
126+
detectMicro bool
123127
)
124128

125129
func searchConfig(repoDir string) (*search.Config, error) {
@@ -142,7 +146,7 @@ func searchConfig(repoDir string) (*search.Config, error) {
142146
}
143147

144148
return &search.Config{
145-
Ignore: ignore,
149+
Matcher: ignore,
146150
DetectMicro: detectMicro,
147151
AlgoParams: params,
148152
}, nil
@@ -169,11 +173,16 @@ Can accept special value "WORKTREE" to specify the current worktree.`)
169173
pfs.StringVar(&logLevel, "log-level", "", "Log level (debug, info, warn, error)")
170174
pfs.StringVar(&algorithm, "algorithm", "fleccs", "Clone search algorithm to use (fleccs, ncdsearch)")
171175
pfs.StringArrayVar(&algorithmParam, "algorithm-param", nil, "(Advanced) Parameters of the algorithm, consult code for syntax.")
176+
172177
pfs.StringArrayVar(&ignoreCLIOptions, "ignore", nil, `Regexp of file paths (and its contents) to ignore.
173178
If specifying both file paths and contents ignore regexp, split them by ':'.
174179
Example (ignore dist directory): --ignore '^dist/'
175180
Example (ignore import statements in js files): --ignore '\.m?[jt]s$:^import'`)
176181
pfs.BoolVar(&disableDefaultIgnore, "disable-default-ignore", false, "Disable default ignore configs")
182+
pfs.StringArrayVar(&includeCLIOptions, "include", nil, `Regexp of file paths (and its contents) to include.
183+
If specified, only matching files will be considered.
184+
Example (include only src directory): --include '^src/'`)
185+
177186
pfs.BoolVar(&detectMicro, "micro", false, "Splits query to detect micro-clones (has performance implications!)")
178187

179188
// Disable "completion" command
@@ -239,8 +248,8 @@ func setLogLevel() error {
239248
return nil
240249
}
241250

242-
func readIgnoreRules(repoDir string) (domain.IgnoreRules, error) {
243-
ignore, err := domain.ReadIgnoreRules(repoDir, ignoreCLIOptions, disableDefaultIgnore)
251+
func readIgnoreRules(repoDir string) (*domain.MatcherRules, error) {
252+
ignore, err := domain.ReadMatcherRules(repoDir, disableDefaultIgnore, ignoreCLIOptions, includeCLIOptions)
244253
if err != nil {
245254
return nil, errors.Wrapf(err, "reading ignore rules")
246255
}

‎pkg/domain/ignore.go‎

Lines changed: 50 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,21 @@ package domain
33
import (
44
"errors"
55
"fmt"
6-
"github.com/salab/iccheck/pkg/utils/ds"
7-
"github.com/salab/iccheck/pkg/utils/files"
8-
"gopkg.in/yaml.v3"
96
"os"
107
"path/filepath"
118
"regexp"
129
"sort"
1310
"strings"
11+
12+
"github.com/salab/iccheck/pkg/utils/ds"
13+
"github.com/salab/iccheck/pkg/utils/files"
14+
"github.com/samber/lo"
15+
"gopkg.in/yaml.v3"
1416
)
1517

1618
// defaultIgnoreConfigs lists rules (that are pretty much safe to assume) for some languages.
1719
// To contributors: Feel free to add more to this default config.
18-
var defaultIgnoreConfigs = IgnoreConfigs{
20+
var defaultIgnoreConfigs = []*IgnoreConfig{
1921
{
2022
Files: []string{"\\.go$"},
2123
Patterns: []string{
@@ -39,10 +41,10 @@ var defaultIgnoreConfigs = IgnoreConfigs{
3941
},
4042
}
4143

42-
func ReadIgnoreRules(repoDir string, cliOptions []string, disableDefault bool) (IgnoreRules, error) {
43-
allConfigs := make(IgnoreConfigs, 0)
44+
func ReadMatcherRules(repoDir string, disableDefault bool, ignoreCLIOptions, includeCLIOptions []string) (*MatcherRules, error) {
45+
var matchers MatcherConfigs
4446
if !disableDefault {
45-
allConfigs = append(allConfigs, defaultIgnoreConfigs...)
47+
matchers.ignores = append(matchers.ignores, defaultIgnoreConfigs...)
4648
}
4749

4850
// Check if ignore file is present in the following locations:
@@ -59,29 +61,28 @@ func ReadIgnoreRules(repoDir string, cliOptions []string, disableDefault bool) (
5961
}
6062
for _, path := range paths {
6163
if f, err := os.Open(path); err == nil {
62-
var configs IgnoreConfigs
64+
var configs []*IgnoreConfig
6365
if err = yaml.NewDecoder(f).Decode(&configs); err != nil {
6466
return nil, fmt.Errorf("decoding %s: %w", path, err)
6567
}
66-
allConfigs = append(allConfigs, configs...)
68+
matchers.ignores = append(matchers.ignores, configs...)
6769
}
6870
// Ignore os.Open() error - file might not exist
6971
}
7072

7173
// Parse CLI options, if any
72-
for _, cliOption := range cliOptions {
73-
config, err := readIgnoreCLIOption(cliOption)
74+
for _, ignoreCLIOption := range ignoreCLIOptions {
75+
config, err := readIgnoreCLIOption(ignoreCLIOption)
7476
if err != nil {
7577
return nil, err
7678
}
77-
allConfigs = append(allConfigs, config)
79+
matchers.ignores = append(matchers.ignores, config)
7880
}
81+
matchers.includes = includeCLIOptions
7982

80-
return allConfigs.Compile()
83+
return matchers.Compile()
8184
}
8285

83-
type IgnoreConfigs []*IgnoreConfig
84-
8586
type IgnoreConfig struct {
8687
Files []string `yaml:"files"`
8788
Patterns []string `yaml:"patterns"`
@@ -131,11 +132,30 @@ func (i *IgnoreConfig) Compile() (*IgnoreRule, error) {
131132
return &ret, nil
132133
}
133134

134-
func (i IgnoreConfigs) Compile() (IgnoreRules, error) {
135-
return ds.MapError(i, (*IgnoreConfig).Compile)
135+
type MatcherConfigs struct {
136+
includes []string
137+
ignores []*IgnoreConfig
136138
}
137139

138-
type IgnoreRules []*IgnoreRule
140+
func (m *MatcherConfigs) Compile() (*MatcherRules, error) {
141+
ignoreRules, err := ds.MapError(m.ignores, (*IgnoreConfig).Compile)
142+
if err != nil {
143+
return nil, err
144+
}
145+
includeFiles, err := ds.MapError(m.includes, regexp.Compile)
146+
if err != nil {
147+
return nil, err
148+
}
149+
return &MatcherRules{
150+
includeFiles: includeFiles,
151+
ignoreRules: ignoreRules,
152+
}, nil
153+
}
154+
155+
type MatcherRules struct {
156+
includeFiles []*regexp.Regexp
157+
ignoreRules []*IgnoreRule
158+
}
139159

140160
type IgnoreRule struct {
141161
files []*regexp.Regexp
@@ -184,10 +204,20 @@ func (i *IgnoreRule) matchContents(contents []byte) (ignoreLines map[int]struct{
184204
// If skipEntireFile is true, callers are expected to skip this entire file (ignoreRule is nil).
185205
// Otherwise, callers are expected to call IgnoreLineRule.CanSkip() method according to its doc to
186206
// check file ignore pattern.
187-
func (i IgnoreRules) Match(path string, contents []byte) (skipEntireFile bool, ignoreRule *IgnoreLineRule) {
207+
func (m *MatcherRules) Match(path string, contents []byte) (skipEntireFile bool, ignoreRule *IgnoreLineRule) {
208+
// Check if we have --include option specified, if so, only consider the matching files
209+
if len(m.includeFiles) > 0 {
210+
match := lo.SomeBy(m.includeFiles, func(r *regexp.Regexp) bool {
211+
return r.MatchString(path)
212+
})
213+
if !match {
214+
return true, nil
215+
}
216+
}
217+
188218
// Check if any patterns match the whole file first
189219
instances := make([]*IgnoreRule, 0)
190-
for _, instance := range i {
220+
for _, instance := range m.ignoreRules {
191221
matchFile := instance.matchFile(path)
192222
if !matchFile {
193223
continue

‎pkg/domain/ignore_test.go‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func main() {
178178
}
179179
`
180180

181-
i := IgnoreConfigs{
181+
i := []*IgnoreConfig{
182182
// Ignore whole file rule
183183
{
184184
Files: []string{"^dist/"},
@@ -197,7 +197,11 @@ func main() {
197197
},
198198
},
199199
}
200-
c, err := i.Compile()
200+
m := MatcherConfigs{
201+
includes: nil,
202+
ignores: i,
203+
}
204+
c, err := m.Compile()
201205
if err != nil {
202206
t.Fatal(err)
203207
}
@@ -245,3 +249,46 @@ func main() {
245249
}
246250
}
247251
}
252+
253+
func TestIncludeRules(t *testing.T) {
254+
const testContent = `package main
255+
256+
import "fmt"
257+
258+
import (
259+
"fmt"
260+
)
261+
262+
func main() {
263+
fmt.Println("Hello World")
264+
}
265+
`
266+
267+
m := MatcherConfigs{
268+
includes: []string{"^src/", "^test/"},
269+
}
270+
c, err := m.Compile()
271+
if err != nil {
272+
t.Fatal(err)
273+
}
274+
275+
tests := []struct {
276+
filename string
277+
want bool
278+
}{
279+
{"src/main.go", false},
280+
{"test/main.go", false},
281+
{"main.go", true},
282+
{"src/foo/main.go", false},
283+
{"test/foo/main.go", false},
284+
{"foo/main.go", true},
285+
}
286+
for _, tt := range tests {
287+
t.Run(tt.filename, func(t *testing.T) {
288+
skip, _ := c.Match(tt.filename, []byte(testContent))
289+
if skip != tt.want {
290+
t.Errorf("got %v, want %v", skip, tt.want)
291+
}
292+
})
293+
}
294+
}

‎pkg/fleccs/fleccs.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package fleccs
77

88
import (
99
"context"
10+
"runtime"
11+
1012
"github.com/cespare/xxhash"
1113
"github.com/pkg/errors"
1214
"github.com/salab/iccheck/pkg/domain"
@@ -15,7 +17,6 @@ import (
1517
"github.com/salab/iccheck/pkg/utils/strs"
1618
"github.com/samber/lo"
1719
"github.com/sourcegraph/conc/pool"
18-
"runtime"
1920
)
2021

2122
type Source struct {
@@ -168,7 +169,7 @@ func fileSearch(
168169
queries []*Query,
169170
searchTree domain.Searcher,
170171
searchFilename string,
171-
ignore domain.IgnoreRules,
172+
matcher *domain.MatcherRules,
172173
similarityThreshold float64,
173174
) ([]*Candidate, error) {
174175
if ctx.Err() != nil { // check for deadline
@@ -195,7 +196,7 @@ func fileSearch(
195196
}
196197

197198
// Check ignore settings
198-
skipEntireFile, ignoreRule := ignore.Match(searchFilename, fileContent)
199+
skipEntireFile, ignoreRule := matcher.Match(searchFilename, fileContent)
199200
if skipEntireFile {
200201
return nil, nil
201202
}
@@ -226,7 +227,7 @@ func Search(
226227
queriesTree domain.Searcher,
227228
queries []*Query,
228229
searchTree domain.Searcher,
229-
ignore domain.IgnoreRules,
230+
matcher *domain.MatcherRules,
230231
options ...ConfigFunc,
231232
) ([]*Candidate, error) {
232233
// Calculate config
@@ -255,7 +256,7 @@ func Search(
255256
WithFirstError()
256257
for _, searchFile := range searchFiles {
257258
p.Go(func(ctx context.Context) ([]*Candidate, error) {
258-
return fileSearch(ctx, queries, searchTree, searchFile, ignore, c.similarityThreshold)
259+
return fileSearch(ctx, queries, searchTree, searchFile, matcher, c.similarityThreshold)
259260
})
260261
}
261262
candidates, err := p.Wait()

‎pkg/ncdsearch/ncdsearch.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,16 @@ package ncdsearch
88
import (
99
"context"
1010
"fmt"
11+
"math"
12+
"runtime"
13+
"slices"
14+
"sort"
15+
1116
"github.com/pkg/errors"
1217
"github.com/salab/iccheck/pkg/domain"
1318
"github.com/salab/iccheck/pkg/utils/ds"
1419
"github.com/samber/lo"
1520
"github.com/sourcegraph/conc/pool"
16-
"math"
17-
"runtime"
18-
"slices"
19-
"sort"
2021

2122
"github.com/salab/iccheck/pkg/utils/files"
2223
"github.com/salab/iccheck/pkg/utils/strs"
@@ -149,7 +150,7 @@ func fileSearch(
149150
queries []*Query,
150151
searchTree domain.Searcher,
151152
searchFilename string,
152-
ignore domain.IgnoreRules,
153+
matcher *domain.MatcherRules,
153154
) ([]*Clone, error) {
154155
if ctx.Err() != nil { // check for deadline
155156
return nil, ctx.Err()
@@ -175,7 +176,7 @@ func fileSearch(
175176
}
176177

177178
// Check ignore settings
178-
skipEntireFile, ignoreRule := ignore.Match(searchFilename, fileContent)
179+
skipEntireFile, ignoreRule := matcher.Match(searchFilename, fileContent)
179180
if skipEntireFile {
180181
return nil, nil
181182
}
@@ -246,7 +247,7 @@ func Search(
246247
queriesTree domain.Searcher,
247248
queries []*Query,
248249
searchTree domain.Searcher,
249-
ignore domain.IgnoreRules,
250+
matcher *domain.MatcherRules,
250251
options ...ConfigFunc,
251252
) ([]*Clone, error) {
252253
c := applyConfig(options...)
@@ -273,7 +274,7 @@ func Search(
273274
WithFirstError()
274275
for _, searchFile := range searchFiles {
275276
p.Go(func(ctx context.Context) ([]*Clone, error) {
276-
return fileSearch(ctx, c, queries, searchTree, searchFile, ignore)
277+
return fileSearch(ctx, c, queries, searchTree, searchFile, matcher)
277278
})
278279
}
279280
results, err := p.Wait()

0 commit comments

Comments
 (0)