Skip to content

Commit 0c16c2a

Browse files
foresturquhartforesturquhart
authored
Merge pull request #30 from foresturquhart/individual-file-token-counts
Warn when individual files have an especially high token count
2 parents 4a9dba5 + 91586ce commit 0c16c2a

14 files changed

Lines changed: 157 additions & 129 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ grimoire --redact-secrets -o output.md .
5555
* **Git Integration:** Prioritizes files by commit frequency when working within a Git repository.
5656
* **Secret Detection:** Scans files for potential secrets or sensitive information to prevent accidental exposure.
5757
* **Secret Redaction:** Optionally redacts detected secrets in the output while preserving the overall code structure.
58-
* **Token Counting:** Estimates the token count of generated output to help manage LLM context limits.
58+
* **Token Counting:** Calculates the token count of generated output to help manage LLM context limits.
5959
* **Minified File Detection:** Automatically identifies minified JavaScript and CSS files to warn about high token usage.
6060
* **Flexible Output:** Supports output to stdout or a specified file.
6161

@@ -145,7 +145,6 @@ grimoire [options] <target directory>
145145
- `--ignore-secrets`: Proceed with output generation even if secrets are detected.
146146
- `--redact-secrets`: Redact detected secrets in output rather than failing.
147147
- `--skip-token-count`: Skip counting output tokens.
148-
- `--token-count-mode <mode>`: Token counting mode: `fast` (default) or `exact` (slower but more accurate).
149148
- `--version`: Display the current version.
150149

151150
### Examples
@@ -174,10 +173,6 @@ grimoire [options] <target directory>
174173
```bash
175174
grimoire --redact-secrets -o output.md ./myproject
176175
```
177-
7. Use exact token counting mode:
178-
```bash
179-
grimoire --token-count-mode exact -o output.md ./myproject
180-
```
181176

182177
## Configuration
183178

@@ -223,14 +218,7 @@ Each format includes metadata, a summary section, an optional directory tree, an
223218

224219
## Token Counting
225220

226-
Grimoire includes built-in token counting to help you manage LLM context limits. The token count is estimated using the same tokenizer used by many LLMs.
227-
228-
Two token counting modes are available:
229-
230-
1. **Fast Mode** (default): Counts tokens in chunks as the output is generated. Slightly less accurate but more efficient for large codebases.
231-
2. **Exact Mode**: Counts all tokens in the entire output at once. More accurate but can be slower and more memory-intensive for large outputs.
232-
233-
You can disable token counting entirely using the `--skip-token-count` flag.
221+
Grimoire includes built-in token counting to help you manage LLM context limits. The token count is estimated using the same tokenizer used by many LLMs. You can disable token counting entirely using the `--skip-token-count` flag.
234222

235223
## Secret Detection
236224

‎cmd/grimoire/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ func main() {
6363
Usage: "Output format (md, xml, or txt). Defaults to md.",
6464
Value: "md",
6565
},
66+
&cli.IntFlag{
67+
Name: "high-token-threshold",
68+
Usage: "Threshold for warning about files with high token counts. Defaults to 5000.",
69+
Value: 5000,
70+
},
6671
},
6772
Action: func(ctx context.Context, cmd *cli.Command) error {
6873
return core.Run(

‎go.mod‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,25 @@ module github.com/foresturquhart/grimoire
33
go 1.24.3
44

55
require (
6+
github.com/BurntSushi/toml v1.5.0
67
github.com/rs/zerolog v1.34.0
78
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
9+
github.com/tiktoken-go/tokenizer v0.6.2
10+
github.com/urfave/cli/v3 v3.3.3
11+
github.com/zricethezav/gitleaks/v8 v8.26.0
12+
golang.org/x/text v0.25.0
813
)
914

1015
require (
1116
dario.cat/mergo v1.0.1 // indirect
1217
github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
1318
github.com/Masterminds/goutils v1.1.1 // indirect
14-
github.com/Masterminds/semver/v3 v3.3.1 // indirect
19+
github.com/Masterminds/semver/v3 v3.3.0 // indirect
1520
github.com/Masterminds/sprig/v3 v3.3.0 // indirect
1621
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
17-
github.com/charmbracelet/lipgloss v1.0.0 // indirect
18-
github.com/charmbracelet/x/ansi v0.8.0 // indirect
22+
github.com/charmbracelet/lipgloss v0.5.0 // indirect
1923
github.com/dlclark/regexp2 v1.11.5 // indirect
20-
github.com/fatih/semgroup v1.3.0 // indirect
24+
github.com/fatih/semgroup v1.2.0 // indirect
2125
github.com/fsnotify/fsnotify v1.8.0 // indirect
2226
github.com/gitleaks/go-gitdiff v0.9.1 // indirect
2327
github.com/google/uuid v1.6.0 // indirect
@@ -28,13 +32,14 @@ require (
2832
github.com/magiconair/properties v1.8.9 // indirect
2933
github.com/mattn/go-colorable v0.1.14 // indirect
3034
github.com/mattn/go-isatty v0.0.20 // indirect
31-
github.com/mattn/go-runewidth v0.0.16 // indirect
35+
github.com/mattn/go-runewidth v0.0.14 // indirect
3236
github.com/mitchellh/copystructure v1.2.0 // indirect
3337
github.com/mitchellh/mapstructure v1.5.0 // indirect
3438
github.com/mitchellh/reflectwalk v1.0.2 // indirect
35-
github.com/muesli/termenv v0.16.0 // indirect
39+
github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect
40+
github.com/muesli/termenv v0.15.1 // indirect
3641
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
37-
github.com/rivo/uniseg v0.4.7 // indirect
42+
github.com/rivo/uniseg v0.2.0 // indirect
3843
github.com/sagikazarmark/locafero v0.7.0 // indirect
3944
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
4045
github.com/shopspring/decimal v1.4.0 // indirect
@@ -46,20 +51,12 @@ require (
4651
github.com/subosito/gotenv v1.6.0 // indirect
4752
github.com/tetratelabs/wazero v1.9.0 // indirect
4853
github.com/wasilibs/go-re2 v1.9.0 // indirect
49-
github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect
54+
github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
5055
go.uber.org/multierr v1.11.0 // indirect
51-
golang.org/x/crypto v0.36.0 // indirect
52-
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
56+
golang.org/x/crypto v0.32.0 // indirect
57+
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect
5358
golang.org/x/sync v0.14.0 // indirect
54-
golang.org/x/sys v0.31.0 // indirect
59+
golang.org/x/sys v0.30.0 // indirect
5560
gopkg.in/ini.v1 v1.67.0 // indirect
5661
gopkg.in/yaml.v3 v3.0.1 // indirect
5762
)
58-
59-
require (
60-
github.com/BurntSushi/toml v1.5.0
61-
github.com/tiktoken-go/tokenizer v0.6.2
62-
github.com/urfave/cli/v3 v3.3.3
63-
github.com/zricethezav/gitleaks/v8 v8.26.0
64-
golang.org/x/text v0.25.0
65-
)

‎go.sum‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,23 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg
66
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
77
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
88
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
9-
github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
10-
github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
9+
github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0=
10+
github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
1111
github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs=
1212
github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
1313
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
1414
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
15-
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
16-
github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
17-
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
18-
github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
15+
github.com/charmbracelet/lipgloss v0.5.0 h1:lulQHuVeodSgDez+3rGiuxlPVXSnhth442DATR2/8t8=
16+
github.com/charmbracelet/lipgloss v0.5.0/go.mod h1:EZLha/HbzEt7cYqdFPovlqy5FZPj0xFhg5SaqxScmgs=
1917
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
2018
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2119
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2220
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
2321
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2422
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
2523
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
26-
github.com/fatih/semgroup v1.3.0 h1:pTEnmcEze/BUf4UmVn9f1ZT1OckkBTNRV9w9k/I2/y4=
27-
github.com/fatih/semgroup v1.3.0/go.mod h1:thVp+PGZMO9KJ+k96oNGJo06hWgsKOWxTfYfx5R2VaE=
24+
github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg=
25+
github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8=
2826
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
2927
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
3028
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
@@ -53,29 +51,35 @@ github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
5351
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
5452
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
5553
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
54+
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
5655
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
5756
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
5857
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
5958
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
60-
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
61-
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
59+
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
60+
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
61+
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
62+
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
6263
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
6364
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
6465
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
6566
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
6667
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
6768
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
68-
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
69-
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
69+
github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 h1:y1p/ycavWjGT9FnmSjdbWUlLGvcxrY0Rw3ATltrxOhk=
70+
github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68/go.mod h1:Xk+z4oIWdQqJzsxyjgl3P22oYZnHdZ8FFTHAQQt5BMQ=
71+
github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs=
72+
github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
73+
github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
7074
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
7175
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
7276
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
7377
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
7478
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
7579
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
80+
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
81+
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
7682
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
77-
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
78-
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
7983
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
8084
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
8185
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
@@ -113,23 +117,24 @@ github.com/urfave/cli/v3 v3.3.3 h1:byCBaVdIXuLPIDm5CYZRVG6NvT7tv1ECqdU4YzlEa3I=
113117
github.com/urfave/cli/v3 v3.3.3/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
114118
github.com/wasilibs/go-re2 v1.9.0 h1:kjAd8qbNvV4Ve2Uf+zrpTCrDHtqH4dlsRXktywo73JQ=
115119
github.com/wasilibs/go-re2 v1.9.0/go.mod h1:0sRtscWgpUdNA137bmr1IUgrRX0Su4dcn9AEe61y+yI=
116-
github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8=
117-
github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
120+
github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
121+
github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
118122
github.com/zricethezav/gitleaks/v8 v8.26.0 h1:TG+9xbX+q+kMPikIKcuTrmPziVDSsEBlQhlslrEJBFA=
119123
github.com/zricethezav/gitleaks/v8 v8.26.0/go.mod h1:D3AhHRLVp0DigFQNxAgHcQks8EbF7wCZanT/UbGd0Jo=
120124
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
121125
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
122-
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
123-
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
124-
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
125-
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
126+
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
127+
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
128+
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
129+
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
126130
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
127131
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
132+
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
128133
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
129134
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
130135
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
131-
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
132-
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
136+
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
137+
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
133138
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
134139
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
135140
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎internal/config/config.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,12 @@ type Config struct {
5050
// and a warning will be logged. Default is 1MB.
5151
LargeFileSizeThreshold int64
5252

53+
// HighTokenThreshold defines the token count above which a file is considered
54+
// to have a high token count and a warning will be logged. Default is 5000.
55+
HighTokenThreshold int
56+
5357
// SkipTokenCount indicates whether to skip counting output tokens.
5458
SkipTokenCount bool
55-
56-
// TokenCountMode specifies how tokens should be counted: "exact" or "fast".
57-
TokenCountMode string
5859
}
5960

6061
// NewConfigFromCommand constructs a Config by extracting relevant values from
@@ -101,9 +102,6 @@ func NewConfigFromCommand(cmd *cli.Command) *Config {
101102
// Check if we should skip counting tokens
102103
skipTokenCount := cmd.Bool("skip-token-count")
103104

104-
// Get token counting mode
105-
tokenCountMode := cmd.String("token-count-mode")
106-
107105
// Get output format
108106
format := cmd.String("format")
109107
// Validate and normalize format
@@ -155,6 +153,12 @@ func NewConfigFromCommand(cmd *cli.Command) *Config {
155153
// Use the default large file size threshold
156154
largeFileSizeThreshold := DefaultLargeFileSizeThreshold
157155

156+
// Get high token count threshold from CLI or use default
157+
highTokenThreshold := cmd.Int("high-token-threshold")
158+
if highTokenThreshold <= 0 {
159+
highTokenThreshold = DefaultHighTokenThreshold
160+
}
161+
158162
cfg := &Config{
159163
TargetDir: targetDir,
160164
OutputFile: outputFile,
@@ -167,8 +171,8 @@ func NewConfigFromCommand(cmd *cli.Command) *Config {
167171
IgnoreSecrets: ignoreSecrets,
168172
RedactSecrets: redactSecrets,
169173
LargeFileSizeThreshold: largeFileSizeThreshold,
174+
HighTokenThreshold: highTokenThreshold,
170175
SkipTokenCount: skipTokenCount,
171-
TokenCountMode: tokenCountMode,
172176
}
173177

174178
return cfg

‎internal/config/defaults.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,26 +31,30 @@ var DefaultAllowedFileExtensions = []string{
3131
// a file is considered "large" and a warning will be logged.
3232
var DefaultLargeFileSizeThreshold int64 = 1024 * 1024
3333

34+
// DefaultHighTokenThreshold defines the default token count (5000) above which
35+
// a file is considered to have a high token count and a warning will be logged.
36+
var DefaultHighTokenThreshold = 5000
37+
3438
// DefaultIgnoredPathPatterns defines the default path patterns that are excluded from processing.
3539
// These include directories, build artifacts, caches, and temporary files.
3640
var DefaultIgnoredPathPatterns = []string{
3741
// Common directories to ignore anywhere in the path
3842
// Using (^|/) to match either the start of a string or after a slash
39-
`(^|/)\.git/`, `(^|/)\.next/`, `(^|/)node_modules/`,
43+
`(^|/)\.git/`, `(^|/)\.next/`, `(^|/)node_modules/`,
4044
`(^|/)dist/`, `(^|/)build/`, `(^|/)out/`, `(^|/)target/`,
4145
`(^|/)\.cache/`, `(^|/)coverage/`, `(^|/)test-results/`,
42-
`(^|/)\.idea/`, `(^|/)\.vscode/`, `(^|/)\.vs/`,
46+
`(^|/)\.idea/`, `(^|/)\.vscode/`, `(^|/)\.vs/`,
4347
`(^|/)\.gradle/`, `(^|/)\.mvn/`, `(^|/)\.pytest_cache/`,
4448
`(^|/)__pycache__/`, `(^|/)\.sass-cache/`, `(^|/)\.vercel/`,
4549
`(^|/)\.turbo/`,
46-
50+
4751
// Directories that should be more specifically matched to avoid false positives
4852
`(^|/)vendor/`, `(^|/)bin/`, `(^|/)obj/`, `(^|/)\.settings/`,
4953

5054
// Lock files and dependency metadata (full path matches to avoid false positives)
51-
`(^|/)pnpm-lock\.yaml$`, `(^|/)package-lock\.json$`, `(^|/)yarn\.lock$`,
52-
`(^|/)Cargo\.lock$`, `(^|/)Gemfile\.lock$`, `(^|/)composer\.lock$`,
53-
`(^|/)mix\.lock$`, `(^|/)poetry\.lock$`, `(^|/)Pipfile\.lock$`,
55+
`(^|/)pnpm-lock\.yaml$`, `(^|/)package-lock\.json$`, `(^|/)yarn\.lock$`,
56+
`(^|/)Cargo\.lock$`, `(^|/)Gemfile\.lock$`, `(^|/)composer\.lock$`,
57+
`(^|/)mix\.lock$`, `(^|/)poetry\.lock$`, `(^|/)Pipfile\.lock$`,
5458
`(^|/)packages\.lock\.json$`, `(^|/)paket\.lock$`,
5559

5660
// Temporary and binary files (match full extensions)
@@ -63,4 +67,4 @@ var DefaultIgnoredPathPatterns = []string{
6367

6468
// Specific files
6569
`(^|/)LICENSE$`, `(^|/)\.gitignore$`,
66-
}
70+
}

0 commit comments

Comments
 (0)