Skip to content

Commit 283a3c7

Browse files
pelikhanpelikhan
committed
Update effort levels in documentation and code: add 'min' level, adjust validation and defaults
1 parent e04761b commit 283a3c7

File tree

7 files changed

+43
-46
lines changed

7 files changed

+43
-46
lines changed

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ graph TD
130130
You can customize the test generation process with various options:
131131

132132
```shell
133-
# Specify effort level (low, medium, high)
133+
# Specify effort level (min, low, medium, high)
134134
gh models generate --effort high my_prompt.prompt.yml
135135

136136
# Use a specific model for groundtruth generation
@@ -148,6 +148,7 @@ gh models generate --instruction-intent "Focus on edge cases" my_prompt.prompt.y
148148

149149
The `effort` flag controls a few flags in the test generation engine and is a tradeoff
150150
between how much tests you want generated and how much tokens/time you are willing to spend.
151+
- `min` is just enough to generate a few tests and make sure things are probably configured.
151152
- `low` should be used to do a quick try of the test generation. It limits the number of rules to `3`.
152153
- `medium` provides much better coverage
153154
- `high` spends more token per rule to generate tests, which typically leads to longer, more complex inputs

‎cmd/generate/effort.go‎

Lines changed: 26 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,37 @@ package generate
22

33
// EffortConfiguration defines the configuration for different effort levels
44
type EffortConfiguration struct {
5-
TestsPerRule int
6-
MaxRules int
7-
MaxRulesPerTestGeneration int
8-
RulesPerGen int
5+
MaxRules int
6+
TestsPerRule int
7+
RulesPerGen int
98
}
109

1110
// GetEffortConfiguration returns the configuration for a given effort level
1211
// Based on the reference TypeScript implementation in constants.mts
1312
func GetEffortConfiguration(effort string) *EffortConfiguration {
1413
switch effort {
14+
case EffortMin:
15+
return &EffortConfiguration{
16+
MaxRules: 3,
17+
TestsPerRule: 1,
18+
RulesPerGen: 100,
19+
}
1520
case EffortLow:
1621
return &EffortConfiguration{
17-
MaxRules: 3,
18-
TestsPerRule: 2,
19-
MaxRulesPerTestGeneration: 5,
20-
RulesPerGen: 10,
22+
MaxRules: 10,
23+
TestsPerRule: 1,
24+
RulesPerGen: 10,
2125
}
2226
case EffortMedium:
2327
return &EffortConfiguration{
24-
MaxRules: 20,
25-
TestsPerRule: 3,
26-
MaxRulesPerTestGeneration: 5,
27-
RulesPerGen: 5,
28+
MaxRules: 20,
29+
TestsPerRule: 3,
30+
RulesPerGen: 5,
2831
}
2932
case EffortHigh:
3033
return &EffortConfiguration{
31-
MaxRules: 50,
32-
MaxRulesPerTestGeneration: 2,
33-
RulesPerGen: 3,
34+
TestsPerRule: 4,
35+
RulesPerGen: 3,
3436
}
3537
default:
3638
return nil
@@ -43,22 +45,18 @@ func ApplyEffortConfiguration(options *PromptPexOptions, effort string) {
4345
return
4446
}
4547

46-
config := GetEffortConfiguration(effort)
47-
if config == nil {
48+
effortConfig := GetEffortConfiguration(effort)
49+
if effortConfig == nil {
4850
return
4951
}
50-
51-
// Apply configuration settings only if not already set
52-
if options.TestsPerRule == 0 {
53-
options.TestsPerRule = config.TestsPerRule
54-
}
55-
if options.MaxRules == 0 {
56-
options.MaxRules = config.MaxRules
52+
// Apply effort if set
53+
if effortConfig.TestsPerRule != 0 {
54+
options.TestsPerRule = effortConfig.TestsPerRule
5755
}
58-
if options.MaxRulesPerTestGen == 0 {
59-
options.MaxRulesPerTestGen = config.MaxRulesPerTestGeneration
56+
if effortConfig.MaxRules != 0 {
57+
options.MaxRules = effortConfig.MaxRules
6058
}
61-
if options.RulesPerGen == 0 {
62-
options.RulesPerGen = config.RulesPerGen
59+
if effortConfig.RulesPerGen != 0 {
60+
options.RulesPerGen = effortConfig.RulesPerGen
6361
}
6462
}

‎cmd/generate/generate.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ func NewGenerateCommand(cfg *command.Config) *cobra.Command {
116116
func AddCommandLineFlags(cmd *cobra.Command) {
117117
flags := cmd.Flags()
118118
flags.String("org", "", "Organization to attribute usage to")
119-
flags.String("effort", "", "Effort level (low, medium, high)")
119+
flags.String("effort", "", "Effort level (min, low, medium, high)")
120120
flags.String("groundtruth-model", "", "Model to use for generating groundtruth outputs. Defaults to openai/gpt-4o. Use 'none' to disable groundtruth generation.")
121121
flags.String("session-file", "", "Session file to load existing context from")
122122
flags.StringArray("var", []string{}, "Template variables for prompt files (can be used multiple times: --var name=value)")
@@ -135,8 +135,8 @@ func ParseFlags(cmd *cobra.Command, options *PromptPexOptions) error {
135135
// Parse effort first so it can set defaults
136136
if effort, _ := flags.GetString("effort"); effort != "" {
137137
// Validate effort value
138-
if effort != EffortLow && effort != EffortMedium && effort != EffortHigh {
139-
return fmt.Errorf("invalid effort level '%s': must be one of %s, %s, or %s", effort, EffortLow, EffortMedium, EffortHigh)
138+
if effort != EffortMin && effort != EffortLow && effort != EffortMedium && effort != EffortHigh {
139+
return fmt.Errorf("invalid effort level '%s': must be one of %s, %s, %s, or %s", effort, EffortMin, EffortLow, EffortMedium, EffortHigh)
140140
}
141141
options.Effort = effort
142142
}

‎cmd/generate/generate_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ func TestParseFlagsInvalidEffort(t *testing.T) {
181181
{
182182
name: "invalid effort value",
183183
effort: "invalid",
184-
expectedErr: "invalid effort level 'invalid': must be one of low, medium, or high",
184+
expectedErr: "invalid effort level 'invalid': must be one of min, low, medium, or high",
185185
},
186186
{
187187
name: "empty effort value",
@@ -191,12 +191,12 @@ func TestParseFlagsInvalidEffort(t *testing.T) {
191191
{
192192
name: "case sensitive effort",
193193
effort: "Low",
194-
expectedErr: "invalid effort level 'Low': must be one of low, medium, or high",
194+
expectedErr: "invalid effort level 'Low': must be one of min, low, medium, or high",
195195
},
196196
{
197197
name: "numeric effort",
198198
effort: "1",
199-
expectedErr: "invalid effort level '1': must be one of low, medium, or high",
199+
expectedErr: "invalid effort level '1': must be one of min, low, medium, or high",
200200
},
201201
}
202202

‎cmd/generate/options.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ func GetDefaultOptions() *PromptPexOptions {
55
return &PromptPexOptions{
66
TestsPerRule: 3,
77
RulesPerGen: 3,
8-
MaxRulesPerTestGen: 3,
98
Verbose: false,
109
IntentMaxTokens: 100,
1110
InputSpecMaxTokens: 500,

‎cmd/generate/pipeline.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ Input Specification:`, RenderMessagesToString(context.Prompt.Messages))
176176

177177
// generateOutputRules generates output rules for the prompt
178178
func (h *generateCommandHandler) generateOutputRules(context *PromptPexContext) error {
179-
h.WriteStartBox("Output rules", "")
179+
h.WriteStartBox("Output rules", fmt.Sprintf("max rules: %d", h.options.MaxRules))
180180
if len(context.Rules) == 0 {
181181
system := `Analyze the following prompt and generate a list of output rules.
182182
These rules should describe what makes a valid output from this prompt.
@@ -220,6 +220,10 @@ Output Rules:`, RenderMessagesToString(context.Prompt.Messages))
220220
return fmt.Errorf("failed to parse output rules: %s", rules)
221221
}
222222

223+
if h.options.MaxRules > 0 && len(parsed) > h.options.MaxRules {
224+
parsed = parsed[:h.options.MaxRules]
225+
}
226+
223227
context.Rules = parsed
224228
}
225229

@@ -284,12 +288,7 @@ Inverse Output Rules:`, strings.Join(context.Rules, "\n"))
284288
func (h *generateCommandHandler) generateTests(context *PromptPexContext) error {
285289
h.WriteStartBox("Tests", fmt.Sprintf("%d rules x %d tests per rule", len(context.Rules)+len(context.InverseRules), h.options.TestsPerRule))
286290
if len(context.Tests) == 0 {
287-
defaultOptions := GetDefaultOptions()
288-
testsPerRule := defaultOptions.TestsPerRule
289-
if h.options.TestsPerRule != 0 {
290-
testsPerRule = h.options.TestsPerRule
291-
}
292-
291+
testsPerRule := h.options.TestsPerRule
293292
allRules := append(context.Rules, context.InverseRules...)
294293

295294
// Generate tests iteratively for groups of rules
@@ -313,7 +312,7 @@ func (h *generateCommandHandler) generateTests(context *PromptPexContext) error
313312
// render to terminal
314313
for _, test := range groupTests {
315314
h.WriteToLine(test.Input)
316-
h.WriteToLine(fmt.Sprintf(" %s%s", BOX_END, test.Reasoning))
315+
h.WriteToLine(fmt.Sprintf(" %s%s", BOX_END, test.Reasoning))
317316
}
318317

319318
// Accumulate tests
@@ -531,7 +530,7 @@ func (h *generateCommandHandler) generateGroundtruth(context *PromptPexContext)
531530
h.cfg.WriteToOut(fmt.Sprintf("Saving context failed: %v", err))
532531
}
533532
}
534-
h.WriteToLine(fmt.Sprintf(" %s%s", BOX_END, test.Expected)) // Write groundtruth output
533+
h.WriteToLine(fmt.Sprintf(" %s%s", BOX_END, test.Expected)) // Write groundtruth output
535534
}
536535

537536
h.WriteEndBox(fmt.Sprintf("%d items", len(context.Tests)))

‎cmd/generate/types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ type PromptPexOptions struct {
2727
TestsPerRule int `yaml:"testsPerRule,omitempty" json:"testsPerRule,omitempty"`
2828
RulesPerGen int `yaml:"rulesPerGen,omitempty" json:"rulesPerGen,omitempty"`
2929
MaxRules int `yaml:"maxRules,omitempty" json:"maxRules,omitempty"`
30-
MaxRulesPerTestGen int `yaml:"maxRulesPerTestGen,omitempty" json:"maxRulesPerTestGen,omitempty"`
3130
IntentMaxTokens int `yaml:"intentMaxTokens,omitempty" json:"intentMaxTokens,omitempty"`
3231
InputSpecMaxTokens int `yaml:"inputSpecMaxTokens,omitempty" json:"inputSpecMaxTokens,omitempty"`
3332

@@ -63,6 +62,7 @@ type PromptPexTest struct {
6362

6463
// Effort levels
6564
const (
65+
EffortMin = "min"
6666
EffortLow = "low"
6767
EffortMedium = "medium"
6868
EffortHigh = "high"

0 commit comments

Comments
 (0)