Implement ParseRules function to clean up rules text; add tests for I…

…sUnassistedResponse, Unfence, and SplitLines functions
github · sgoedecke · Aug 4, 2025 · Jul 21, 2025 · Jul 21, 2025 · Jul 21, 2025
commit d3b430af1646f5f054ea42d8980de93911b0d892
@@ -0,0 +1,32 @@
+package generate
+
+import (
+	"regexp"
+	"strings"
+)
+
+// IsUnassistedResponse returns true if the text is an unassisted response, like "i'm sorry" or "i can't assist with that".
+func IsUnassistedResponse(text string) bool {
+	re := regexp.MustCompile(`i can't assist with that|i'm sorry`)
+	return re.MatchString(strings.ToLower(text))
+}
+
+// unfence removes code fences and splits text into lines.
+func Unfence(text string) string {
+	text = strings.TrimSpace(text)
+	// Remove triple backtick code fences if present
+	if strings.HasPrefix(text, "```") {
+		parts := strings.SplitN(text, "\n", 2)
+		if len(parts) == 2 {
+			text = parts[1]
+		}
+		text = strings.TrimSuffix(text, "```")
+	}
+	return text
+}
+
+// splits text into lines.
+func SplitLines(text string) []string {
+	lines := strings.Split(text, "\n")
+	return lines
+}
@@ -0,0 +1,225 @@
+package generate
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsUnassistedResponse(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected bool
+	}{
+		{
+			name:     "detects 'i can't assist with that' lowercase",
+			input:    "i can't assist with that request",
+			expected: true,
+		},
+		{
+			name:     "detects 'i can't assist with that' mixed case",
+			input:    "I Can't Assist With That Request",
+			expected: true,
+		},
+		{
+			name:     "detects 'i'm sorry' lowercase",
+			input:    "i'm sorry, but i cannot help",
+			expected: true,
+		},
+		{
+			name:     "detects 'i'm sorry' mixed case",
+			input:    "I'm Sorry, But I Cannot Help",
+			expected: true,
+		},
+		{
+			name:     "detects phrase within larger text",
+			input:    "Unfortunately, I can't assist with that particular request. Please try something else.",
+			expected: true,
+		},
+		{
+			name:     "detects 'i'm sorry' within larger text",
+			input:    "Well, I'm sorry to say this but I cannot proceed.",
+			expected: true,
+		},
+		{
+			name:     "returns false for regular response",
+			input:    "Here is the code you requested",
+			expected: false,
+		},
+		{
+			name:     "returns false for empty string",
+			input:    "",
+			expected: false,
+		},
+		{
+			name:     "returns false for similar but different phrases",
+			input:    "i can assist with that",
+			expected: false,
+		},
+		{
+			name:     "returns false for partial matches",
+			input:    "sorry for the delay",
+			expected: false,
+		},
+		{
+			name:     "handles apostrophe variations",
+			input:    "i can't assist with that",
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := IsUnassistedResponse(tt.input)
+			require.Equal(t, tt.expected, result)
+		})
+	}
+}
+
+func TestUnfence(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "removes code fences with language",
+			input:    "```go\npackage main\nfunc main() {}\n```",
+			expected: "package main\nfunc main() {}\n",
+		},
+		{
+			name:     "removes code fences without language",
+			input:    "```\nsome code\nmore code\n```",
+			expected: "some code\nmore code\n",
+		},
+		{
+			name:     "handles text without code fences",
+			input:    "just plain text",
+			expected: "just plain text",
+		},
+		{
+			name:     "handles empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "handles whitespace around text",
+			input:    "  \n  some text  \n  ",
+			expected: "some text",
+		},
+		{
+			name:     "handles only opening fence",
+			input:    "```go\ncode without closing",
+			expected: "code without closing",
+		},
+		{
+			name:     "handles fence with no content",
+			input:    "```\n```",
+			expected: "",
+		},
+		{
+			name:     "handles fence with only language - no newline",
+			input:    "```python",
+			expected: "```python",
+		},
+		{
+			name:     "preserves content that looks like fences but isn't at start",
+			input:    "some text\n```\nmore text",
+			expected: "some text\n```\nmore text",
+		},
+		{
+			name:     "handles multiple lines after fence",
+			input:    "```javascript\nfunction test() {\n  return 'hello';\n}\nconsole.log('world');\n```",
+			expected: "function test() {\n  return 'hello';\n}\nconsole.log('world');\n",
+		},
+		{
+			name:     "handles single line with fences - no newline",
+			input:    "```const x = 5;```",
+			expected: "```const x = 5;",
+		},
+		{
+			name:     "handles content with leading/trailing whitespace inside fences",
+			input:    "```\n  \n  code content  \n  \n```",
+			expected: "  \n  code content  \n  \n",
+		},
+		{
+			name:     "handles fence with language and content on same line",
+			input:    "```go func main() {}```",
+			expected: "```go func main() {}",
+		},
+		{
+			name:     "removes only trailing fence markers",
+			input:    "```\ncode with ``` in middle\n```",
+			expected: "code with ``` in middle\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := Unfence(tt.input)
+			require.Equal(t, tt.expected, result)
+		})
+	}
+}
+
+func TestSplitLines(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected []string
+	}{
+		{
+			name:     "splits multi-line text",
+			input:    "line 1\nline 2\nline 3",
+			expected: []string{"line 1", "line 2", "line 3"},
+		},
+		{
+			name:     "handles single line",
+			input:    "single line",
+			expected: []string{"single line"},
+		},
+		{
+			name:     "handles empty string",
+			input:    "",
+			expected: []string{""},
+		},
+		{
+			name:     "handles string with only newlines",
+			input:    "\n\n\n",
+			expected: []string{"", "", "", ""},
+		},
+		{
+			name:     "handles text with trailing newline",
+			input:    "line 1\nline 2\n",
+			expected: []string{"line 1", "line 2", ""},
+		},
+		{
+			name:     "handles text with leading newline",
+			input:    "\nline 1\nline 2",
+			expected: []string{"", "line 1", "line 2"},
+		},
+		{
+			name:     "handles mixed line endings and content",
+			input:    "start\n\nmiddle\n\nend",
+			expected: []string{"start", "", "middle", "", "end"},
+		},
+		{
+			name:     "handles single newline",
+			input:    "\n",
+			expected: []string{"", ""},
+		},
+		{
+			name:     "preserves empty lines between content",
+			input:    "first\n\n\nsecond",
+			expected: []string{"first", "", "", "second"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := SplitLines(tt.input)
+			require.Equal(t, tt.expected, result)
+		})
+	}
+}
@@ -31,7 +31,7 @@ func (h *generateCommandHandler) CreateContextFromPrompt(promptFile string) (*Pr
 		PromptHash: promptHash,
 		// Infered intent of the prompt
 		Intent:       "",
-		Rules:        "",
+		Rules:        []string{},
 		InverseRules: "",
 		InputSpec:    "",
 		Tests:        "",

diff --git a/cmd/generate/context_test.go b/cmd/generate/context_test.go
@@ -3,6 +3,7 @@ package generate
 import (
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -151,7 +152,7 @@ messages:
 						t.Errorf("Expected %s to be %q, got %q", field, expectedValue, context.Intent)
 					}
 				case "rules":
-					if context.Rules != expectedValue.(string) {
+					if !reflect.DeepEqual(context.Rules, expectedValue.([]string)) {
 						t.Errorf("Expected %s to be %q, got %q", field, expectedValue, context.Rules)
 					}
 				case "inverseRules":

diff --git a/cmd/generate/export_test.go b/cmd/generate/export_test.go
@@ -36,7 +36,7 @@ func TestToGitHubModelsPrompt(t *testing.T) {
 						},
 					},
 				},
-				Rules: "1. Be helpful\n2. Be accurate",
+				Rules: []string{"1. Be helpful", "2. Be accurate"},
 				PromptPexTests: []PromptPexTest{
 					{
 						TestInput:   `{"input": "world"}`,
@@ -73,7 +73,7 @@ func TestToGitHubModelsPrompt(t *testing.T) {
 						},
 					},
 				},
-				Rules: "Test rules",
+				Rules: []string{"Test rules"},
 				PromptPexTests: []PromptPexTest{
 					{
 						TestInput: "simple test",
@@ -106,7 +106,7 @@ func TestToGitHubModelsPrompt(t *testing.T) {
 						},
 					},
 				},
-				Rules: "Process data correctly",
+				Rules: []string{"Process data correctly"},
 				PromptPexTests: []PromptPexTest{
 					{
 						TestInput:   `{"data": "test data", "type": "analysis"}`,
@@ -140,7 +140,7 @@ func TestToGitHubModelsPrompt(t *testing.T) {
 						},
 					},
 				},
-				Rules: "Test rules",
+				Rules: []string{"Test rules"},
 				PromptPexTests: []PromptPexTest{
 					{
 						TestInput: "",

@@ -18,6 +18,7 @@ func GetDefaultOptions() *PromptPexOptions {
 		Models: &PromptPexModelAliases{
 			Rules:       util.Ptr("openai/gpt-4o"),
 			Groundtruth: util.Ptr("openai/gpt-4o"),
+			Compliance:  util.Ptr("openai/gpt-4o"),
 		},
 	}
 }

@@ -3,8 +3,38 @@ package generate
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
+	"strings"
 )
 
+// parseRules removes numbering, bullets, and extraneous "Rules:" lines from a rules text block.
+func ParseRules(text string) []string {
+	if IsUnassistedResponse(text) {
+		return nil
+	}
+	lines := SplitLines(Unfence(text))
+	itemsRe := regexp.MustCompile(`^\s*(\d+\.|_|-|\*)\s+`) // remove leading item numbers or bullets
+	rulesRe := regexp.MustCompile(`^\s*Rules:\s*$`)
+	pythonWrapRe := regexp.MustCompile(`^\["(.*)"\]$`)
+	var cleaned []string
+	for _, line := range lines {
+		// Remove leading numbering or bullets
+		replaced := itemsRe.ReplaceAllString(line, "")
+		// Skip empty lines
+		if strings.TrimSpace(replaced) == "" {
+			continue
+		}
+		// Skip "Rules:" header lines
+		if rulesRe.MatchString(replaced) {
+			continue
+		}
+		// Remove ["..."] wrapping
+		replaced = pythonWrapRe.ReplaceAllString(replaced, "$1")
+		cleaned = append(cleaned, replaced)
+	}
+	return cleaned
+}
+
 // ParseTestsFromLLMResponse parses test cases from LLM response with robust error handling
 func (h *generateCommandHandler) ParseTestsFromLLMResponse(content string) ([]PromptPexTest, error) {
 	jsonStr := ExtractJSON(content)