Skip to content

Commit 5614b04

Browse files
pelikhanpelikhan
committed
Refactor integration tests: remove unused auth token check and streamline test cases
1 parent 1754609 commit 5614b04

File tree

2 files changed

+12
-160
lines changed

2 files changed

+12
-160
lines changed

‎Makefile‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@ build:
77
.PHONY: build
88

99
integration: build
10-
@echo "==> building gh-models-test binary for integration tests <=="
11-
go build -o gh-models-test main.go
1210
@echo "==> running integration tests <=="
13-
cd integration && go test -v -timeout=5m
11+
cd integration && go mod tidy && go test -v -timeout=5m
1412
.PHONY: integration
1513

1614
fmt:

‎integration/integration_test.go‎

Lines changed: 11 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,6 @@ func getBinaryPath(t *testing.T) string {
3232
return binaryPath
3333
}
3434

35-
// hasAuthToken checks if GitHub authentication is available
36-
func hasAuthToken() bool {
37-
// Check if gh CLI is available and authenticated
38-
cmd := exec.Command("gh", "auth", "status")
39-
return cmd.Run() == nil
40-
}
41-
4235
// runCommand executes the gh-models binary with given arguments
4336
func runCommand(t *testing.T, args ...string) (stdout, stderr string, err error) {
4437
binaryPath := getBinaryPath(t)
@@ -72,26 +65,8 @@ func runCommand(t *testing.T, args ...string) (stdout, stderr string, err error)
7265
}
7366
}
7467

75-
func TestIntegrationHelp(t *testing.T) {
76-
stdout, stderr, err := runCommand(t, "--help")
77-
78-
// Help should always work, even without auth
79-
require.NoError(t, err, "stderr: %s", stderr)
80-
require.Contains(t, stdout, "GitHub Models CLI extension")
81-
require.Contains(t, stdout, "Available Commands:")
82-
require.Contains(t, stdout, "list")
83-
require.Contains(t, stdout, "run")
84-
require.Contains(t, stdout, "view")
85-
require.Contains(t, stdout, "eval")
86-
}
87-
88-
func TestIntegrationList(t *testing.T) {
89-
if !hasAuthToken() {
90-
t.Skip("Skipping integration test - no GitHub authentication available")
91-
}
92-
68+
func TestList(t *testing.T) {
9369
stdout, stderr, err := runCommand(t, "list")
94-
9570
if err != nil {
9671
t.Logf("List command failed. stdout: %s, stderr: %s", stdout, stderr)
9772
// If the command fails due to auth issues, skip the test
@@ -105,143 +80,22 @@ func TestIntegrationList(t *testing.T) {
10580
require.NotEmpty(t, stdout, "List should produce output")
10681
// Should contain some indication of models or table headers
10782
lowerOut := strings.ToLower(stdout)
108-
hasExpectedContent := strings.Contains(lowerOut, "model") ||
109-
strings.Contains(lowerOut, "name") ||
110-
strings.Contains(lowerOut, "id") ||
111-
strings.Contains(lowerOut, "display")
83+
hasExpectedContent := strings.Contains(lowerOut, "openai/gpt-4.1")
11284
require.True(t, hasExpectedContent, "List output should contain model information")
11385
}
11486

115-
func TestIntegrationListHelp(t *testing.T) {
116-
stdout, stderr, err := runCommand(t, "list", "--help")
117-
118-
require.NoError(t, err, "stderr: %s", stderr)
119-
require.Contains(t, stdout, "Returns a list of models")
120-
require.Contains(t, stdout, "Usage:")
121-
}
122-
123-
func TestIntegrationView(t *testing.T) {
124-
if !hasAuthToken() {
125-
t.Skip("Skipping integration test - no GitHub authentication available")
126-
}
127-
128-
// First get a model to view
129-
listOut, _, listErr := runCommand(t, "list")
130-
if listErr != nil {
131-
t.Skip("Cannot run view test - list command failed")
132-
}
133-
134-
// Extract a model name from list output (this is basic parsing)
135-
lines := strings.Split(listOut, "\n")
136-
var modelName string
137-
for _, line := range lines {
138-
line = strings.TrimSpace(line)
139-
// Look for lines that might contain model IDs (containing forward slash)
140-
if strings.Contains(line, "/") && !strings.HasPrefix(line, "Usage:") &&
141-
!strings.HasPrefix(line, "gh models") && line != "" {
142-
// Try to extract what looks like a model ID
143-
fields := strings.Fields(line)
144-
for _, field := range fields {
145-
if strings.Contains(field, "/") {
146-
modelName = field
147-
break
148-
}
149-
}
150-
if modelName != "" {
151-
break
152-
}
153-
}
154-
}
155-
156-
if modelName == "" {
157-
t.Skip("Could not extract model name from list output")
158-
}
159-
160-
stdout, stderr, err := runCommand(t, "view", modelName)
161-
162-
if err != nil {
163-
t.Logf("View command failed for model %s. stdout: %s, stderr: %s", modelName, stdout, stderr)
164-
// If the command fails due to auth issues, skip the test
165-
if strings.Contains(stderr, "authentication") || strings.Contains(stderr, "token") {
166-
t.Skip("Skipping - authentication issue")
167-
}
168-
require.NoError(t, err, "View command should succeed with valid model")
169-
}
170-
171-
// Basic verification that view command produces expected output
172-
require.NotEmpty(t, stdout, "View should produce output")
173-
lowerOut := strings.ToLower(stdout)
174-
hasExpectedContent := strings.Contains(lowerOut, "model") ||
175-
strings.Contains(lowerOut, "name") ||
176-
strings.Contains(lowerOut, "description") ||
177-
strings.Contains(lowerOut, "publisher")
178-
require.True(t, hasExpectedContent, "View output should contain model details")
179-
}
180-
181-
func TestIntegrationViewHelp(t *testing.T) {
182-
stdout, stderr, err := runCommand(t, "view", "--help")
183-
184-
require.NoError(t, err, "stderr: %s", stderr)
185-
require.Contains(t, stdout, "Returns details about the specified model")
186-
require.Contains(t, stdout, "Usage:")
187-
}
188-
189-
func TestIntegrationRunHelp(t *testing.T) {
190-
stdout, stderr, err := runCommand(t, "run", "--help")
191-
192-
require.NoError(t, err, "stderr: %s", stderr)
193-
require.Contains(t, stdout, "Prompts the specified model")
194-
require.Contains(t, stdout, "Usage:")
195-
}
196-
197-
func TestIntegrationEvalHelp(t *testing.T) {
198-
stdout, stderr, err := runCommand(t, "eval", "--help")
199-
200-
require.NoError(t, err, "stderr: %s", stderr)
201-
require.Contains(t, stdout, "Runs evaluation tests against a model")
202-
require.Contains(t, stdout, "Usage:")
203-
}
204-
205-
// TestIntegrationRun tests the run command with a simple prompt
87+
// TestRun tests the run command with a simple prompt
20688
// This test is more limited since it requires actual model inference
207-
func TestIntegrationRun(t *testing.T) {
208-
if !hasAuthToken() {
209-
t.Skip("Skipping integration test - no GitHub authentication available")
210-
}
211-
212-
// We'll test with a very simple prompt to minimize cost and time
213-
// Using a basic model and short prompt
214-
stdout, _, err := runCommand(t, "run", "--help")
215-
require.NoError(t, err, "Run help should work")
216-
217-
// For now, just verify the help works.
218-
// A full test would require setting up a prompt and model,
219-
// which might be expensive for CI
220-
require.Contains(t, stdout, "Prompts the specified model")
89+
func TestRun(t *testing.T) {
90+
stdout, _, err := runCommand(t, "run", "openai/gpt-4.1-nano", "say 'pain' in french")
91+
require.NoError(t, err, "Run should work")
92+
require.Contains(t, strings.ToLower(stdout), "pain")
22193
}
22294

22395
// TestIntegrationRunWithOrg tests the run command with --org flag
224-
func TestIntegrationRunWithOrg(t *testing.T) {
225-
if !hasAuthToken() {
226-
t.Skip("Skipping integration test - no GitHub authentication available")
227-
}
228-
96+
func TestRunWithOrg(t *testing.T) {
22997
// Test run command with --org flag (using help to avoid expensive API calls)
230-
stdout, _, err := runCommand(t, "run", "--org", "test-org", "--help")
231-
require.NoError(t, err, "Run help with --org should work")
232-
require.Contains(t, stdout, "Prompts the specified model")
233-
require.Contains(t, stdout, "--org string")
234-
}
235-
236-
// TestIntegrationEvalWithOrg tests the eval command with --org flag
237-
func TestIntegrationEvalWithOrg(t *testing.T) {
238-
if !hasAuthToken() {
239-
t.Skip("Skipping integration test - no GitHub authentication available")
240-
}
241-
242-
// Test eval command with --org flag (using help to avoid expensive API calls)
243-
stdout, _, err := runCommand(t, "eval", "--org", "test-org", "--help")
244-
require.NoError(t, err, "Eval help with --org should work")
245-
require.Contains(t, stdout, "Runs evaluation tests against a model")
246-
require.Contains(t, stdout, "--org string")
98+
stdout, _, err := runCommand(t, "run", "openai/gpt-4.1-nano", "say 'pain' in french", "--org", "github")
99+
require.NoError(t, err, "Run should work")
100+
require.Contains(t, strings.ToLower(stdout), "pain")
247101
}

0 commit comments

Comments
 (0)