Skip to content

Add a --json mode to evals #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sgoedecke merged 6 commits into from
Jun 5, 2025
Merged

Add a --json mode to evals #55

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0ae63f6
Add docs link and remove unneeded type
sgoedecke Jun 5, 2025
0b65416
JSON mode for evals
sgoedecke Jun 5, 2025
53887d6
go fmt
sgoedecke Jun 5, 2025
eecd38f
Refactor out BuildChatCompletionOptions
sgoedecke Jun 5, 2025
b2cb120
Refactor out message role function
sgoedecke Jun 5, 2025
41f4662
Add docs to readme and fixup pass rate logic
sgoedecke Jun 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor out message role function
  • Loading branch information
@sgoedecke
sgoedecke committed Jun 5, 2025
commit b2cb12076a739098e19f3824213a92c402d61f25
13 changes: 3 additions & 10 deletions cmd/eval/eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,16 +286,9 @@ func (h *evalCommandHandler) templateMessages(testCase map[string]interface{}) (
return nil, fmt.Errorf("failed to template message content: %w", err)
}

var role azuremodels.ChatMessageRole
switch strings.ToLower(msg.Role) {
case "system":
role = azuremodels.ChatMessageRoleSystem
case "user":
role = azuremodels.ChatMessageRoleUser
case "assistant":
role = azuremodels.ChatMessageRoleAssistant
default:
return nil, fmt.Errorf("unknown message role: %s", msg.Role)
role, err := prompt.GetAzureChatMessageRole(msg.Role)
if err != nil {
return nil, err
}

messages = append(messages, azuremodels.ChatMessage{
Expand Down
13 changes: 9 additions & 4 deletions cmd/run/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,12 +288,17 @@ func NewRunCommand(cfg *command.Config) *cobra.Command {
return err
}

switch strings.ToLower(m.Role) {
case "system":
role, err := prompt.GetAzureChatMessageRole(m.Role)
if err != nil {
return err
}

switch role {
case azuremodels.ChatMessageRoleSystem:
conversation.systemPrompt = content
case "user":
case azuremodels.ChatMessageRoleUser:
conversation.AddMessage(azuremodels.ChatMessageRoleUser, content)
case "assistant":
case azuremodels.ChatMessageRoleAssistant:
conversation.AddMessage(azuremodels.ChatMessageRoleAssistant, content)
}
}
Expand Down
14 changes: 14 additions & 0 deletions pkg/prompt/prompt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,20 @@ func TemplateString(templateStr string, data interface{}) (string, error) {
return result, nil
}

// GetAzureChatMessageRole converts a role string to azuremodels.ChatMessageRole
func GetAzureChatMessageRole(role string) (azuremodels.ChatMessageRole, error) {
switch strings.ToLower(role) {
case "system":
return azuremodels.ChatMessageRoleSystem, nil
case "user":
return azuremodels.ChatMessageRoleUser, nil
case "assistant":
return azuremodels.ChatMessageRoleAssistant, nil
default:
return "", fmt.Errorf("unknown message role: %s", role)
}
}

// BuildChatCompletionOptions creates a ChatCompletionOptions with the file's model and parameters
func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) azuremodels.ChatCompletionOptions {
Copy link
Member

@cheshire137 cheshire137 Jun 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the new functions!

req := azuremodels.ChatCompletionOptions{
Expand Down
Loading