github · pelikhan · Jul 24, 2025 · Jul 24, 2025 · Jul 24, 2025 · Jul 24, 2025
diff --git a/cmd/eval/eval.go b/cmd/eval/eval.go
@@ -11,7 +11,7 @@

 	"github.com/MakeNowJust/heredoc"
 	"github.com/cli/go-gh/v2/pkg/tableprinter"
 	"github.com/github/gh-models/internal/azuremodels"
 	"github.com/github/gh-models/pkg/command"
 	"github.com/github/gh-models/pkg/prompt"
 	"github.com/github/gh-models/pkg/util"
@@ -111,6 +111,9 @@
 			// Get the org flag
 			org, _ := cmd.Flags().GetString("org")
 
+			// Get the http-log flag
+			httpLog, _ := cmd.Flags().GetString("http-log")
+
 			// Load the evaluation prompt file
 			evalFile, err := loadEvaluationPromptFile(promptFilePath)
 			if err != nil {
@@ -126,7 +129,13 @@
 				org:        org,
 			}
 
-			err = handler.runEvaluation(cmd.Context())
+			ctx := cmd.Context()
+			// Add HTTP log filename to context if provided
+			if httpLog != "" {
+				ctx = azuremodels.WithHTTPLogFile(ctx, httpLog)
+			}
+
+			err = handler.runEvaluation(ctx)
 			if err == FailedTests {
 				// Cobra by default will show the help message when an error occurs,
 				// which is not what we want for failed evaluations.
@@ -139,6 +148,7 @@
 
 	cmd.Flags().Bool("json", false, "Output results in JSON format")
 	cmd.Flags().String("org", "", "Organization to attribute usage to (omitting will attribute usage to the current actor")
+	cmd.Flags().String("http-log", "", "Path to log HTTP requests to (optional)")
 	return cmd
 }
 

diff --git a/cmd/generate/generate.go b/cmd/generate/generate.go
@@ -50,9 +50,18 @@ func NewGenerateCommand(cfg *command.Config) *cobra.Command {
 			// Get organization
 			org, _ := cmd.Flags().GetString("org")
 
+			// Get http-log flag
+			httpLog, _ := cmd.Flags().GetString("http-log")
+
+			ctx := cmd.Context()
+			// Add HTTP log filename to context if provided
+			if httpLog != "" {
+				ctx = azuremodels.WithHTTPLogFile(ctx, httpLog)
+			}
+
 			// Create the command handler
 			handler := &generateCommandHandler{
-				ctx:     cmd.Context(),
+				ctx:     ctx,
 				cfg:     cfg,
 				client:  cfg.Client,
 				options: options,
@@ -97,6 +106,7 @@ func AddCommandLineFlags(cmd *cobra.Command) {
 	flags.String("custom-metric", "", "Custom evaluation metric")
 	flags.Float64("temperature", 0.0, "Temperature for model inference")
 	flags.Bool("verbose", false, "Enable verbose output including LLM payloads")
+	flags.String("http-log", "", "Path to log HTTP requests to (optional)")
 }
 
 // parseFlags parses command-line flags and applies them to the options

diff --git a/cmd/run/run.go b/cmd/run/run.go
@@ -423,6 +423,7 @@ func NewRunCommand(cfg *command.Config) *cobra.Command {
 	cmd.Flags().String("top-p", "", "Controls text diversity by selecting the most probable words until a set probability is reached.")
 	cmd.Flags().String("system-prompt", "", "Prompt the system.")
 	cmd.Flags().String("org", "", "Organization to attribute usage to (omitting will attribute usage to the current actor")
+	cmd.Flags().String("http-log", "", "Path to log HTTP requests to (optional)")
 
 	return cmd
 }
@@ -472,7 +473,15 @@ type runCommandHandler struct {
 }
 
 func newRunCommandHandler(cmd *cobra.Command, cfg *command.Config, args []string) *runCommandHandler {
-	return &runCommandHandler{ctx: cmd.Context(), cfg: cfg, client: cfg.Client, args: args}
+	ctx := cmd.Context()
+	httpLog, _ := cmd.Flags().GetString("http-log")
+
+	// Add HTTP log filename to context if provided
+	if httpLog != "" {
+		ctx = azuremodels.WithHTTPLogFile(ctx, httpLog)
+	}
+
+	return &runCommandHandler{ctx: ctx, cfg: cfg, client: cfg.Client, args: args}
 }
 
 func (h *runCommandHandler) loadModels() ([]*azuremodels.ModelSummary, error) {

diff --git a/internal/azuremodels/azure_client.go b/internal/azuremodels/azure_client.go
@@ -67,20 +67,16 @@
 		inferenceURL = c.cfg.InferenceRoot + "/" + c.cfg.InferencePath
 	}
 
-	// TODO: remove logging
-	// Write request details to llm.http file for debugging
-	if os.Getenv("DEBUG") != "" {
-		httpFile, err := os.OpenFile("llm.http", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	// Write request details to specified log file for debugging
+	httpLogFile := HTTPLogFileFromContext(ctx)
+	if httpLogFile != "" {
+		logFile, err := os.OpenFile(httpLogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 		if err == nil {
-			defer httpFile.Close()
-			fmt.Fprintf(httpFile, "### %s\n", time.Now().Format(time.RFC3339))
-			fmt.Fprintf(httpFile, "POST %s\n", inferenceURL)
-			fmt.Fprintf(httpFile, "Authorization: Bearer {{$processEnv GITHUB_TOKEN}}\n")
-			fmt.Fprintf(httpFile, "Content-Type: application/json\n")
-			fmt.Fprintf(httpFile, "x-ms-useragent: github-cli-models\n")
-			fmt.Fprintf(httpFile, "x-ms-user-agent: github-cli-models\n")
-			fmt.Fprintf(httpFile, "\n%s\n\n", string(bodyBytes))
+			defer logFile.Close()
+			fmt.Fprintf(logFile, "### %s\nPOST %s\nAuthorization: Bearer {{$processEnv GITHUB_TOKEN}}\nContent-Type: application/json\nx-ms-useragent: github-cli-models\nx-ms-user-agent: github-cli-models\n\n%s\n\n", 
+			const logFormat = "### %s\nPOST %s\nAuthorization: Bearer {{$processEnv GITHUB_TOKEN}}\nContent-Type: application/json\nx-ms-useragent: github-cli-models\nx-ms-user-agent: github-cli-models\n\n%s\n\n"
+			fmt.Fprintf(logFile, logFormat, time.Now().Format(time.RFC3339), inferenceURL, string(bodyBytes))
 		}
 	}
 
 	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, inferenceURL, body)

diff --git a/internal/azuremodels/client.go b/internal/azuremodels/client.go
@@ -2,10 +2,27 @@ package azuremodels
 
 import "context"
 
+// httpLogFileKey is the context key for the HTTP log filename
+type httpLogFileKey struct{}
+
+// WithHTTPLogFile returns a new context with the HTTP log filename attached
+func WithHTTPLogFile(ctx context.Context, httpLogFile string) context.Context {
+	return context.WithValue(ctx, httpLogFileKey{}, httpLogFile)
+}
+
+// HTTPLogFileFromContext returns the HTTP log filename from the context, if any
+func HTTPLogFileFromContext(ctx context.Context) string {
+	if httpLogFile, ok := ctx.Value(httpLogFileKey{}).(string); ok {
+		return httpLogFile
+	}
+	return ""
+}
+
 // Client represents a client for interacting with an API about models.
 type Client interface {
 	// GetChatCompletionStream returns a stream of chat completions using the given options.
-	GetChatCompletionStream(context.Context, ChatCompletionOptions, string) (*ChatCompletionResponse, error)
+	// HTTP logging configuration is extracted from the context if present.
+	GetChatCompletionStream(ctx context.Context, req ChatCompletionOptions, org string) (*ChatCompletionResponse, error)
 	// GetModelDetails returns the details of the specified model in a particular registry.
 	GetModelDetails(ctx context.Context, registry, modelName, version string) (*ModelDetails, error)
 	// ListModels returns a list of available models.