wip #4 on getLargeLog()

mikebd · Oct 16, 2023 · 61468c0 · 61468c0
1 parent faf4255
commit 61468c0
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 16 deletions.
diff --git a/config/constants.go b/config/constants.go
@@ -3,8 +3,9 @@ package config
 const (
 	HttpPort             = 8080
 	LargeFileBytes       = 10_000_000
+	LargeFileSeekBytes   = 1_000_000
 	LogDirectory         = "/var/log"
+	MaxResultLines       = 2_000
 	NumberOfLogLines     = 25
-	SeekBytes            = 1_000_000
 	UnsupportedFileTypes = "gz|zip|bz2|tar|tgz|7z|rar"
 )
diff --git a/service/getLog.go b/service/getLog.go
@@ -43,9 +43,8 @@ func selectLogStrategy(filepath string) getLogStrategy {
 func getLargeLog(params *GetLogParams) GetLogResult {
 	const strategy = "large"
 
-	// TODO: What if the result size is larger than the available memory?  e.g. no filters
-	//       We should cap params.MaxLines to a reasonable value, e.g. 2000
-	var result []string
+	maxResultLines := maxLines(params)
+	result := make([]string, 0, maxResultLines*2)
 
 	// Open the file for reading
 	file, errOpen := os.OpenFile(params.filepath(), os.O_RDONLY, 0)
@@ -67,20 +66,20 @@ func getLargeLog(params *GetLogParams) GetLogResult {
 
 	// Seek to the beginning of the next block to scan
 	partialFirstLineLength := int64(0)
-	for pos > 0 {
-		pos, err = file.Seek(-min(pos, config.SeekBytes), io.SeekCurrent)
+	for pos > 0 && len(result) < maxResultLines {
+		pos, err = file.Seek(-min(pos, config.LargeFileSeekBytes), io.SeekCurrent)
 		if err != nil {
 			return errorGetLogResult(strategy, fmt.Errorf("error seeking to beginning of next block to scan in file '%s': %s", params.filepath(), err))
 		}
 
-		reader := io.NewSectionReader(file, pos, config.SeekBytes+partialFirstLineLength)
+		reader := io.NewSectionReader(file, pos, config.LargeFileSeekBytes+partialFirstLineLength)
 		scanner := bufio.NewScanner(reader)
 
 		// skip the first line (assume it is a partial line) and ensure it will be captured in the next block
 		scanner.Scan()
 		partialFirstLineLength = int64(len(scanner.Text()))
 
-		var blockResult []string
+		blockResult := make([]string, 0, maxResultLines)
 		for scanner.Scan() {
 			line := scanner.Text()
 
@@ -96,12 +95,11 @@ func getLargeLog(params *GetLogParams) GetLogResult {
 			}
 		}
 
-		// TODO: limit result size to params.MaxLines
 		// prepend blockResult to result
 		result = append(blockResult, result...)
 	}
 
-	return successGetLogResult(result, strategy)
+	return successGetLogResult(reduceToMaxLines(result, maxResultLines), strategy)
 }
 
 // getSmallLog returns the contents of a small log file that easily fits in memory
@@ -126,12 +124,21 @@ func getSmallLog(params *GetLogParams) GetLogResult {
 		})
 	}
 
-	// Restrict output to at most maxLines
-	if params.MaxLines > 0 && len(result) > params.MaxLines {
+	return successGetLogResult(reduceToMaxLines(result, maxLines(params)), strategy)
+}
+
+func maxLines(params *GetLogParams) int {
+	if params.MaxLines > 0 {
+		return min(params.MaxLines, config.MaxResultLines)
+	}
+	return config.MaxResultLines
+}
+
+func reduceToMaxLines(result []string, maxLines int) []string {
+	if len(result) > maxLines {
 		endIndex := len(result) - 1
-		startIndex := max(0, endIndex-params.MaxLines)
+		startIndex := max(0, endIndex-maxLines)
 		result = result[startIndex:endIndex]
 	}
-
-	return successGetLogResult(result, strategy)
+	return result
 }
diff --git a/service/getLog_test.go b/service/getLog_test.go
@@ -47,7 +47,7 @@ func ExampleGetLog_logDir1_99lines_log_text_filter() {
 
 func ExampleGetLog_logDir1_99lines_log_regex_filter() {
 	regex := regexp.MustCompile("[9]\\s.z")
-	// Requesting 0 lines returns all available lines.
+	// Requesting 0 lines returns all available lines (up to config.MaxResultLines).
 	getLogResult := GetLog(&GetLogParams{
 		DirectoryPath: "testdata/logDir1",
 		Filename:      "99lines.log",