regex match with param r

mikebd · Oct 9, 2023 · b8a83ed · b8a83ed
1 parent 56fbe21
commit b8a83ed
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -13,7 +13,7 @@ Some assumptions could be considered TODO items for future enhancement
 * Log events are separated by a newline character
 * REST response Content-Type is text/plain; charset=utf-8, regardless of the Accept request header
 * All errors are returned to REST callers as HTTP status 500, even if these might be correctable by the caller
-* LRU result caching is not implemented
+* LRU result caching of compiled regexes and search results are not implemented
 
 ## Endpoints
 

diff --git a/controller/rest/v1/logs.go b/controller/rest/v1/logs.go
@@ -1,12 +1,14 @@
 package v1
 
 import (
+	"fmt"
 	"github.com/julienschmidt/httprouter"
 	"go-read-var-log/config"
 	"go-read-var-log/controller/rest/util"
 	"go-read-var-log/service"
 	"log"
 	"net/http"
+	"regexp"
 	"slices"
 	"time"
 )
@@ -28,10 +30,20 @@ func GetLog(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 	logFilename := p.ByName("log")
 
 	textMatch := r.URL.Query().Get("q")
+	regexPattern := r.URL.Query().Get("r")
+	var regex *regexp.Regexp
+	if regexPattern != "" {
+		regexCompiled, err := regexp.Compile(regexPattern)
+		if err != nil {
+			util.RenderTextPlain(w, nil, fmt.Errorf("invalid regex pattern '%s': %s", regexPattern, err))
+			return
+		}
+		regex = regexCompiled
+	}
 
 	maxLines, _ := util.PositiveIntParamStrict(w, r, config.GetArguments().NumberOfLogLines, "n")
 	if maxLines >= 0 {
-		logEvents, err := service.GetLog(config.LogDirectory, logFilename, textMatch, maxLines)
+		logEvents, err := service.GetLog(config.LogDirectory, logFilename, textMatch, regex, maxLines)
 
 		if err == nil {
 			// Reverse the slice - we want the most recent events first.

diff --git a/service/getLog.go b/service/getLog.go
@@ -3,33 +3,52 @@ package service
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"slices"
 	"strings"
 )
 
 // GetLog returns the contents of a log file
 // directoryPath: the path to the directory containing the log file
 // filename: the name of the log file
-// textMatch: a string to search for in the log file (case sensitive)
-// maxLines: the maximum number of lines to return
-func GetLog(directoryPath string, filename string, textMatch string, maxLines int) ([]string, error) {
+// textMatch: a string to search for in the log file (case-sensitive, empty string if not required)
+// regex: a compiled regular expression to search for in the log file (nil if not required)
+// maxLines: the maximum number of lines to return (0 for all lines)
+func GetLog(directoryPath string, filename string, textMatch string, regex *regexp.Regexp, maxLines int) ([]string, error) {
+	// TODO - REFACTOR: Consider a struct to hold the arguments to this function if the number of parameters grows
+
 	filepath := strings.Join([]string{directoryPath, filename}, "/")
 
 	if !validLogFromName(directoryPath, filename) {
 		return nil, fmt.Errorf("invalid, unreadable or unsupported log file '%s'", filepath)
 	}
 
-	// TODO: this is the simplest possible approach.  It will likely not work well for large files.
+	// TODO: This is the simplest possible approach.  It will likely not work well for extremely large files.
+	//       Consider seek() near the end of the file, backwards iteratively, until the desired number of lines is found.
+	//       This will be more efficient for large files, but will be more complex to implement and maintain.
+	//	     On my machine:
+	//	     - First scan of a 1GB file with 10.5 million lines takes ≈ 2-3µs returning all (1) lines matching both
+	//         a textMatch and regex.
+	//       - Subsequent scans of the same file for a different textMatch and regex, returning all (1) matching lines,
+	//         takes ≈ 1-1.3µs.  This is likely due to the file fitting in the filesystem page cache on my system.
+	//           kern.vm_page_free_min: 3500
+	//           kern.vm_page_free_reserved: 912
+	//           kern.vm_page_free_target: 4000
 	byteSlice, err := os.ReadFile(filepath)
 	if err != nil {
 		return nil, err
 	}
 	result := strings.Split(string(byteSlice), "\n")
 
 	// Filter out lines that do not match the filters
-	if textMatch != "" {
+	textMatchRequested := textMatch != ""
+	regexMatchRequested := regex != nil
+	if textMatchRequested || regexMatchRequested {
+		// Single pass through the slice for efficiency
 		result = slices.DeleteFunc(result, func(line string) bool {
-			return !strings.Contains(line, textMatch)
+			textMatchFailed := textMatchRequested && !strings.Contains(line, textMatch)
+			regexMatchFailed := regexMatchRequested && !regex.MatchString(line)
+			return textMatchFailed || regexMatchFailed
 		})
 	}
 

diff --git a/service/getLog_test.go b/service/getLog_test.go
@@ -2,22 +2,23 @@ package service
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 )
 
 // Log output is in natural order at the service layer.
 // Controllers may reverse the order as required by their clients.
 
 func ExampleGetLog_logDir1_10KiB_log() {
-	lines, _ := GetLog("testdata/logDir1", "10KiB.log", "", 2)
+	lines, _ := GetLog("testdata/logDir1", "10KiB.log", "", nil, 2)
 	fmt.Println(strings.Join(lines, "\n"))
 	// Output:
 	// 2023-10-06T15:18:24.408740Z|info |olaret esanus ivo hey enug tewos ebad it u tuge po elora e iwemat o
 	// 2023-10-06T15:18:24.408762Z|debug|tucev uho e u ela opif ce igodeto hudegor ivosu ehab eaunopi balohan tagused gicefas
 }
 
-func ExampleGetLog_logDir1_99lines_log() {
-	lines, _ := GetLog("testdata/logDir1", "99lines.log", "9 ", 0)
+func ExampleGetLog_logDir1_99lines_log_text_filter() {
+	lines, _ := GetLog("testdata/logDir1", "99lines.log", "9 ", nil, 0)
 	fmt.Println(strings.Join(lines, "\n"))
 	// Output:
 	// line 9 yz
@@ -32,9 +33,36 @@ func ExampleGetLog_logDir1_99lines_log() {
 	// line 99 yz
 }
 
+func ExampleGetLog_logDir1_99lines_log_regex_filter() {
+	regex := regexp.MustCompile("[9]\\s.z")
+	// Requesting 0 lines returns all available lines.
+	lines, _ := GetLog("testdata/logDir1", "99lines.log", "", regex, 0)
+	fmt.Println(strings.Join(lines, "\n"))
+	// Output:
+	// line 9 yz
+	// line 19 yz
+	// line 29 yz
+	// line 39 yz
+	// line 49 yz
+	// line 59 yz
+	// line 69 yz
+	// line 79 yz
+	// line 89 yz
+	// line 99 yz
+}
+
+func ExampleGetLog_logDir1_99lines_log_text_and_regex_filter() {
+	regex := regexp.MustCompile("[9]\\s.z")
+	// Requesting 0 lines returns all available lines.
+	lines, _ := GetLog("testdata/logDir1", "99lines.log", "7", regex, 0)
+	fmt.Println(strings.Join(lines, "\n"))
+	// Output:
+	// line 79 yz
+}
+
 func ExampleGetLog_logDir1_1line_log() {
 	// Requesting more lines than available returns all available lines.
-	lines, _ := GetLog("testdata/logDir1", "1line.log", "", 10)
+	lines, _ := GetLog("testdata/logDir1", "1line.log", "", nil, 10)
 	fmt.Println(strings.Join(lines, "\n"))
 	// Output:
 	// 2023-10-06T15:18:24.406350Z|debug|toyeni vate riwehu ato ped afe ral bo h redi esohet sir moyireh nema lidef