cloudquery · kodiakhq · Apr 8, 2024 · Apr 5, 2024 · Apr 8, 2024 · Apr 8, 2024
@@ -33,6 +33,8 @@ type Client struct {
 	orgs            []string
 	orgRepositories map[string][]*github.Repository
 	repos           []string
+
+	Spec Spec
 }
 
 func (c *Client) Logger() *zerolog.Logger {
@@ -139,6 +141,7 @@ func New(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.ClientMe
 		orgServices: ghServices,
 		orgs:        spec.Orgs,
 		repos:       spec.Repos,
+		Spec:        spec,
 	}
 	c.logger.Info().Msg("Discovering repositories")
 	orgRepositories, err := c.discoverRepositories(ctx, spec.DiscoveryConcurrency, spec.Orgs, spec.Repos, spec.IncludeArchivedRepos)

@@ -4,8 +4,10 @@ import (
 	_ "embed"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/invopop/jsonschema"
+	"github.com/tj/go-naturaldate"
 )
 
 // Spec is the (nested) spec used by GitHub Source Plugin
@@ -28,6 +30,21 @@ type Spec struct {
 	IncludeArchivedRepos bool `json:"include_archived_repos,omitempty"`
 	// Path to a local directory that will hold the cache. If set, the plugin will cache the GitHub API responses in this directory. Defaults to an empty string (no cache)
 	LocalCachePath string `json:"local_cache_path,omitempty"`
+
+	// Table options to set for specific tables.
+	TableOptions TableOptions `json:"table_options,omitempty"`
+}
+
+type TableOptions struct {
+	// Table options for the github_workflow_runs table.
+	WorkflowRuns WorkflowRunsOptions `json:"github_workflow_runs,omitempty"`
+}
+
+type WorkflowRunsOptions struct {
+	// Time to look back for workflow runs in natural date format. Defaults to all workflows.
+	// Examples: "14 days ago", "last month"
+	CreatedSince    string `json:"created_since,omitempty" jsonschema:"example=14 days ago,example=last month"`
+	ParsedTimeSince string `json:"-"`
 }
 
 type EnterpriseSettings struct {
@@ -99,6 +116,13 @@ func (s *Spec) Validate() error {
 			return fmt.Errorf("local cache path is not a directory")
 		}
 	}
+	if s.TableOptions.WorkflowRuns.CreatedSince != "" {
+		parsedTimeSince, err := naturaldate.Parse(s.TableOptions.WorkflowRuns.CreatedSince, time.Now(), naturaldate.WithDirection(naturaldate.Past))
+		if err != nil {
+			return fmt.Errorf("failed to parse created_since: %w", err)
+		}
+		s.TableOptions.WorkflowRuns.ParsedTimeSince = parsedTimeSince.Format(time.RFC3339)
+	}
 	return nil
 }
 

@@ -45,3 +45,16 @@ This is the (nested) spec used by GitHub Source Plugin
 - `local_cache_path` (`string`, optional, default: empty):
   Path to a local directory that will hold the cache. If set, the plugin will cache the GitHub API responses in this directory. Defaults to an empty string (no cache).
   By using a cache, the plugin can use [conditional requests when appropriate](https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api?#use-conditional-requests-if-appropriate), and help avoid hitting GitHub API rate limits.
+
+- `table_options` ([Table Options](#github-table-options-spec) spec) (optional)
+
+  Options to apply to specific tables. See [Table Options](#Table Options) for more information.
+
+### GitHub Table Options Spec
+
+- `github_workflow_runs`
+
+  - `created_since` (`string` in natural date format) (optional)
+
+    Sync only workflow runs created after this date (inclusive). Defaults to all workflows.
+    Examples of valid formats are: `7 days ago`, `last month` (see more [here](https://github.com/tj/go-naturaldate?tab=readme-ov-file#go-natural-date))
diff --git a/plugins/source/github/go.mod b/plugins/source/github/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/google/go-github/v59 v59.0.0
 	github.com/invopop/jsonschema v0.12.0
 	github.com/rs/zerolog v1.32.0
+	github.com/tj/go-naturaldate v1.3.0
 	golang.org/x/sync v0.6.0
 )
 

diff --git a/plugins/source/github/go.sum b/plugins/source/github/go.sum
@@ -268,6 +268,10 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03
 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
 github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
 github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160 h1:NSWpaDaurcAJY7PkL8Xt0PhZE7qpvbZl5ljd8r6U0bI=
+github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/go-naturaldate v1.3.0 h1:OgJIPkR/Jk4bFMBLbxZ8w+QUxwjqSvzd9x+yXocY4RI=
+github.com/tj/go-naturaldate v1.3.0/go.mod h1:rpUbjivDKiS1BlfMGc2qUKNZ/yxgthOfmytQs8d8hKk=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=

@@ -2,6 +2,7 @@ package actions
 
 import (
 	"context"
+	"time"
 
 	"github.com/cloudquery/cloudquery/plugins/source/github/client"
 	"github.com/cloudquery/plugin-sdk/v4/schema"
@@ -31,11 +32,31 @@ func fetchWorkflowRuns(ctx context.Context, meta schema.ClientMeta, _ *schema.Re
 	repo := c.Repository
 	listOpts := &github.ListOptions{PerPage: 100}
 	actionOpts := &github.ListWorkflowRunsOptions{ListOptions: *listOpts}
+
+	if c.Spec.TableOptions.WorkflowRuns.ParsedTimeSince != "" {
+		actionOpts.Created = ">=" + c.Spec.TableOptions.WorkflowRuns.ParsedTimeSince
+	}
+	var earliestCreatedAt string
 	for {
 		workflowRuns, resp, err := c.Github.Actions.ListRepositoryWorkflowRuns(ctx, *repo.Owner.Login, *repo.Name, actionOpts)
 		if err != nil {
 			return err
 		}
+		// When setting created_at, the API will return a maximum of 1000 records, so if we reached the limit we need to get the other records
+		if len(workflowRuns.WorkflowRuns) == 0 && actionOpts.Page > 0 && actionOpts.Created != "" {
+			actionOpts.Page = 0
+			// Workflows runs are sorted by created_at in descending order, so we need to scope down the query "back in time"
+			// We use the earliest created_at to set it as the upper bound of the query so
+			// First 1000 results are the most recent ones, the next 1000 are the ones before that, and so on
+			actionOpts.Created = c.Spec.TableOptions.WorkflowRuns.ParsedTimeSince + ".." + earliestCreatedAt
+			workflowRuns, resp, err = c.Github.Actions.ListRepositoryWorkflowRuns(ctx, *repo.Owner.Login, *repo.Name, actionOpts)
+			if err != nil {
+				return err
+			}
+		}
+		if len(workflowRuns.WorkflowRuns) > 0 {
+			earliestCreatedAt = workflowRuns.WorkflowRuns[len(workflowRuns.WorkflowRuns)-1].GetCreatedAt().Format(time.RFC3339)
+		}
 		res <- workflowRuns.WorkflowRuns
 
 		if resp.NextPage == 0 {