Skip to content

Commit 5995e61

Browse files
kyleconroykyleconroy
authored
test(endtoend): Verify all schemas in endtoend (#2744)
* test(endtoend): Verify all schemas in endtoend For each test in the endtoend package, use the new sqlc managed databases to verify that the schemas are valid. In the future, we'll attempt to prepare all the queries as well. * Fix CI * Remove skip verify
1 parent 33398b7 commit 5995e61

File tree

4 files changed

+701
-0
lines changed

4 files changed

+701
-0
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ jobs:
5555
with:
5656
go-version: '1.21.1'
5757

58+
- run: curl -I https://debug.fly.dev
59+
5860
- name: install gotestsum
5961
run: go install gotest.tools/gotestsum@latest
6062

@@ -76,6 +78,8 @@ jobs:
7678
MYSQL_HOST: localhost
7779
MYSQL_PORT: ${{ job.services.mysql.ports['3306'] }}
7880
MYSQL_ROOT_PASSWORD: mysecretpassword
81+
DDL_SQLC_PROJECT_ID: ${{ secrets.DDL_SQLC_PROJECT_ID }}
82+
DDL_SQLC_AUTH_TOKEN: ${{ secrets.DDL_SQLC_AUTH_TOKEN }}
7983

8084
- name: build internal/endtoend
8185
run: go build ./...

‎internal/endtoend/ddl_test.go‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io/fs"
7+
"os"
8+
"path/filepath"
9+
"strings"
10+
"testing"
11+
"time"
12+
13+
"github.com/jackc/pgx/v5"
14+
15+
"github.com/sqlc-dev/sqlc/internal/config"
16+
"github.com/sqlc-dev/sqlc/internal/migrations"
17+
"github.com/sqlc-dev/sqlc/internal/quickdb"
18+
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
19+
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
20+
)
21+
22+
func TestValidSchema(t *testing.T) {
23+
ctx := context.Background()
24+
25+
projectID := os.Getenv("DDL_SQLC_PROJECT_ID")
26+
authToken := os.Getenv("DDL_SQLC_AUTH_TOKEN")
27+
28+
if projectID == "" || authToken == "" {
29+
if os.Getenv("CI") == "" {
30+
t.Skip("skiping ddl tests outside of CI")
31+
} else {
32+
t.Fatal("missing project id and auth token")
33+
}
34+
}
35+
36+
client, err := quickdb.NewClient(projectID, authToken)
37+
if err != nil {
38+
t.Fatal(err)
39+
}
40+
41+
files := []string{}
42+
43+
// Find all tests that do not have a stderr.txt file
44+
err = filepath.Walk("testdata", func(path string, info fs.FileInfo, err error) error {
45+
if err != nil {
46+
return err
47+
}
48+
if filepath.Base(path) == "sqlc.json" || filepath.Base(path) == "sqlc.yaml" {
49+
stderr := filepath.Join(filepath.Dir(path), "stderr.txt")
50+
if _, err := os.Stat(stderr); !os.IsNotExist(err) {
51+
return nil
52+
}
53+
files = append(files, path)
54+
}
55+
return nil
56+
})
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
61+
for _, file := range files {
62+
file := file // https://golang.org/doc/faq#closures_and_goroutines
63+
rd, err := os.Open(file)
64+
if err != nil {
65+
t.Fatal(err)
66+
}
67+
68+
conf, err := config.ParseConfig(rd)
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
73+
for j, pkg := range conf.SQL {
74+
j, pkg := j, pkg
75+
if pkg.Engine != config.EnginePostgreSQL {
76+
continue
77+
}
78+
t.Run(fmt.Sprintf("endtoend-%s-%d", file, j), func(t *testing.T) {
79+
t.Parallel()
80+
81+
var schema []string
82+
for _, path := range pkg.Schema {
83+
schema = append(schema, filepath.Join(filepath.Dir(file), path))
84+
}
85+
86+
files, err := sqlpath.Glob(schema)
87+
if err != nil {
88+
t.Fatal(err)
89+
}
90+
91+
var sqls []string
92+
for _, f := range files {
93+
contents, err := os.ReadFile(f)
94+
if err != nil {
95+
t.Fatalf("%s: %s", f, err)
96+
}
97+
// TODO: Split schema into separate files
98+
before, _, _ := strings.Cut(string(contents), "-- name:")
99+
before, _, _ = strings.Cut(before, "/* name:")
100+
// Support loading pg_dump SQL files
101+
before = strings.ReplaceAll(before, "CREATE SCHEMA public;", "CREATE SCHEMA IF NOT EXISTS public;")
102+
sqls = append(sqls, migrations.RemoveRollbackStatements(before))
103+
}
104+
105+
start := time.Now()
106+
resp, err := client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{
107+
Engine: "postgresql",
108+
Region: "iad",
109+
Migrations: sqls,
110+
})
111+
t.Logf("%s", time.Since(start))
112+
if err != nil {
113+
t.Fatal(err)
114+
}
115+
116+
t.Cleanup(func() {
117+
_, err = client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{
118+
DatabaseId: resp.DatabaseId,
119+
})
120+
if err != nil {
121+
t.Fatal(err)
122+
}
123+
})
124+
125+
conn, err := pgx.Connect(ctx, resp.Uri)
126+
if err != nil {
127+
t.Fatalf("connect %s: %s", resp.Uri, err)
128+
}
129+
defer conn.Close(ctx)
130+
})
131+
}
132+
}
133+
}

‎internal/quickdb/rpc.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package quickdb
2+
3+
import (
4+
"crypto/tls"
5+
"os"
6+
7+
"github.com/riza-io/grpc-go/credentials/basic"
8+
"google.golang.org/grpc"
9+
"google.golang.org/grpc/credentials"
10+
11+
"github.com/sqlc-dev/sqlc/internal/config"
12+
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
13+
)
14+
15+
const defaultHostname = "grpc.sqlc.dev"
16+
17+
func NewClientFromConfig(cloudConfig config.Cloud) (pb.QuickClient, error) {
18+
projectID := cloudConfig.Project
19+
authToken := os.Getenv("SQLC_AUTH_TOKEN")
20+
return NewClient(projectID, authToken, WithHost(cloudConfig.Hostname))
21+
}
22+
23+
type options struct {
24+
hostname string
25+
}
26+
27+
type Option func(*options)
28+
29+
func WithHost(host string) Option {
30+
return func(o *options) {
31+
o.hostname = host
32+
}
33+
}
34+
35+
func NewClient(project, token string, opts ...Option) (pb.QuickClient, error) {
36+
var o options
37+
for _, apply := range opts {
38+
apply(&o)
39+
}
40+
41+
dialOpts := []grpc.DialOption{
42+
grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})),
43+
grpc.WithPerRPCCredentials(basic.NewPerRPCCredentials(project, token)),
44+
}
45+
46+
hostname := o.hostname
47+
if hostname == "" {
48+
hostname = defaultHostname
49+
}
50+
51+
conn, err := grpc.Dial(hostname+":443", dialOpts...)
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
return pb.NewQuickClient(conn), nil
57+
}

0 commit comments

Comments
 (0)