Skip to content

Commit 11a82f0

Browse files
kyleconroykyleconroy
authored
feat(analyzer): Cache query analysis (#2889)
* feat(analyzer): Cache query analysis When using managed databases, cache the query analysis if the query, schema and configuration file hasn't change. Also take into account the version of sqlc. Analysis can only be cached for managed databases as we can't know if a connected database has been changed.
1 parent ca9ec01 commit 11a82f0

File tree

10 files changed

+2870
-67
lines changed

10 files changed

+2870
-67
lines changed

‎internal/analysis/analysis.pb.go‎

Lines changed: 546 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/analysis/analysis_vtproto.pb.go‎

Lines changed: 2078 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/analyzer/analyzer.go‎

Lines changed: 98 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,45 +2,115 @@ package analyzer
22

33
import (
44
"context"
5+
"encoding/json"
6+
"fmt"
7+
"hash/fnv"
8+
"log/slog"
9+
"os"
10+
"path/filepath"
511

12+
"google.golang.org/protobuf/proto"
13+
14+
"github.com/sqlc-dev/sqlc/internal/analysis"
15+
"github.com/sqlc-dev/sqlc/internal/cache"
16+
"github.com/sqlc-dev/sqlc/internal/config"
17+
"github.com/sqlc-dev/sqlc/internal/info"
618
"github.com/sqlc-dev/sqlc/internal/sql/ast"
719
"github.com/sqlc-dev/sqlc/internal/sql/named"
820
)
921

10-
type Column struct {
11-
Name string
12-
OriginalName string
13-
DataType string
14-
NotNull bool
15-
Unsigned bool
16-
IsArray bool
17-
ArrayDims int
18-
Comment string
19-
Length *int
20-
IsNamedParam bool
21-
IsFuncCall bool
22-
23-
// XXX: Figure out what PostgreSQL calls `foo.id`
24-
Scope string
25-
Table *ast.TableName
26-
TableAlias string
27-
Type *ast.TypeName
28-
EmbedTable *ast.TableName
29-
30-
IsSqlcSlice bool // is this sqlc.slice()
22+
type CachedAnalyzer struct {
23+
a Analyzer
24+
config config.Config
25+
configBytes []byte
26+
db config.Database
27+
}
28+
29+
func Cached(a Analyzer, c config.Config, db config.Database) *CachedAnalyzer {
30+
return &CachedAnalyzer{
31+
a: a,
32+
config: c,
33+
db: db,
34+
}
3135
}
3236

33-
type Parameter struct {
34-
Number int
35-
Column *Column
37+
// Create a new error here
38+
39+
func (c *CachedAnalyzer) Analyze(ctx context.Context, n ast.Node, q string, schema []string, np *named.ParamSet) (*analysis.Analysis, error) {
40+
result, rerun, err := c.analyze(ctx, n, q, schema, np)
41+
if rerun {
42+
if err != nil {
43+
slog.Warn("first analysis failed with error", "err", err)
44+
}
45+
return c.a.Analyze(ctx, n, q, schema, np)
46+
}
47+
return result, err
48+
}
49+
50+
func (c *CachedAnalyzer) analyze(ctx context.Context, n ast.Node, q string, schema []string, np *named.ParamSet) (*analysis.Analysis, bool, error) {
51+
// Only cache queries for managed databases. We can't be certain the the
52+
// database is in an unchanged state otherwise
53+
if !c.db.Managed {
54+
return nil, true, nil
55+
}
56+
57+
dir, err := cache.AnalysisDir()
58+
if err != nil {
59+
return nil, true, err
60+
}
61+
62+
if c.configBytes == nil {
63+
c.configBytes, err = json.Marshal(c.config)
64+
if err != nil {
65+
return nil, true, err
66+
}
67+
}
68+
69+
// Calculate cache key
70+
h := fnv.New64()
71+
h.Write([]byte(info.Version))
72+
h.Write(c.configBytes)
73+
for _, m := range schema {
74+
h.Write([]byte(m))
75+
}
76+
h.Write([]byte(q))
77+
78+
key := fmt.Sprintf("%x", h.Sum(nil))
79+
path := filepath.Join(dir, key)
80+
if _, err := os.Stat(path); err == nil {
81+
contents, err := os.ReadFile(path)
82+
if err != nil {
83+
return nil, true, err
84+
}
85+
var a analysis.Analysis
86+
if err := proto.Unmarshal(contents, &a); err != nil {
87+
return nil, true, err
88+
}
89+
return &a, false, nil
90+
}
91+
92+
result, err := c.a.Analyze(ctx, n, q, schema, np)
93+
94+
if err == nil {
95+
contents, err := proto.Marshal(result)
96+
if err != nil {
97+
slog.Warn("unable to marshal analysis", "err", err)
98+
return result, false, nil
99+
}
100+
if err := os.WriteFile(path, contents, 0644); err != nil {
101+
slog.Warn("saving analysis to disk failed", "err", err)
102+
return result, false, nil
103+
}
104+
}
105+
106+
return result, false, err
36107
}
37108

38-
type Analysis struct {
39-
Columns []Column
40-
Params []Parameter
109+
func (c *CachedAnalyzer) Close(ctx context.Context) error {
110+
return c.a.Close(ctx)
41111
}
42112

43113
type Analyzer interface {
44-
Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*Analysis, error)
114+
Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*analysis.Analysis, error)
45115
Close(context.Context) error
46116
}

‎internal/cache/cache.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package cache
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
)
8+
9+
// The cache directory defaults to os.UserCacheDir(). This location can be
10+
// overridden by the SQLCCACHE environment variable.
11+
//
12+
// Currently the cache stores two types of data: plugins and query analysis
13+
func Dir() (string, error) {
14+
cache := os.Getenv("SQLCCACHE")
15+
if cache != "" {
16+
return cache, nil
17+
}
18+
cacheHome, err := os.UserCacheDir()
19+
if err != nil {
20+
return "", err
21+
}
22+
return filepath.Join(cacheHome, "sqlc"), nil
23+
}
24+
25+
func PluginsDir() (string, error) {
26+
cacheRoot, err := Dir()
27+
if err != nil {
28+
return "", err
29+
}
30+
dir := filepath.Join(cacheRoot, "plugins")
31+
if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) {
32+
return "", fmt.Errorf("failed to create %s directory: %w", dir, err)
33+
}
34+
return dir, nil
35+
}
36+
37+
func AnalysisDir() (string, error) {
38+
cacheRoot, err := Dir()
39+
if err != nil {
40+
return "", err
41+
}
42+
dir := filepath.Join(cacheRoot, "query_analysis")
43+
if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) {
44+
return "", fmt.Errorf("failed to create %s directory: %w", dir, err)
45+
}
46+
return dir, nil
47+
}

‎internal/compiler/analyze.go‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package compiler
33
import (
44
"sort"
55

6-
"github.com/sqlc-dev/sqlc/internal/analyzer"
6+
analyzer "github.com/sqlc-dev/sqlc/internal/analysis"
77
"github.com/sqlc-dev/sqlc/internal/config"
88
"github.com/sqlc-dev/sqlc/internal/source"
99
"github.com/sqlc-dev/sqlc/internal/sql/ast"
@@ -13,32 +13,54 @@ import (
1313
)
1414

1515
type analysis struct {
16-
Table *ast.TableName
17-
Columns []*Column
18-
QueryCatalog *QueryCatalog
19-
Parameters []Parameter
20-
Named *named.ParamSet
21-
Query string
16+
Table *ast.TableName
17+
Columns []*Column
18+
Parameters []Parameter
19+
Named *named.ParamSet
20+
Query string
2221
}
2322

24-
func convertColumn(c analyzer.Column) *Column {
23+
func convertTableName(id *analyzer.Identifier) *ast.TableName {
24+
if id == nil {
25+
return nil
26+
}
27+
return &ast.TableName{
28+
Catalog: id.Catalog,
29+
Schema: id.Schema,
30+
Name: id.Name,
31+
}
32+
}
33+
34+
func convertTypeName(id *analyzer.Identifier) *ast.TypeName {
35+
if id == nil {
36+
return nil
37+
}
38+
return &ast.TypeName{
39+
Catalog: id.Catalog,
40+
Schema: id.Schema,
41+
Name: id.Name,
42+
}
43+
}
44+
45+
func convertColumn(c *analyzer.Column) *Column {
46+
length := int(c.Length)
2547
return &Column{
2648
Name: c.Name,
2749
OriginalName: c.OriginalName,
2850
DataType: c.DataType,
2951
NotNull: c.NotNull,
3052
Unsigned: c.Unsigned,
3153
IsArray: c.IsArray,
32-
ArrayDims: c.ArrayDims,
54+
ArrayDims: int(c.ArrayDims),
3355
Comment: c.Comment,
34-
Length: c.Length,
56+
Length: &length,
3557
IsNamedParam: c.IsNamedParam,
3658
IsFuncCall: c.IsFuncCall,
3759
Scope: c.Scope,
38-
Table: c.Table,
60+
Table: convertTableName(c.Table),
3961
TableAlias: c.TableAlias,
40-
Type: c.Type,
41-
EmbedTable: c.EmbedTable,
62+
Type: convertTypeName(c.Type),
63+
EmbedTable: convertTableName(c.EmbedTable),
4264
IsSqlcSlice: c.IsSqlcSlice,
4365
}
4466
}
@@ -51,8 +73,8 @@ func combineAnalysis(prev *analysis, a *analyzer.Analysis) *analysis {
5173
var params []Parameter
5274
for _, p := range a.Params {
5375
params = append(params, Parameter{
54-
Number: p.Number,
55-
Column: convertColumn(*p.Column),
76+
Number: int(p.Number),
77+
Column: convertColumn(p.Column),
5678
})
5779
}
5880
if len(prev.Columns) == len(cols) {
@@ -189,11 +211,10 @@ func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool)
189211
}
190212

191213
return &analysis{
192-
Table: table,
193-
Columns: cols,
194-
Parameters: params,
195-
QueryCatalog: qc,
196-
Query: expanded,
197-
Named: namedParams,
214+
Table: table,
215+
Columns: cols,
216+
Parameters: params,
217+
Query: expanded,
218+
Named: namedParams,
198219
}, rerr
199220
}

‎internal/compiler/engine.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,11 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err
5151
c.catalog = postgresql.NewCatalog()
5252
if conf.Database != nil {
5353
if conf.Analyzer.Database == nil || *conf.Analyzer.Database {
54-
c.analyzer = pganalyze.New(c.client, *conf.Database)
54+
c.analyzer = analyzer.Cached(
55+
pganalyze.New(c.client, *conf.Database),
56+
combo.Global,
57+
*conf.Database,
58+
)
5559
}
5660
}
5761
default:

‎internal/compiler/parse.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
7070

7171
var anlys *analysis
7272
if c.analyzer != nil {
73-
// TODO: Handle panics
7473
inference, _ := c.inferQuery(raw, rawSQL)
7574
if inference == nil {
7675
inference = &analysis{}

‎internal/engine/postgresql/analyzer/analyze.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/jackc/pgx/v5/pgconn"
1212
"github.com/jackc/pgx/v5/pgxpool"
1313

14-
core "github.com/sqlc-dev/sqlc/internal/analyzer"
14+
core "github.com/sqlc-dev/sqlc/internal/analysis"
1515
"github.com/sqlc-dev/sqlc/internal/config"
1616
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
1717
"github.com/sqlc-dev/sqlc/internal/sql/ast"
@@ -250,14 +250,14 @@ func (a *Analyzer) Analyze(ctx context.Context, n ast.Node, query string, migrat
250250
dt, isArray, _ := parseType(col.DataType)
251251
notNull := col.NotNull
252252
name := field.Name
253-
result.Columns = append(result.Columns, core.Column{
253+
result.Columns = append(result.Columns, &core.Column{
254254
Name: name,
255255
OriginalName: field.Name,
256256
DataType: dt,
257257
NotNull: notNull,
258258
IsArray: isArray,
259-
ArrayDims: col.ArrayDims,
260-
Table: &ast.TableName{
259+
ArrayDims: int32(col.ArrayDims),
260+
Table: &core.Identifier{
261261
Schema: tbl.SchemaName,
262262
Name: tbl.TableName,
263263
},
@@ -271,13 +271,13 @@ func (a *Analyzer) Analyze(ctx context.Context, n ast.Node, query string, migrat
271271
notNull := false
272272
name := field.Name
273273
dt, isArray, dims := parseType(dataType)
274-
result.Columns = append(result.Columns, core.Column{
274+
result.Columns = append(result.Columns, &core.Column{
275275
Name: name,
276276
OriginalName: field.Name,
277277
DataType: dt,
278278
NotNull: notNull,
279279
IsArray: isArray,
280-
ArrayDims: dims,
280+
ArrayDims: int32(dims),
281281
})
282282
}
283283
}
@@ -293,13 +293,13 @@ func (a *Analyzer) Analyze(ctx context.Context, n ast.Node, query string, migrat
293293
if ps != nil {
294294
name, _ = ps.NameFor(i + 1)
295295
}
296-
result.Params = append(result.Params, core.Parameter{
297-
Number: i + 1,
296+
result.Params = append(result.Params, &core.Parameter{
297+
Number: int32(i + 1),
298298
Column: &core.Column{
299299
Name: name,
300300
DataType: dt,
301301
IsArray: isArray,
302-
ArrayDims: dims,
302+
ArrayDims: int32(dims),
303303
NotNull: notNull,
304304
},
305305
})

0 commit comments

Comments
 (0)