Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Pipelines #6372

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Pipelines #6372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe05087
Proto changes
tom-andersen Oct 10, 2024
159cb37
Copyright
tom-andersen Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions firebase-firestore/src/proto/google/firestore/v1/document.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,50 @@ message Value {

// A map value.
MapValue map_value = 6;


// Value which references a field.
//
// This is considered relative (vs absolute) since it only refers to a field
// and not a field within a particular document.
//
// **Requires:**
//
// * Must follow [field reference][FieldReference.field_path] limitations.
//
// * Not allowed to be used when writing documents.
//
// (-- NOTE(batchik): long term, there is no reason this type should not be
// allowed to be used on the write path. --)
string field_reference_value = 19;

// A value that represents an unevaluated expression.
//
// **Requires:**
//
// * Not allowed to be used when writing documents.
//
// (-- NOTE(batchik): similar to above, there is no reason to not allow
// storing expressions into the database, just no plan to support in
// the near term.
//
// This would actually be an interesting way to represent user-defined
// functions or more expressive rules-based systems. --)
Function function_value = 20;

// A value that represents an unevaluated pipeline.
//
// **Requires:**
//
// * Not allowed to be used when writing documents.
//
// (-- NOTE(batchik): similar to above, there is no reason to not allow
// storing expressions into the database, just no plan to support in
// the near term.
//
// This would actually be an interesting way to represent user-defined
// functions or more expressive rules-based systems. --)
Pipeline pipeline_value = 21;
}
}

Expand All @@ -149,3 +193,73 @@ message MapValue {
// not exceed 1,500 bytes and cannot be empty.
map<string, Value> fields = 1;
}

// Represents an unevaluated scalar expression.
//
// For example, the expression `like(user_name, "%alice%")` is represented as:
//
// ```
// name: "like"
// args { field_reference: "user_name" }
// args { string_value: "%alice%" }
// ```
//
// (-- api-linter: core::0123::resource-annotation=disabled
// aip.dev/not-precedent: this is not a One Platform API resource. --)
message Function {
// The name of the function to evaluate.
//
// **Requires:**
//
// * must be in snake case (lower case with underscore separator).
//
string name = 1;

// Ordered list of arguments the given function expects.
repeated Value args = 2;

// Optional named arguments that certain functions may support.
map<string, Value> options = 3;
}

// A Firestore query represented as an ordered list of operations / stages.
message Pipeline {
// A single operation within a pipeline.
//
// A stage is made up of a unique name, and a list of arguments. The exact
// number of arguments & types is dependent on the stage type.
//
// To give an example, the stage `filter(state = "MD")` would be encoded as:
//
// ```
// name: "filter"
// args {
// function_value {
// name: "eq"
// args { field_reference_value: "state" }
// args { string_value: "MD" }
// }
// }
// ```
//
// See public documentation for the full list.
message Stage {
// The name of the stage to evaluate.
//
// **Requires:**
//
// * must be in snake case (lower case with underscore separator).
//
string name = 1;

// Ordered list of arguments the given stage expects.
repeated Value args = 2;

// Optional named arguments that certain functions may support.
map<string, Value> options = 3;
}

// Ordered list of stages to evaluate.
repeated Stage stages = 1;
}

84 changes: 84 additions & 0 deletions firebase-firestore/src/proto/google/firestore/v1/firestore.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import "google/api/annotations.proto";
import "google/firestore/v1/aggregation_result.proto";
import "google/firestore/v1/common.proto";
import "google/firestore/v1/document.proto";
import "google/firestore/v1/pipeline.proto";
import "google/firestore/v1/query.proto";
import "google/firestore/v1/write.proto";
import "google/protobuf/empty.proto";
Expand Down Expand Up @@ -138,6 +139,15 @@ service Firestore {
};
}

// Executes a pipeline query.
rpc ExecutePipeline(ExecutePipelineRequest)
returns (stream ExecutePipelineResponse) {
option (google.api.http) = {
post: "/v1beta1/{database=projects/*/databases/*}:executePipeline"
body: "*"
};
}

// Runs an aggregation query.
//
// Rather than producing [Document][google.firestore.v1.Document] results like [Firestore.RunQuery][google.firestore.v1.Firestore.RunQuery],
Expand Down Expand Up @@ -510,6 +520,80 @@ message RunQueryResponse {
int32 skipped_results = 4;
}

// The request for [Firestore.ExecutePipeline][].
message ExecutePipelineRequest {
// Database identifier, in the form `projects/{project}/databases/{database}`.
string database = 1;

oneof pipeline_type {
// A pipelined operation.
StructuredPipeline structured_pipeline = 2;
}

// Optional consistency arguments, defaults to strong consistency.
oneof consistency_selector {
// Run the query within an already active transaction.
//
// The value here is the opaque transaction ID to execute the query in.
bytes transaction = 5;

// Execute the pipeline in a new transaction.
//
// The identifier of the newly created transaction will be returned in the
// first response on the stream. This defaults to a read-only transaction.
TransactionOptions new_transaction = 6;

// Execute the pipeline in a snapshot transaction at the given time.
//
// This must be a microsecond precision timestamp within the past one hour,
// or if Point-in-Time Recovery is enabled, can additionally be a whole
// minute timestamp within the past 7 days.
google.protobuf.Timestamp read_time = 7;
}

// Explain / analyze options for the pipeline.
// ExplainOptions explain_options = 8 [(google.api.field_behavior) = OPTIONAL];
}

// The response for [Firestore.Execute][].
message ExecutePipelineResponse {
// Newly created transaction identifier.
//
// This field is only specified on the first response from the server when
// the request specified [ExecuteRequest.new_transaction][].
bytes transaction = 1;

// An ordered batch of results returned executing a pipeline.
//
// The batch size is variable, and can even be zero for when only a partial
// progress message is returned.
//
// The fields present in the returned documents are only those that were
// explicitly requested in the pipeline, this include those like
// [`__name__`][Document.name] & [`__update_time__`][Document.update_time].
// This is explicitly a divergence from `Firestore.RunQuery` /
// `Firestore.GetDocument` RPCs which always return such fields even when they
// are not specified in the [`mask`][DocumentMask].
repeated Document results = 2;

// The time at which the document(s) were read.
//
// This may be monotonically increasing; in this case, the previous documents
// in the result stream are guaranteed not to have changed between their
// `execution_time` and this one.
//
// If the query returns no results, a response with `execution_time` and no
// `results` will be sent, and this represents the time at which the operation
// was run.
google.protobuf.Timestamp execution_time = 3;

// Query explain metrics.
//
// Set on the last response when [ExecutePipelineRequest.explain_options][]
// was specified on the request.
// ExplainMetrics explain_metrics = 4;
}

// The request for [Firestore.RunAggregationQuery][google.firestore.v1.Firestore.RunAggregationQuery].
message RunAggregationQueryRequest {
// Required. The parent resource name. In the format:
Expand Down
40 changes: 40 additions & 0 deletions firebase-firestore/src/proto/google/firestore/v1/pipeline.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2024 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";
package google.firestore.v1;
import "google/firestore/v1/document.proto";
option csharp_namespace = "Google.Cloud.Firestore.V1";
option php_namespace = "Google\\Cloud\\Firestore\\V1";
option ruby_package = "Google::Cloud::Firestore::V1";
option java_multiple_files = true;
option java_package = "com.google.firestore.v1";
option java_outer_classname = "PipelineProto";
option objc_class_prefix = "GCFS";
// A Firestore query represented as an ordered list of operations / stages.
//
// This is considered the top-level function which plans & executes a query.
// It is logically equivalent to `query(stages, options)`, but prevents the
// client from having to build a function wrapper.
message StructuredPipeline {
// The pipeline query to execute.
Pipeline pipeline = 1;
// Optional query-level arguments.
//
// (-- Think query statement hints. --)
//
// (-- TODO(batchik): define the api contract of using an unsupported hint --)
map<string, Value> options = 2;
}

Loading