Skip to content

Commit 00910bc

Browse files
tokokotokoko
authored
chore: Commit generated python proto files (feast-dev#4546)
* chore: commit generated python files to repo Signed-off-by: tokoko <[email protected]> * merge from master Signed-off-by: tokoko <[email protected]> * chore: remove protos from gitignore Signed-off-by: tokoko <[email protected]> --------- Signed-off-by: tokoko <[email protected]>
1 parent fcd1bd4 commit 00910bc

114 files changed

Lines changed: 9568 additions & 665 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,6 @@ dmypy.json
185185

186186
# Protos
187187
sdk/python/docs/html
188-
sdk/python/feast/protos/
189188
sdk/go/protos/
190189
go/protos/
191190

‎Makefile‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,29 +40,20 @@ build: protos build-java build-docker
4040
install-python-ci-dependencies:
4141
python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
4242
pip install --no-deps -e .
43-
python setup.py build_python_protos --inplace
4443

4544
install-python-ci-dependencies-uv:
4645
uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
4746
uv pip install --system --no-deps -e .
48-
python setup.py build_python_protos --inplace
4947

5048
install-python-ci-dependencies-uv-venv:
5149
uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
5250
uv pip install --no-deps -e .
53-
python setup.py build_python_protos --inplace
54-
55-
install-protoc-dependencies:
56-
pip install "protobuf<5" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1"
5751

5852
lock-python-ci-dependencies:
5953
uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt
6054

61-
package-protos:
62-
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
63-
64-
compile-protos-python: install-protoc-dependencies
65-
python setup.py build_python_protos --inplace
55+
compile-protos-python:
56+
python infra/scripts/generate_protos.py
6657

6758
install-python:
6859
python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-requirements.txt

‎infra/scripts/generate_protos.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import os
2+
import sys
3+
import glob
4+
import subprocess
5+
from pathlib import Path
6+
7+
repo_root = str(Path(__file__).resolve().parent)
8+
9+
PROTO_SUBDIRS = ["core", "registry", "serving", "types", "storage"]
10+
PYTHON_CODE_PREFIX = "sdk/python"
11+
12+
class BuildPythonProtosCommand:
13+
description = "Builds the proto files into Python files."
14+
user_options = [
15+
("inplace", "i", "Write generated proto files to source directory."),
16+
]
17+
18+
def __init__(self):
19+
self.python_protoc = [
20+
sys.executable,
21+
"-m",
22+
"grpc_tools.protoc",
23+
]
24+
self.proto_folder = "protos"
25+
self.sub_folders = PROTO_SUBDIRS
26+
self.inplace = 0
27+
28+
@property
29+
def python_folder(self):
30+
return "sdk/python/feast/protos"
31+
32+
def _generate_python_protos(self, path: str):
33+
proto_files = glob.glob(os.path.join(self.proto_folder, path))
34+
Path(self.python_folder).mkdir(parents=True, exist_ok=True)
35+
subprocess.check_call(
36+
self.python_protoc
37+
+ [
38+
"-I",
39+
self.proto_folder,
40+
"--python_out",
41+
self.python_folder,
42+
"--grpc_python_out",
43+
self.python_folder,
44+
"--mypy_out",
45+
self.python_folder,
46+
]
47+
+ proto_files
48+
)
49+
50+
def run(self):
51+
for sub_folder in self.sub_folders:
52+
self._generate_python_protos(f"feast/{sub_folder}/*.proto")
53+
# We need the __init__ files for each of the generated subdirs
54+
# so that they are regular packages, and don't need the `--namespace-packages` flags
55+
# when being typechecked using mypy.
56+
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"):
57+
pass
58+
59+
with open(f"{self.python_folder}/__init__.py", "w"):
60+
pass
61+
with open(f"{self.python_folder}/feast/__init__.py", "w"):
62+
pass
63+
64+
for path in Path(self.python_folder).rglob("*.py"):
65+
for folder in self.sub_folders:
66+
# Read in the file
67+
with open(path, "r") as file:
68+
filedata = file.read()
69+
70+
# Replace the target string
71+
filedata = filedata.replace(
72+
f"from feast.{folder}", f"from feast.protos.feast.{folder}"
73+
)
74+
75+
# Write the file out again
76+
with open(path, "w") as file:
77+
file.write(filedata)
78+
79+
if __name__ == "__main__":
80+
BuildPythonProtosCommand().run()

‎pyproject.toml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
[build-system]
22
requires = [
3-
"protobuf<5",
4-
"grpcio-tools>=1.56.2,<2",
5-
"mypy-protobuf>=3.1",
63
"pybindgen==0.22.0",
74
"setuptools>=60",
85
"setuptools_scm>=6.2",

‎sdk/python/feast/protos/__init__.py‎

Whitespace-only changes.

‎sdk/python/feast/protos/feast/__init__.py‎

Whitespace-only changes.

‎sdk/python/feast/protos/feast/core/Aggregation_pb2.py‎

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/python/feast/protos/feast/core/Aggregation_pb2.pyi‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""
2+
@generated by mypy-protobuf. Do not edit manually!
3+
isort:skip_file
4+
"""
5+
import builtins
6+
import google.protobuf.descriptor
7+
import google.protobuf.duration_pb2
8+
import google.protobuf.message
9+
import sys
10+
11+
if sys.version_info >= (3, 8):
12+
import typing as typing_extensions
13+
else:
14+
import typing_extensions
15+
16+
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
17+
18+
class Aggregation(google.protobuf.message.Message):
19+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
20+
21+
COLUMN_FIELD_NUMBER: builtins.int
22+
FUNCTION_FIELD_NUMBER: builtins.int
23+
TIME_WINDOW_FIELD_NUMBER: builtins.int
24+
SLIDE_INTERVAL_FIELD_NUMBER: builtins.int
25+
column: builtins.str
26+
function: builtins.str
27+
@property
28+
def time_window(self) -> google.protobuf.duration_pb2.Duration: ...
29+
@property
30+
def slide_interval(self) -> google.protobuf.duration_pb2.Duration: ...
31+
def __init__(
32+
self,
33+
*,
34+
column: builtins.str = ...,
35+
function: builtins.str = ...,
36+
time_window: google.protobuf.duration_pb2.Duration | None = ...,
37+
slide_interval: google.protobuf.duration_pb2.Duration | None = ...,
38+
) -> None: ...
39+
def HasField(self, field_name: typing_extensions.Literal["slide_interval", b"slide_interval", "time_window", b"time_window"]) -> builtins.bool: ...
40+
def ClearField(self, field_name: typing_extensions.Literal["column", b"column", "function", b"function", "slide_interval", b"slide_interval", "time_window", b"time_window"]) -> None: ...
41+
42+
global___Aggregation = Aggregation

‎sdk/python/feast/protos/feast/core/Aggregation_pb2_grpc.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
2+
"""Client and server classes corresponding to protobuf-defined services."""
3+
import grpc
4+

‎sdk/python/feast/protos/feast/core/DataFormat_pb2.py‎

Lines changed: 39 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)