fix conflicts

ray-project · Jul 13, 2023 · ae6599c · ae6599c
2 parents dec6153 + 16aada2
commit ae6599c
Show file tree

Hide file tree

Showing 16 changed files with 287 additions and 187 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,7 +1,9 @@
 [flake8]
 exclude =
-  fed_pb2_grpc.py
-  fed_pb2.py
+  fed_pb2_grpc_in_protobuf3.py
+  fed_pb2_in_protobuf3.py
+  fed_pb2_grpc_in_protobuf4.py
+  fed_pb2_in_protobuf4.py
   build
   py3
 max-line-length = 88
@@ -40,3 +42,5 @@ ignore =
 avoid-escape = no
 per-file-ignores =
     fed/_private/fed_call_holder.py:E402
+    fed/proxy/barriers.py:F401
+    tests/test_transport_proxy.py:F401
diff --git a/.github/workflows/test_on_ray2.0.0.yml → ...ub/workflows/unit_tests_for_protobuf4.yml b/.github/workflows/test_on_ray2.0.0.yml → ...ub/workflows/unit_tests_for_protobuf4.yml
@@ -1,4 +1,4 @@
-name: test on ray2.0.0
+name: test for protobuf4
 
 on:
   push:
@@ -7,7 +7,8 @@ on:
     branches: [ main ]
 
 jobs:
-  run-unit-tests:
+  run-unit-tests-on-for-protobuf4:
+
     timeout-minutes: 60
     runs-on: ubuntu-latest
     container: docker.io/library/ubuntu:latest
@@ -28,10 +29,11 @@ jobs:
             . py3/bin/activate
             which python
             pip install pytest torch cloudpickle cryptography
-            pip install ray==2.0.0
+            pip install "protobuf>=4.0"
+            pip install ray==2.4.0
 
     - name: Build and test
       run: |
            . py3/bin/activate
            pip install -e . -v
-           sh test.sh
+           sh test.sh
diff --git a/.github/workflows/unit_tests_on_ray_matrix.yml b/.github/workflows/unit_tests_on_ray_matrix.yml
@@ -0,0 +1,44 @@
+name: test on many rays
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  run-unit-tests-on-many-rays:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        ray_version: [2.0.0, 2.1.0, 2.2.0, 2.3.0, 2.4.0]
+
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    container: docker.io/library/ubuntu:latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install basic dependencies
+      run:  |
+            apt-get update
+            apt-get install -yq wget gcc g++ python3.9 zlib1g-dev zip libuv1.dev
+            apt-get install -yq pip
+
+    - name: Install python dependencies
+      run:  |
+            python3 -m pip install virtualenv
+            python3 -m virtualenv -p python3 py3
+            . py3/bin/activate
+            which python
+            pip install pytest torch cloudpickle cryptography
+            # six is required in ray-2.1.0
+            pip install "protobuf<4.0" six
+            pip install ray==${{ matrix.ray_version }}
+
+    - name: Build and test
+      run: |
+           . py3/bin/activate
+           pip install -e . -v
+           sh test.sh
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 # RayFed
-![docs building](https://readthedocs.org/projects/rayfed/badge/?version=latest) ![test on ray 2.0.0](https://github.com/ray-project/rayfed/actions/workflows/test_on_ray2.0.0.yml/badge.svg) ![test on ray 1.13.0](https://github.com/ray-project/rayfed/actions/workflows/test_on_ray1.13.0.yml/badge.svg)
+![docs building](https://readthedocs.org/projects/rayfed/badge/?version=latest) ![test on many rays](https://github.com/ray-project/rayfed/actions/workflows/unit_tests_on_ray_matrix.yml/badge.svg) ![test on ray 1.13.0](https://github.com/ray-project/rayfed/actions/workflows/test_on_ray1.13.0.yml/badge.svg)
 
 A multiple parties joint, distributed execution engine based on Ray, to help build your own federated learning frameworks in minutes.
 
@@ -35,10 +35,10 @@ The code view in each party is exactly the same, but the execution differs based
 
 
 ## Supported Ray Versions
-| RayFed Versions | ray-1.13.0 | ray-2.0.0 | ray-2.1.0 | ray-2.2.0 | ray-2.3.0 |
-|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|
-| 0.1.0           |✅      | ✅      | ✅      | ✅      | ❌      |
-| 0.2.0           |not released|not released|not released|not released|not released|
+| RayFed Versions | ray-1.13.0 | ray-2.0.0 | ray-2.1.0 | ray-2.2.0 | ray-2.3.0 | ray-2.4.0 |
+|:---------------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|
+| 0.1.0           |✅      | ✅      | ✅      | ✅      | ✅      | ✅      |
+| 0.2.0           |not released|not released|not released|not released|not released|not released|
 
 
 ## Installation

diff --git a/fed/_private/compatible_utils.py b/fed/_private/compatible_utils.py
@@ -17,7 +17,6 @@
 import fed._private.constants as fed_constants
 
 import ray.experimental.internal_kv as ray_internal_kv
-from ray._private.gcs_utils import GcsClient
 from fed._private import constants
 
 
@@ -102,6 +101,13 @@ def __init__(self) -> None:
         super().__init__()
 
     def initialize(self):
+        try:
+            from ray._private.gcs_utils import GcsClient
+        except ImportError:
+            # The GcsClient was moved to `ray._raylet` module in `ray-2.5.0`.
+            assert _compare_version_strings(ray.__version__, "2.4.0")
+            from ray._raylet import GcsClient
+
         gcs_client = GcsClient(
             address=_get_gcs_address_from_ray_worker(),
             nums_reconnect_retry=10)

diff --git a/fed/_private/fed_call_holder.py b/fed/_private/fed_call_holder.py
@@ -56,6 +56,9 @@ def options(self, **options):
         return self
 
     def internal_remote(self, *args, **kwargs):
+        if not self._node_party:
+            raise ValueError("You should specify a party name on the fed actor.")
+
         # Generate a new fed task id for this call.
         fed_task_id = get_global_context().next_seq_id()
         if self._party == self._node_party:

diff --git a/fed/api.py b/fed/api.py
@@ -258,9 +258,9 @@ def options(self, **options):
         return self
 
     def remote(self, *args, **kwargs):
-        assert (
-            self._node_party is not None
-        ), "A fed function should be specified within a party to execute."
+        if not self._node_party:
+            raise ValueError("You should specify a party name on the fed function.")
+
         return self._fed_call_holder.internal_remote(*args, **kwargs)
 
     def _execute_impl(self, args, kwargs):

diff --git a/fed/grpc/fed_pb2.py b/fed/grpc/fed_pb2.py
diff --git a/fed/grpc/fed_pb2_grpc.py → fed/grpc/fed_pb2_grpc_in_protobuf3.py b/fed/grpc/fed_pb2_grpc.py → fed/grpc/fed_pb2_grpc_in_protobuf3.py
@@ -16,7 +16,7 @@
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
 
-import fed.grpc.fed_pb2 as fed__pb2
+import fed.grpc.fed_pb2_in_protobuf3 as fed__pb2
 
 
 class GrpcServiceStub(object):