Skip to content

Commit 75ea17f

Browse files
jeanbzajeanbza
authored
benchmarking: add pubsub benchwrapper (googleapis#6462)
1 parent 149138a commit 75ea17f

5 files changed

Lines changed: 348 additions & 0 deletions

File tree

‎google-cloud-testing/pubsub-benchwrapper/README.md‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# pubsub-benchwrapper
2+
3+
pubsub-benchwrapper is a gRPC wrapper around the pubsub library for benchmarking purposes.
4+
5+
## Running
6+
7+
```
8+
mvn clean install -DskipTests=true (one time)
9+
cd google-cloud-testing/pubsub-benchwrapper
10+
export PUBSUB_EMULATOR_HOST=localhost:8080
11+
mvn clean install exec:java -DskipTests=true -Dport=8081
12+
```
13+
14+
## Generating .proto sources
15+
16+
Sources are generated as part of the protobuf-maven-plugin plugin, but if you'd
17+
like to generate them yourself to see the output you can run:
18+
19+
```
20+
cd google-cloud-testing/pubsub-benchwrapper
21+
protoc \
22+
--plugin=protoc-gen-grpc-java=build/exe/java_plugin/protoc-gen-grpc-java \
23+
--java_out=src/main/java \
24+
--grpc-java_out=src/main/java \
25+
--proto_path=src/main/proto \
26+
src/main/proto/*.proto
27+
```
28+
29+
Note that you'll need to delete these, or uncomment the plugin, since multiple
30+
definitions of the same class are not allowed.
31+
32+
Note that these should not be committed into git.
33+
34+
Note that you can also `mvn compile -DskipTests=true` and see sources in
35+
`target/generated-sources/`.
36+
37+
## Debugging HTTP requests
38+
39+
To debug HTTP requests, place a `logging.properties` at
40+
`google-cloud-testing/pubsub-benchwrapper/logging.properties` with the
41+
following contents:
42+
43+
```
44+
# Properties file which configures the operation of the JDK logging facility.
45+
# The system will look for this config file to be specified as a system property:
46+
# -Djava.util.logging.config.file=${project_loc:googleplus-simple-cmdline-sample}/logging.properties
47+
48+
# Set up the console handler (uncomment "level" to show more fine-grained messages)
49+
handlers = java.util.logging.ConsoleHandler
50+
java.util.logging.ConsoleHandler.level = CONFIG
51+
52+
# Set up logging of HTTP requests and responses (uncomment "level" to show)
53+
com.google.api.client.http.level = CONFIG
54+
```

‎google-cloud-testing/pubsub-benchwrapper/pom.xml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<?xml version="1.0"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<artifactId>benchwrapper</artifactId>
5+
<version>0.0.0</version>
6+
<packaging>jar</packaging>
7+
<groupId>com.google.cloud</groupId>
8+
9+
<properties>
10+
<grpcVersion>1.23.0</grpcVersion>
11+
<protobufVersion>3.9.1</protobufVersion>
12+
</properties>
13+
14+
<build>
15+
<extensions>
16+
<extension>
17+
<groupId>kr.motd.maven</groupId>
18+
<artifactId>os-maven-plugin</artifactId>
19+
<version>1.5.0.Final</version>
20+
</extension>
21+
</extensions>
22+
23+
<plugins>
24+
<!-- Auto-generate classes from .protos -->
25+
<plugin>
26+
<groupId>org.apache.maven.plugins</groupId>
27+
<artifactId>maven-compiler-plugin</artifactId>
28+
<version>3.8.0</version>
29+
<configuration>
30+
<source>1.8</source>
31+
<target>1.8</target>
32+
</configuration>
33+
</plugin>
34+
<plugin>
35+
<groupId>org.xolstice.maven.plugins</groupId>
36+
<artifactId>protobuf-maven-plugin</artifactId>
37+
<version>0.6.1</version>
38+
<configuration>
39+
<protocArtifact>com.google.protobuf:protoc:${protobufVersion}:exe:${os.detected.classifier}</protocArtifact>
40+
<pluginId>grpc-java</pluginId>
41+
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}</pluginArtifact>
42+
</configuration>
43+
<executions>
44+
<execution>
45+
<goals>
46+
<goal>compile</goal>
47+
<goal>compile-custom</goal>
48+
</goals>
49+
</execution>
50+
</executions>
51+
</plugin>
52+
53+
<!-- Protobuf runtime -->
54+
<plugin>
55+
<groupId>org.codehaus.mojo</groupId>
56+
<artifactId>exec-maven-plugin</artifactId>
57+
<version>1.6.0</version>
58+
<executions>
59+
<execution>
60+
<goals>
61+
<goal>exec</goal>
62+
</goals>
63+
</execution>
64+
</executions>
65+
<configuration>
66+
<mainClass>com.google.cloud.benchwrapper.Main</mainClass>
67+
<executable>maven</executable>
68+
<skip>false</skip>
69+
</configuration>
70+
</plugin>
71+
</plugins>
72+
</build>
73+
74+
<dependencies>
75+
<!-- pubsub -->
76+
<dependency>
77+
<groupId>com.google.cloud</groupId>
78+
<artifactId>google-cloud-pubsub</artifactId>
79+
<version>1.91.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-pubsub:current} -->
80+
</dependency>
81+
82+
<!-- protobuf -->
83+
<dependency>
84+
<groupId>com.google.protobuf</groupId>
85+
<artifactId>protobuf-java</artifactId>
86+
<version>${protobufVersion}</version>
87+
</dependency>
88+
<dependency>
89+
<groupId>io.grpc</groupId>
90+
<artifactId>grpc-protobuf</artifactId>
91+
<version>${grpcVersion}</version>
92+
</dependency>
93+
94+
<!-- grpc -->
95+
<dependency>
96+
<groupId>io.grpc</groupId>
97+
<artifactId>grpc-netty-shaded</artifactId>
98+
<version>${grpcVersion}</version>
99+
</dependency>
100+
<dependency>
101+
<groupId>io.grpc</groupId>
102+
<artifactId>grpc-stub</artifactId>
103+
<version>${grpcVersion}</version>
104+
</dependency>
105+
<dependency>
106+
<groupId>io.grpc</groupId>
107+
<artifactId>grpc-netty</artifactId>
108+
<version>${grpcVersion}</version>
109+
</dependency>
110+
<dependency>
111+
<groupId>io.netty</groupId>
112+
<artifactId>netty-tcnative-boringssl-static</artifactId>
113+
<version>2.0.25.Final</version>
114+
</dependency>
115+
116+
<!-- Generated classes are generated with a @javax.annotation.Generated
117+
annotation, so we need this dependency or else they don't compile. -->
118+
<dependency>
119+
<groupId>javax.annotation</groupId>
120+
<artifactId>javax.annotation-api</artifactId>
121+
<version>1.3.2</version>
122+
</dependency>
123+
</dependencies>
124+
</project>

‎google-cloud-testing/pubsub-benchwrapper/src/main/java/com/google/cloud/benchwrapper/Main.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.benchwrapper;
18+
19+
import java.util.Properties;
20+
import io.grpc.Server;
21+
import io.grpc.netty.NettyServerBuilder;
22+
23+
class Main {
24+
public static void main(String[] args) throws Exception {
25+
Properties properties = new Properties(System.getProperties());
26+
String port = properties.getProperty("port");
27+
if (port == null || port.equals("")) {
28+
System.err.println("Usage: mvn clean install exec:java -DskipTests=true -Dport=8081");
29+
System.exit(1);
30+
}
31+
32+
String pubsubEmulatorHost = System.getenv("PUBSUB_EMULATOR_HOST");
33+
if (pubsubEmulatorHost == null || pubsubEmulatorHost.equals("")) {
34+
// We could use system properties here too, but every other language uses
35+
// an environment variable called PUBSUB_EMULATOR_HOST, so the
36+
// consistency is nice to maintain.
37+
System.err.println("Please set PUBSUB_EMULATOR_HOST=localhost:8080");
38+
System.exit(1);
39+
}
40+
41+
System.out.println("Server starting up...");
42+
43+
int portInt = Integer.parseInt(port);
44+
final Server server = NettyServerBuilder
45+
.forPort(portInt)
46+
.addService(new PubsubBenchWrapperImpl(pubsubEmulatorHost))
47+
.build()
48+
.start();
49+
50+
System.out.println("Server starting up... done. Listening on " + port);
51+
Runtime.getRuntime().addShutdownHook(new Thread() {
52+
@Override
53+
public void run() {
54+
// Use stderr here since the logger may have been reset by its JVM shutdown hook.
55+
System.err.println("Shutting down gRPC server since JVM is shutting down");
56+
server.shutdown();
57+
}
58+
});
59+
60+
server.awaitTermination();
61+
}
62+
}
63+

‎google-cloud-testing/pubsub-benchwrapper/src/main/java/com/google/cloud/benchwrapper/PubsubBenchWrapperImpl.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright 2019 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.benchwrapper;
18+
19+
import io.grpc.stub.StreamObserver;
20+
import java.io.IOException;
21+
import java.nio.ByteBuffer;
22+
23+
import com.google.cloud.benchwrapper.PubsubBenchWrapperGrpc.PubsubBenchWrapperImplBase;
24+
import com.google.cloud.ReadChannel;
25+
import com.google.cloud.pubsub.v1.AckReplyConsumer;
26+
import com.google.cloud.pubsub.v1.MessageReceiver;
27+
import com.google.cloud.pubsub.v1.Subscriber;
28+
import com.google.pubsub.v1.ProjectSubscriptionName;
29+
import com.google.pubsub.v1.PubsubMessage;
30+
import com.google.api.gax.core.InstantiatingExecutorProvider;
31+
32+
class PubsubBenchWrapperImpl extends PubsubBenchWrapperImplBase {
33+
private Pubsub client;
34+
35+
public PubsubBenchWrapperImpl(String pubsubEmulatorHost) {
36+
37+
}
38+
39+
public void recv(PubsubRecv request, StreamObserver<EmptyResponse> responseObserver) {
40+
System.out.println("recv has been called");
41+
42+
ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(
43+
"some-project", request.getSubName());
44+
45+
Subscriber subscriber = null;
46+
try {
47+
InstantiatingExecutorProvider executorProvider =
48+
InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build();
49+
50+
subscriber =
51+
Subscriber.newBuilder(subscriptionName, new SimpleReceiver())
52+
.setExecutorProvider(executorProvider)
53+
.build();
54+
subscriber.startAsync().awaitRunning();
55+
56+
// Allow the subscriber to run indefinitely unless an unrecoverable error occurs.
57+
subscriber.awaitTerminated();
58+
} catch (IllegalStateException e) {
59+
System.out.println("Subscriber unexpectedly stopped: " + e);
60+
}
61+
62+
EmptyResponse reply = EmptyResponse.newBuilder().build();
63+
responseObserver.onNext(reply);
64+
responseObserver.onCompleted();
65+
}
66+
67+
static class SimpleReceiver implements MessageReceiver {
68+
@Override
69+
public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) {
70+
consumer.ack();
71+
}
72+
}
73+
}

‎google-cloud-testing/pubsub-benchwrapper/src/main/proto/pubsub.proto‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package pubsub_bench;
18+
19+
option java_multiple_files = true;
20+
option java_package = "com.google.cloud.benchwrapper";
21+
22+
message PubsubRecv {
23+
// The subscription identifier corresponding to number of messages sent.
24+
string sub_name = 1;
25+
}
26+
27+
// TODO(deklerk): Replace with Google's canonical Empty.
28+
message EmptyResponse {}
29+
30+
service PubsubBenchWrapper {
31+
// Recv represents opening a streaming pull stream to receive messages on.
32+
rpc Recv(PubsubRecv) returns (EmptyResponse) {}
33+
34+
}

0 commit comments

Comments
 (0)