Revert "Revert "added S3 storage binding""

This reverts commit 0e5866e. Restores brianfrankcooper#372
adally · Nov 25, 2015 · ff25df0 · ff25df0
1 parent 0794af4
commit ff25df0
Show file tree

Hide file tree

Showing 8 changed files with 746 additions and 2 deletions.
diff --git a/bin/ycsb b/bin/ycsb
@@ -71,6 +71,7 @@ DATABASES = {
     "nosqldb"      : "com.yahoo.ycsb.db.NoSqlDbClient",
     "orientdb"     : "com.yahoo.ycsb.db.OrientDBClient",
     "redis"        : "com.yahoo.ycsb.db.RedisClient",
+    "s3"           : "com.yahoo.ycsb.db.S3Client",
     "tarantool"    : "com.yahoo.ycsb.db.TarantoolClient",
     "voldemort"    : "com.yahoo.ycsb.db.VoldemortClient"
 }

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -129,6 +129,11 @@ LICENSE file.
       <artifactId>redis-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>s3-binding</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>
       <artifactId>tarantool-binding</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -85,6 +85,7 @@ LICENSE file.
     <mongodb.async.version>2.0.1</mongodb.async.version>
     <orientdb.version>1.0.1</orientdb.version>
     <redis.version>2.0.0</redis.version>
+    <s3.version>1.10.20</s3.version>
     <voldemort.version>0.81</voldemort.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <thrift.version>0.8.0</thrift.version>
@@ -121,6 +122,7 @@ LICENSE file.
     <!--module>nosqldb</module-->
     <module>orientdb</module>
     <module>redis</module>
+    <module>s3</module>
     <module>tarantool</module>
     <!--<module>voldemort</module>-->
   </modules>
@@ -132,8 +134,8 @@ LICENSE file.
         <artifactId>maven-compiler-plugin</artifactId>
         <version>3.3</version>
         <configuration>
-          <source>1.6</source>
-          <target>1.6</target>
+          <source>1.7</source>
+          <target>1.7</target>
         </configuration>
       </plugin>
       <plugin>

diff --git a/s3/README.md b/s3/README.md
@@ -0,0 +1,79 @@
+<!--
+Copyright (c) 2015 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+Quick Start
+===============
+### 1. Set Up YCSB
+
+Download the YCSB from this website:
+
+    https://github.com/brianfrankcooper/YCSB/releases/
+
+You can choose to download either the full stable version or just one of the available binding.
+
+### 2. Configuration of the AWS credentials
+
+The access key ID and secret access key as well as the endPoint and region and the Client configurations like the maxErrorRetry can be set in a properties file under s3-binding/conf/s3.properties or sent by command line (see below).
+It is highly suggested to use the property file instead of to send the credentials through the command line.
+
+
+### 3. Run YCSB
+
+To execute the benchmark using the S3 storage binding, first files must be uploaded using the "load" option with this command:
+
+       ./bin/ycsb load s3 -p table=theBucket -p s3.endPoint=s3.amazonaws.com -p s3.accessKeyId=yourAccessKeyId -p s3.secretKey=yourSecretKey -p fieldlength=10 -p fieldcount=20 -P workloads/workloada
+
+With this command, the workload A will be executing with the loading phase. The file size is determined by the number of fields (fieldcount) and by the field size (fieldlength). In this case each file is 200 bytes (10 bytes for each field multiplied by 20 fields).
+
+Running the command:
+
+       ./bin/ycsb -t s3 -p table=theBucket -p s3.endPoint=s3.amazonaws.com -p s3.accessKeyId=yourAccessKeyId -p s3.secretKey=yourSecretKey -p fieldlength=10 -p fieldcount=20 -P workloads/workloada
+
+the workload A will be executed with file size 200 bytes. 
+
+#### S3 Storage Configuration Parameters
+
+The parameters to configure the S3 client can be set using the file "s3-binding/conf/s3.properties". This is highly advisable for the parameters s3.accessKeyId and s3.secretKey. All the other parameters can be set also on the command line. Here the list of all the parameters that is possible to configure:
+
+- `table`
+  - This should be a S3 Storage bucket name and it replace the standard table name assigned by YCSB. 
+
+- `s3.endpoint`
+  - This indicate the endpoint used to connect to the S3 Storage service.
+  - Default value is `s3.amazonaws.com`.
+
+- `s3.region`
+  - This indicate the region where your buckets are.
+  - Default value is `us-east-1`.
+
+- `s3.accessKeyId`
+  - This is the accessKey of your S3 account.
+
+- `s3.secretKey`
+  - This is the secret associated with your S3 account.
+
+- `s3.maxErrorRetry`
+  - This is the maxErrorRetry parameter for the S3Client.
+
+- `s3.protocol`
+  - This is the protocol parameter for the S3Client. The default value is HTTPS.
+
+- `s3.sse`
+  - This parameter set to true activates the Server Side Encryption.
+
+- `s3.ssec`
+  - This parameter if not null activates the SSE-C client side encryption. The value passed with this parameter is the client key used to encrpyt the files.
+
diff --git a/s3/pom.xml b/s3/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+Copyright (c) 2015 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.ycsb</groupId>
+        <artifactId>binding-parent</artifactId>
+        <version>0.5.0-SNAPSHOT</version>
+        <relativePath>../binding-parent</relativePath>
+    </parent>
+
+  <artifactId>s3-binding</artifactId>
+  <name>S3 Storage Binding</name>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+	<groupId>com.amazonaws</groupId>
+	<artifactId>aws-java-sdk-s3</artifactId>
+	<version>${s3.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+            <artifactId>core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+    </dependency>
+   </dependencies>
+<build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>2.15</version>
+        <configuration>
+          <consoleOutput>true</consoleOutput>
+          <configLocation>../checkstyle.xml</configLocation>
+          <failOnViolation>true</failOnViolation>
+          <failsOnError>true</failsOnError>
+        </configuration>
+        <executions>
+          <execution>
+            <id>validate</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>checkstyle</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/s3/src/main/conf/s3.properties b/s3/src/main/conf/s3.properties
@@ -0,0 +1,31 @@
+#
+# Sample S3 configuration properties
+#
+# You may either set properties here or via the command line.
+#
+
+# the AWS S3 access key ID
+s3.accessKeyId=yourKey
+
+# the AWS S3 secret access key ID
+s3.secretKey=YourSecret
+
+# the AWS endpoint
+s3.endpoint=s3.amazonaws.com
+
+# activating the SSE server side encryption if true
+s3.sse=false
+
+# activating the SSE-C client side encryption if used
+#s3.ssec=U2CccCI40he2mZtg2aCEzofP7nQsfy4nP14VSYu6bFA=
+
+# set the protocol to use for the Client, default is HTTPS
+#s3.protocol=HTTPS
+
+# set the maxConnections to use for the Client, it should be not less than the
+# threads since only one client is created and shared between threads
+#s3.maxConnections=
+
+# set the maxErrorRetry parameter to use for the Client
+#s3.maxErrorRetry=
+