[Core] Add retry for insertion logic to make the load process more ro…

…bust The default behavior is NO RETRY, which is the current behavior. But optionally user can enable retry if they are working with a DB backend that is better tolerated by allowing a few retries.
adally · Dec 12, 2015 · ad8404a · ad8404a
1 parent ed0a318
commit ad8404a
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 5 deletions.
diff --git a/core/src/main/java/com/yahoo/ycsb/workloads/CoreWorkload.java b/core/src/main/java/com/yahoo/ycsb/workloads/CoreWorkload.java
@@ -296,6 +296,16 @@ public class CoreWorkload extends Workload {
    */
   public static final String HOTSPOT_OPN_FRACTION_DEFAULT = "0.8";
 
+  /**
+   * How many times to retry when insertion of a single item to a DB fails.
+   */
+  public static final String INSERTION_RETRY_LIMIT_DEFAULT = "0";
+
+  /**
+   * On average, how long to wait between the retries, in seconds.
+   */
+  public static final String INSERTION_RETRY_INTERVAL_DEFAULT = "3";
+
   IntegerGenerator keysequence;
 
   DiscreteGenerator operationchooser;
@@ -312,6 +322,9 @@ public class CoreWorkload extends Workload {
 
   int recordcount;
 
+  int insertionRetryLimit;
+  int insertionRetryInterval;
+
   private Measurements _measurements = Measurements.getMeasurements();
 
   protected static IntegerGenerator getFieldLengthGenerator(Properties p) throws WorkloadException {
@@ -475,6 +488,12 @@ public void init(Properties p) throws WorkloadException {
       throw new WorkloadException(
           "Distribution \"" + scanlengthdistrib + "\" not allowed for scan length");
     }
+
+    insertionRetryLimit = Integer.parseInt(p.getProperty(
+        "insertionretrylimit", INSERTION_RETRY_LIMIT_DEFAULT));
+
+    insertionRetryInterval = Integer.parseInt(p.getProperty(
+        "insertionretryinterval", INSERTION_RETRY_INTERVAL_DEFAULT));
   }
 
   public String buildKeyName(long keynum) {
@@ -550,12 +569,37 @@ public boolean doInsert(DB db, Object threadstate) {
     int keynum = keysequence.nextInt();
     String dbkey = buildKeyName(keynum);
     HashMap<String, ByteIterator> values = buildValues(dbkey);
-    if (db.insert(table, dbkey, values).equals(Status.OK))
-      return true;
-    else
-      return false;
-  }
 
+    Status status;
+    int numOfRetries = 0;
+    do {
+      status = db.insert(table, dbkey, values);
+      if (status == Status.OK) {
+        break;
+      }
+      // Retry if configured. Without retrying, the load process will fail
+      // even if one single insertion fails. User can optionally configure
+      // an insertion retry limit (default is 0) to enable retry.
+      if (++numOfRetries <= insertionRetryLimit) {
+        System.err.println("Retrying insertion, retry count: " + numOfRetries);
+        try {
+          // Sleep for a random number between [0.8, 1.2)*insertionRetryInterval.
+          int sleepTime = (int) (1000 * insertionRetryInterval * (0.8 + 0.4 * Math.random()));
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+          break;
+        }
+
+      } else {
+        System.err.println("Error inserting, not retrying any more. number of attempts: " + numOfRetries +
+            "Insertion Retry Limit: " + insertionRetryLimit);
+        break;
+
+      }
+    } while (true);
+
+    return (status == Status.OK);
+  }
 
   /**
    * Do one transaction operation. Because it will be called concurrently from multiple client

diff --git a/workloads/workload_template b/workloads/workload_template
@@ -156,3 +156,15 @@ timeseries.granularity=1000
 #    property.
 # reportlatencyforeacherror=false
 # latencytrackederrors="<comma separated strings of error codes>"
+
+# Insertion error retry
+#
+# By default, YCSB does not retry any operations. However, during the
+# load process, if any insertion fails, the entire load process is terminated.
+# If a user desires to have more robust behavior during this phase, they can
+# enable retry for insertion by setting the following property to a positive
+# number.
+# insertionretrylimit = 0
+#
+# the following number controls the interval between retries (in seconds):
+# insertionretryinterval = 3