Bump the version to 3.11.2

Summary: With experimental feature SuggestCompactRange() we don't restrict running two L0->L1 compactions in parallel. This diff fixes this.

Test Plan: added a unit test to reproduce the failure. fixed the unit test

Reviewers: yhchiang, rven, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D39981

Conflicts:
	db/db_test.cc

HISTORY.md

@@ -1,5 +1,16 @@
 # Rocksdb Change Log
 
+## 3.11.2 (6/11/2015)
+
+### Fixes
+* Adjust the way we compensate for tombstones when chosing compactions. Previous heuristics led to pathological behavior in some cases.
+* Don't let two L0->L1 compactions run in parallel (only affected through experimental feature SuggestCompactRange)
+
+## 3.11.1 (6/1/2015)
+
+### Changes
+* Just a single change to fix the Java linking (github issue #606)
+
 ## 3.11.0 (5/19/2015)
 
 ### New Features

Makefile

@@ -939,7 +939,7 @@ rocksdbjavastatic: $(java_libobjects) libz.a libbz2.a libsnappy.a liblz4.a
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_libobjects) $(COVERAGEFLAGS) \
-	  libz.a libbz2.a libsnappy.a liblz4.a
+	  libz.a libbz2.a libsnappy.a liblz4.a $(LDFLAGS)
 	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)

db/compaction_picker.cc

@@ -763,6 +763,10 @@ void LevelCompactionPicker::PickFilesMarkedForCompactionExperimental(
     *level = level_file.first;
     *output_level = (*level == 0) ? vstorage->base_level() : *level + 1;
 
+    if (*level == 0 && !level0_compactions_in_progress_.empty()) {
+      return false;
+    }
+
     inputs->files = {level_file.second};
     inputs->level = *level;
     return ExpandWhileOverlapping(cf_name, vstorage, inputs);

db/db_test.cc

@@ -12887,7 +12887,6 @@ TEST_F(DBTest, FlushesInParallelWithCompactRange) {
   // iter == 1 -- leveled, but throw in a flush between two levels compacting
   // iter == 2 -- universal
   for (int iter = 0; iter < 3; ++iter) {
-    printf("iter %d\n", iter);
     Options options = CurrentOptions();
     if (iter < 2) {
       options.compaction_style = kCompactionStyleLevel;
@@ -12945,6 +12944,56 @@ TEST_F(DBTest, FlushesInParallelWithCompactRange) {
   }
 }
 
+// This tests for a bug that could cause two level0 compactions running
+// concurrently
+TEST_F(DBTest, SuggestCompactRangeNoTwoLevel0Compactions) {
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleLevel;
+  options.write_buffer_size = 110 << 10;
+  options.level0_file_num_compaction_trigger = 4;
+  options.num_levels = 4;
+  options.compression = kNoCompression;
+  options.max_bytes_for_level_base = 450 << 10;
+  options.target_file_size_base = 98 << 10;
+  options.max_write_buffer_number = 2;
+  options.max_background_compactions = 2;
+
+  DestroyAndReopen(options);
+
+  // fill up the DB
+  Random rnd(301);
+  for (int num = 0; num < 10; num++) {
+    GenerateNewRandomFile(&rnd);
+  }
+  db_->CompactRange(nullptr, nullptr);
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"CompactionJob::Run():Start",
+        "DBTest::SuggestCompactRangeNoTwoLevel0Compactions:1"},
+       {"DBTest::SuggestCompactRangeNoTwoLevel0Compactions:2",
+        "CompactionJob::Run():End"}});
+
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  // trigger L0 compaction
+  for (int num = 0; num < options.level0_file_num_compaction_trigger + 1;
+       num++) {
+    GenerateNewRandomFile(&rnd, /* nowait */ true);
+  }
+
+  TEST_SYNC_POINT("DBTest::SuggestCompactRangeNoTwoLevel0Compactions:1");
+
+  GenerateNewRandomFile(&rnd, /* nowait */ true);
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_OK(experimental::SuggestCompactRange(db_, nullptr, nullptr));
+  for (int num = 0; num < options.level0_file_num_compaction_trigger + 1;
+       num++) {
+    GenerateNewRandomFile(&rnd, /* nowait */ true);
+  }
+
+  TEST_SYNC_POINT("DBTest::SuggestCompactRangeNoTwoLevel0Compactions:2");
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/version_set.cc

@@ -962,9 +962,20 @@ void VersionStorageInfo::ComputeCompensatedSizes() {
       // for files that have been created right now and no other thread has
       // access to them. That's why we can safely mutate compensated_file_size.
       if (file_meta->compensated_file_size == 0) {
-        file_meta->compensated_file_size = file_meta->fd.GetFileSize() +
-            file_meta->num_deletions * average_value_size *
-            kDeletionWeightOnCompaction;
+        file_meta->compensated_file_size = file_meta->fd.GetFileSize();
+        // Here we only boost the size of deletion entries of a file only
+        // when the number of deletion entries is greater than the number of
+        // non-deletion entries in the file.  The motivation here is that in
+        // a stable workload, the number of deletion entries should be roughly
+        // equal to the number of non-deletion entries.  If we compensate the
+        // size of deletion entries in a stable workload, the deletion
+        // compensation logic might introduce unwanted effet which changes the
+        // shape of LSM tree.
+        if (file_meta->num_deletions * 2 >= file_meta->num_entries) {
+          file_meta->compensated_file_size +=
+              (file_meta->num_deletions * 2 - file_meta->num_entries)
+              * average_value_size * kDeletionWeightOnCompaction;
+        }
       }
     }
   }

include/rocksdb/version.h

@@ -6,7 +6,7 @@
 
 #define ROCKSDB_MAJOR 3
 #define ROCKSDB_MINOR 11
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 2
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these