Update HISTORY.md with 3.10.2 release

Summary:
This diff fixes a hang reported by a Github user.
https://www.facebook.com/l.php?u=https%3A%2F%2Fgithub.com%2Ffacebook%2Frocksdb%2Fissues%2F595%23issuecomment-96983273&h=9AQFYOWlo
Multiple large write batches with column families cause a hang.
The issue was caused by not doing flushes/compaction when the
write controller was stopped.

Test Plan: Create a DBTest from the user's test case

Reviewers: igor

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D37929

HISTORY.md

@@ -1,5 +1,10 @@
 # Rocksdb Change Log
 
+## 3.10.2 (5/1/2015)
+* Make sure RocksDB is compiled with fallocate support
+* Fix possible hang in DB::Write()
+* Fix a possibility of SIGSEGV in CompactRange()
+
 ## 3.10.0 (3/24/2015)
 ### New Features
 * GetThreadStatus() is now able to report detailed thread status, including:

build_tools/build_detect_platform

@@ -177,6 +177,7 @@ else
     # Test whether fallocate is available
     $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
       #include <fcntl.h>
+      #include <linux/falloc.h>
       int main() {
 	int fd = open("/dev/null", 0);
   fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);

db/db_impl.cc

@@ -416,6 +416,7 @@ void DBImpl::MaybeDumpStats() {
     // period in rare cases.
     last_stats_dump_time_microsec_ = now_micros;
 
+#ifndef ROCKSDB_LITE
     bool tmp1 = false;
     bool tmp2 = false;
     DBPropertyType cf_property_type =
@@ -436,6 +437,7 @@ void DBImpl::MaybeDumpStats() {
         db_options_.info_log, "------- DUMPING STATS -------");
     Log(InfoLogLevel::INFO_LEVEL,
         db_options_.info_log, "%s", stats.c_str());
+#endif  // !ROCKSDB_LITE
 
     PrintStatistics();
   }
@@ -1241,7 +1243,8 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
   {
     InstrumentedMutexLock l(&mutex_);
     Version* base = cfd->current();
-    for (int level = 1; level < cfd->NumberLevels(); level++) {
+    for (int level = 1; level < base->storage_info()->num_non_empty_levels();
+         level++) {
       if (base->storage_info()->OverlapInLevel(level, begin, end)) {
         max_level_with_files = level;
       }
@@ -2242,7 +2245,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
     assert(c->num_input_files(0) == 1);
     FileMetaData* f = c->input(0, 0);
     c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
-    c->edit()->AddFile(c->level() + 1, f->fd.GetNumber(), f->fd.GetPathId(),
+    c->edit()->AddFile(c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
                        f->fd.GetFileSize(), f->smallest, f->largest,
                        f->smallest_seqno, f->largest_seqno);
     status = versions_->LogAndApply(c->column_family_data(),
@@ -3128,6 +3131,11 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
 
   if (UNLIKELY(status.ok()) &&
       (write_controller_.IsStopped() || write_controller_.GetDelay() > 0)) {
+    // If writer is stopped, we need to get it going,
+    // so schedule flushes/compactions
+    if (context.schedule_bg_work_) {
+      MaybeScheduleFlushOrCompaction();
+    }
     status = DelayWrite(expiration_time);
   }
 

db/db_test.cc

@@ -11952,6 +11952,68 @@ TEST_F(DBTest, FilterCompactionTimeTest) {
   delete itr;
 }
 
+TEST_F(DBTest, EmptyCompactedDB) {
+  Options options;
+  options.max_open_files = -1;
+  options = CurrentOptions(options);
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+  Status s = Put("new", "value");
+  ASSERT_TRUE(s.IsNotSupported());
+  Close();
+}
+
+// Github issue #596
+TEST_F(DBTest, HugeNumberOfLevels) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 2 * 1024 * 1024;         // 2MB
+  options.max_bytes_for_level_base = 2 * 1024 * 1024;  // 2MB
+  options.num_levels = 12;
+  options.max_background_compactions = 10;
+  options.max_bytes_for_level_multiplier = 2;
+  options.level_compaction_dynamic_level_bytes = true;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < 300000; ++i) {
+    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
+  }
+
+  ASSERT_OK(db_->CompactRange(nullptr, nullptr));
+}
+
+// Github issue #595
+// Large write batch with column families
+TEST_F(DBTest, LargeBatchWithColumnFamilies) {
+  Options options;
+  options.env = env_;
+  options = CurrentOptions(options);
+  options.write_buffer_size = 100000;  // Small write buffer
+  CreateAndReopenWithCF({"pikachu"}, options);
+  int64_t j = 0;
+  for (int i = 0; i < 5; i++) {
+    for (int pass = 1; pass <= 3; pass++) {
+      WriteBatch batch;
+      size_t write_size = 1024 * 1024 * (5 + i);
+      fprintf(stderr, "prepare: %ld MB, pass:%d\n", (write_size / 1024 / 1024),
+              pass);
+      for (;;) {
+        std::string data(3000, j++ % 127 + 20);
+        data += std::to_string(j);
+        batch.Put(handles_[0], Slice(data), Slice(data));
+        if (batch.GetDataSize() > write_size) {
+          break;
+        }
+      }
+      fprintf(stderr, "write: %ld MB\n", (batch.GetDataSize() / 1024 / 1024));
+      ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+      fprintf(stderr, "done\n");
+    }
+  }
+  // make sure we can re-open it.
+  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/file_indexer.cc

@@ -20,6 +20,9 @@ FileIndexer::FileIndexer(const Comparator* ucmp)
 size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
 
 size_t FileIndexer::LevelIndexSize(size_t level) const {
+  if (level >= next_level_index_.size()) {
+    return 0;
+  }
   return next_level_index_[level].num_index;
 }
 

db/file_indexer_test.cc

@@ -11,6 +11,7 @@
 #include "db/file_indexer.h"
 #include "db/dbformat.h"
 #include "db/version_edit.h"
+#include "port/stack_trace.h"
 #include "rocksdb/comparator.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -343,6 +344,7 @@ TEST_F(FileIndexerTest, mixed) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

db/internal_stats.h

@@ -366,7 +366,7 @@ class InternalStats {
 
   void IncBytesMoved(int level, uint64_t amount) {}
 
-  void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
+  void RecordLevelNSlowdown(int level, bool soft) {}
 
   void AddCFStats(InternalCFStatsType type, uint64_t value) {}
 

db/managed_iterator.h

@@ -81,4 +81,4 @@ class ManagedIterator : public Iterator {
 };
 
 }  // namespace rocksdb
-#endif  // ROCKSDB_LITE
+#endif  // !ROCKSDB_LITE

db/version_set.cc

@@ -1184,6 +1184,10 @@ bool Version::Unref() {
 bool VersionStorageInfo::OverlapInLevel(int level,
                                         const Slice* smallest_user_key,
                                         const Slice* largest_user_key) {
+  if (level >= num_non_empty_levels_) {
+    // empty level, no overlap
+    return false;
+  }
   return SomeFileOverlapsRange(*internal_comparator_, (level > 0),
                                level_files_brief_[level], smallest_user_key,
                                largest_user_key);
@@ -1227,6 +1231,11 @@ int VersionStorageInfo::PickLevelForMemTableOutput(
 void VersionStorageInfo::GetOverlappingInputs(
     int level, const InternalKey* begin, const InternalKey* end,
     std::vector<FileMetaData*>* inputs, int hint_index, int* file_index) {
+  if (level >= num_non_empty_levels_) {
+    // this level is empty, no overlapping inputs
+    return;
+  }
+
   inputs->clear();
   Slice user_begin, user_end;
   if (begin != nullptr) {
@@ -1571,12 +1580,12 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
       }
 
       // Calculate base level and its size.
-      int base_level_size;
+      uint64_t base_level_size;
       if (cur_level_size <= base_bytes_min) {
         // Case 1. If we make target size of last level to be max_level_size,
         // target size of the first non-empty level would be smaller than
         // base_bytes_min. We set it be base_bytes_min.
-        base_level_size = static_cast<int>(base_bytes_min + 1);
+        base_level_size = base_bytes_min + 1U;
         base_level_ = first_non_empty_level;
         Warn(ioptions.info_log,
              "More existing levels in DB than needed. "
@@ -1592,16 +1601,17 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
         if (cur_level_size > base_bytes_max) {
           // Even L1 will be too large
           assert(base_level_ == 1);
-          base_level_size = static_cast<int>(base_bytes_max);
+          base_level_size = base_bytes_max;
         } else {
-          base_level_size = static_cast<int>(cur_level_size);
+          base_level_size = cur_level_size;
         }
       }
 
-      int level_size = base_level_size;
+      uint64_t level_size = base_level_size;
       for (int i = base_level_; i < num_levels_; i++) {
         if (i > base_level_) {
-          level_size = level_size * options.max_bytes_for_level_multiplier;
+          level_size = MultiplyCheckOverflow(
+              level_size, options.max_bytes_for_level_multiplier);
         }
         level_max_bytes_[i] = level_size;
       }

db/version_set_test.cc

@@ -215,6 +215,25 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
   ASSERT_EQ(0, logger_->log_count);
 }
 
+TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) {
+  uint64_t kOneGB = 1000U * 1000U * 1000U;
+  ioptions_.level_compaction_dynamic_level_bytes = true;
+  mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  Add(0, 1U, "1", "2", 50U);
+  Add(3, 4U, "1", "2", 32U * kOneGB);
+  Add(4, 5U, "1", "2", 500U * kOneGB);
+  Add(5, 6U, "1", "2", 3000U * kOneGB);
+
+  vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
+  ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB);
+  ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB);
+  ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB);
+  ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 3U * kOneGB);
+  ASSERT_EQ(vstorage_.base_level(), 2);
+  ASSERT_EQ(0, logger_->log_count);
+}
+
 class FindLevelFileTest : public testing::Test {
  public:
   LevelFilesBrief file_level_;

include/rocksdb/version.h

@@ -6,7 +6,7 @@
 
 #define ROCKSDB_MAJOR 3
 #define ROCKSDB_MINOR 10
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 2
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

util/autovector.h

@@ -190,16 +190,16 @@ class autovector {
 
   bool empty() const { return size() == 0; }
 
-  // will not check boundry
   const_reference operator[](size_type n) const {
+    assert(n < size());
     return n < kSize ? values_[n] : vect_[n - kSize];
   }
 
   reference operator[](size_type n) {
+    assert(n < size());
     return n < kSize ? values_[n] : vect_[n - kSize];
   }
 
-  // will check boundry
   const_reference at(size_type n) const {
     assert(n < size());
     return (*this)[n];

util/options.cc

@@ -611,6 +611,8 @@ DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
   return this;
 }
 
+#endif  // !ROCKSDB_LITE
+
 ReadOptions::ReadOptions()
     : verify_checksums(true),
       fill_cache(true),
@@ -637,6 +639,4 @@ ReadOptions::ReadOptions(bool cksum, bool cache)
              reinterpret_cast<ReadOptions*>(this));
 }
 
-#endif  // ROCKSDB_LITE
-
 }  // namespace rocksdb

util/thread_status_util_debug.cc

@@ -20,10 +20,11 @@ void ThreadStatusUtil::TEST_SetStateDelay(
   states_delay[state].store(micro, std::memory_order_relaxed);
 }
 
-void ThreadStatusUtil::TEST_StateDelay(
-    const ThreadStatus::StateType state) {
-  Env::Default()->SleepForMicroseconds(
-      states_delay[state].load(std::memory_order_relaxed));
+void ThreadStatusUtil::TEST_StateDelay(const ThreadStatus::StateType state) {
+  auto delay = states_delay[state].load(std::memory_order_relaxed);
+  if (delay > 0) {
+    Env::Default()->SleepForMicroseconds(delay);
+  }
 }
 
 #endif  // !NDEBUG

util/xfunc.cc

@@ -3,13 +3,13 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#ifdef XFUNC
 #include <string>
 #include "db/db_impl.h"
 #include "db/managed_iterator.h"
 #include "rocksdb/options.h"
 #include "util/xfunc.h"
 
-#ifdef XFUNC
 
 namespace rocksdb {
 

util/xfunc.h

@@ -17,13 +17,15 @@ namespace rocksdb {
  *   with XFUNC only being set for debug builds.
  */
 #if defined(ROCKSDB_XFTEST_FORCE)
+#ifndef ROCKSDB_LITE
 #if (ROCKSDB_XFTEST_FORCE == 1)
 #define XFUNC
-#endif
+#endif  // ROCKSDB_XFTEST_FORCE == 1
 #elif defined(NDEBUG)
 #else
 #define XFUNC
-#endif
+#endif  // defined(ROCKSDB_XFTEST_FORCE)
+#endif  // !ROCKSDB_LITE
 
 #ifndef XFUNC
 #define XFUNC_TEST(condition, location, lfname, fname, ...)

utilities/compacted_db/compacted_db_impl.cc

@@ -107,6 +107,9 @@ Status CompactedDBImpl::Init(const Options& options) {
   version_ = cfd_->GetSuperVersion()->current;
   user_comparator_ = cfd_->user_comparator();
   auto* vstorage = version_->storage_info();
+  if (vstorage->num_non_empty_levels() == 0) {
+    return Status::NotSupported("no file exists");
+  }
   const LevelFilesBrief& l0 = vstorage->LevelFilesBrief(0);
   // L0 should not have files
   if (l0.num_files > 1) {