database: various optimizations for SQLite and PostgreSQL

CHANGELOG.rst

@@ -24,6 +24,9 @@ For admins
   persist option
 - Option archive_expires_after has been added to delete older messages from the
   database
+- SQLite enables WAL by default now. This can be changed with the
+  sqlite_journal_mode option.
+- Biboumi will regularely perform maintanance on the database.
 
 For packagers
 -------------

doc/admin.rst

@@ -287,6 +287,16 @@ can be changed by this setting
   # keep messages forever, the default
   archive_expires_after=0
 
+sqlite_journal_mode
+~~~~~~~~~~~~~~~~~~~
+
+Set the `journal_mode https://sqlite.org/pragma.html#pragma_journal_mode`_ to
+be used when connecting to an SQLite database. This option can be one of
+`DELETE`, `TRUNCATE`, `PERSIST`, `MEMORY`, `WAL`, `OFF`, or `none` to not set
+the journal_mode at all.
+
+By default this option is set to `WAL` to use `Write-Ahead Logging
+https://sqlite.org/wal.html`_.
 
 TLS configuration
 -----------------

src/config/config.cpp

@@ -2,6 +2,7 @@
 #include <utils/tolower.hpp>
 #include <utils/split.hpp>
 
+#include <fstream>
 #include <algorithm>
 #include <cstdlib>
 #include <cstring>

src/config/config.hpp

@@ -16,8 +16,6 @@
 #pragma once
 
 #include <functional>
-#include <fstream>
-#include <memory>
 #include <vector>
 #include <string>
 #include <map>

src/database/database_type.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+enum class DatabaseType
+{
+  none,
+  sqlite3,
+  postgresql,
+};
+

src/database/engine.hpp

@@ -7,11 +7,11 @@
  * Everything else (all portable stuf) should go outside of this class.
  */
 
+#include <database/database_type.hpp>
 #include <database/statement.hpp>
 
 #include <memory>
 #include <string>
-#include <vector>
 #include <tuple>
 #include <set>
 
@@ -36,6 +36,7 @@ class DatabaseEngine
     return {};
   }
   virtual std::string id_column_type() = 0;
+  virtual DatabaseType get_type() { return DatabaseType::none; };
 
   int64_t last_inserted_rowid{-1};
 };

src/database/postgresql_engine.cpp

@@ -101,4 +101,7 @@ std::string PostgresqlEngine::id_column_type()
   return "SERIAL";
 }
 
+DatabaseType PostgresqlEngine::get_type() {
+  return DatabaseType::postgresql;
+};
 #endif

src/database/postgresql_engine.hpp

@@ -2,9 +2,9 @@
 
 #include <biboumi.h>
 #include <string>
-#include <stdexcept>
 #include <memory>
 
+#include <database/database_type.hpp>
 #include <database/statement.hpp>
 #include <database/engine.hpp>
 
@@ -30,6 +30,8 @@ class PostgresqlEngine: public DatabaseEngine
   void extract_last_insert_rowid(Statement& statement) override;
   std::string get_returning_id_sql_string(const std::string& col_name) override;
   std::string id_column_type() override;
+  DatabaseType get_type() override final;
+
 private:
   PGconn* const conn;
 };

src/database/row.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <string>
 #include <type_traits>
 
 template <typename... T>

src/database/sqlite3_engine.cpp

@@ -7,9 +7,14 @@
 
 #include <database/query.hpp>
 
+#include <config/config.hpp>
 #include <utils/tolower.hpp>
 #include <logger/logger.hpp>
-#include <vector>
+#include <utils/dirname.hpp>
+
+#include <filesystem>
+#include <sys/stat.h>
+
 
 Sqlite3Engine::Sqlite3Engine(sqlite3* db):
     db(db)
@@ -46,6 +51,8 @@ std::set<std::string> Sqlite3Engine::get_all_columns_from_table(const std::strin
 std::unique_ptr<DatabaseEngine> Sqlite3Engine::open(const std::string& filename)
 {
   sqlite3* new_db;
+  const std::string dirname = utils::dirname(filename);
+  std::filesystem::create_directories(dirname);
   auto res = sqlite3_open_v2(filename.data(), &new_db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
   if (res != SQLITE_OK)
     {
@@ -53,7 +60,31 @@ std::unique_ptr<DatabaseEngine> Sqlite3Engine::open(const std::string& filename)
       sqlite3_close(new_db);
       throw std::runtime_error("");
     }
-  return std::make_unique<Sqlite3Engine>(new_db);
+  std::unique_ptr<DatabaseEngine> db = std::make_unique<Sqlite3Engine>(new_db);
+
+  // Enable Write-Ahead Logging
+  // https://sqlite.org/wal.html
+  auto sqlite_journal_mode = Config::get("sqlite_journal_mode", "WAL");
+  if (sqlite_journal_mode != "none") {
+    if (Sqlite3Engine::is_valid_sqlite_journal_mode(sqlite_journal_mode)) {
+      auto [success, err_msg] =
+          db->raw_exec("PRAGMA journal_mode = " + sqlite_journal_mode);
+      if (!success)
+        log_error("Failed enabling SQLite WAL with error: ", err_msg);
+    } else {
+      log_error("Invalid config option sqlite_journal_mode: ",
+                sqlite_journal_mode);
+    }
+  }
+  for (const auto &s : Sqlite3Engine::startup_commands) {
+    auto [success, err_msg] = db->raw_exec(s);
+    if (!success) {
+      log_error("Failed running startup command: ", s,
+                "; with error: ", err_msg);
+    }
+  }
+
+  return db;
 }
 
 std::tuple<bool, std::string> Sqlite3Engine::raw_exec(const std::string& query)
@@ -97,4 +128,17 @@ std::string Sqlite3Engine::id_column_type()
   return "INTEGER PRIMARY KEY AUTOINCREMENT";
 }
 
+DatabaseType Sqlite3Engine::get_type() {
+  return DatabaseType::sqlite3;
+};
+
+
+bool Sqlite3Engine::is_valid_sqlite_journal_mode(std::string &sqlite_journal_mode) {
+  // https://sqlite.org/pragma.html#pragma_journal_mode
+  for (const std::string &s : Sqlite3Engine::valid_sqlite_journal_modes) {
+    if (sqlite_journal_mode == s)
+      return true;
+  }
+  return false;
+}
 #endif

src/database/sqlite3_engine.hpp

@@ -29,8 +29,29 @@ class Sqlite3Engine: public DatabaseEngine
   std::unique_ptr<Statement> prepare(const std::string& query) override;
   void extract_last_insert_rowid(Statement& statement) override;
   std::string id_column_type() override;
+  DatabaseType get_type() override final;
+
 private:
   sqlite3* const db;
+  constexpr static const char *const startup_commands[3] = {
+      // Wait for a maximum of 5 seconds if a write is busy
+      "PRAGMA busy_timeout = 5000",
+      // Reduce disk syncs
+      // >When synchronous is NORMAL (1), the SQLite database engine will still
+      // >sync at the most critical moments
+      // https://sqlite.org/pragma.html#pragma_synchronous
+      "PRAGMA synchronous = NORMAL",
+      // Run PRAGMA command early to analyze all tables, as recommended by
+      // sqlite
+      // >Applications that use long-lived database connections should run
+      // >"PRAGMA optimize=0x10002;" when the connection is first opened
+      // https://sqlite.org/lang_analyze.html
+      "PRAGMA optimize=0x10002",
+  };
+  constexpr static std::string valid_sqlite_journal_modes[6] = {
+      "DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF",
+  };
+  static bool is_valid_sqlite_journal_mode(std::string &sqlite_journal_mode);
 };
 
 #else

src/database/table.hpp

@@ -5,7 +5,6 @@
 #include <database/delete_query.hpp>
 #include <database/row.hpp>
 
-#include <algorithm>
 #include <string>
 #include <set>
 

src/scheduler/scheduler.cpp

@@ -1,10 +1,12 @@
 #include "biboumi.h"
 #include <config/config.hpp>
+#include <logger/logger.hpp>
 #include <scheduler/scheduler.hpp>
 #include <utils/parse_duration.hpp>
 #include <utils/timed_events.hpp>
 
 #ifdef USE_DATABASE
+#include <database/database_type.hpp>
 #include <database/database.hpp>
 #endif
 
@@ -13,8 +15,10 @@ static const std::unordered_map<
     events = {
 #ifdef USE_DATABASE
         {"cleanup_old_messages",
-         {60s * 10 /* 10min */, &Scheduler::cleanup_old_messages}},
-#endif
+         {std::chrono::seconds(10min), &Scheduler::cleanup_old_messages}},
+        {"optimize_db",
+         {std::chrono::seconds(24h), &Scheduler::optimize_db}},
+#endif // USE_DATABASE
 };
 
 void Scheduler::start_scheduler() {
@@ -65,4 +69,39 @@ void Scheduler::cleanup_old_messages() {
               "archive_expires_after is ",
               archive_expires_after);
 }
-#endif
+
+void Scheduler::optimize_db() {
+  log_debug("[optimize_db] Starting. Type: ", (int)Database::db->get_type());
+  switch (Database::db->get_type()) {
+# ifdef SQLITE3_FOUND
+  case DatabaseType::sqlite3: {
+    log_debug("[optimize_db] Optimizing Sqlite database");
+    // Run PRAGMA optimize periodically, as recommended by sqlite
+    // https://sqlite.org/lang_analyze.html#automatically_running_analyze
+    auto [success, err_msg] = Database::db->raw_exec("PRAGMA optimize");
+    if (!success) {
+      log_error("[optimize_db] Failed optimizing Sqlite database: ", err_msg);
+    }
+    break;
+  }
+# endif // SQLITE3_FOUND
+# ifdef PQ_FOUND
+  case DatabaseType::postgresql: {
+    log_debug("[optimize_db] Optimizing PostgreSQL database");
+    // Run VACUUM and ANALYZE periodically
+    // https://www.postgresql.org/docs/current/sql-vacuum.html
+    // https://www.postgresql.org/docs/current/sql-analyze.html
+    auto [success, err_msg] = Database::db->raw_exec("VACUUM ANALYZE");
+    if (!success) {
+      log_error("[optimize_db] Failed optimizing PostgreSQL database: ",
+                err_msg);
+    }
+    break;
+  }
+# endif // PQ_FOUND
+  case DatabaseType::none:
+  default:
+    break;
+  }
+}
+#endif // USE_DATABASE

src/scheduler/scheduler.hpp

@@ -7,4 +7,5 @@ public:
 
   // Jobs
   static void cleanup_old_messages();
+  static void optimize_db();
 };

src/utils/timed_events.hpp

@@ -22,7 +22,7 @@ class TimedEvent
   friend class TimedEventsManager;
 public:
   /**
-   * An event the occurs only once, at the given time_point
+   * An event that occurs only once, at the given time_point
    */
   explicit TimedEvent(std::chrono::steady_clock::time_point&& time_point,
                       std::function<void()> callback, std::string name="");