database: various optimizations for SQLite and PostgreSQL

src/database/sqlite3_engine.cpp

@@ -7,9 +7,14 @@
 
 #include <database/query.hpp>
 
+#include <config/config.hpp>
 #include <utils/tolower.hpp>
 #include <logger/logger.hpp>
-#include <vector>
+#include <utils/dirname.hpp>
+
+#include <filesystem>
+#include <sys/stat.h>
+
 
 Sqlite3Engine::Sqlite3Engine(sqlite3* db):
     db(db)
@@ -46,6 +51,8 @@ std::set<std::string> Sqlite3Engine::get_all_columns_from_table(const std::strin
 std::unique_ptr<DatabaseEngine> Sqlite3Engine::open(const std::string& filename)
 {
   sqlite3* new_db;
+  const std::string dirname = utils::dirname(filename);
+  std::filesystem::create_directories(dirname);
   auto res = sqlite3_open_v2(filename.data(), &new_db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
   if (res != SQLITE_OK)
     {
@@ -53,7 +60,31 @@ std::unique_ptr<DatabaseEngine> Sqlite3Engine::open(const std::string& filename)
       sqlite3_close(new_db);
       throw std::runtime_error("");
     }
-  return std::make_unique<Sqlite3Engine>(new_db);
+  std::unique_ptr<DatabaseEngine> db = std::make_unique<Sqlite3Engine>(new_db);
+
+  // Enable Write-Ahead Logging
+  // https://sqlite.org/wal.html
+  auto sqlite_journal_mode = Config::get("sqlite_journal_mode", "WAL");
+  if (sqlite_journal_mode != "none") {
+    if (Sqlite3Engine::is_valid_sqlite_journal_mode(sqlite_journal_mode)) {
+      auto [success, err_msg] =
+          db->raw_exec("PRAGMA journal_mode = " + sqlite_journal_mode);
+      if (!success)
+        log_error("Failed enabling SQLite WAL with error: ", err_msg);
+    } else {
+      log_error("Invalid config option sqlite_journal_mode: ",
+                sqlite_journal_mode);
+    }
+  }
+  for (const auto &s : Sqlite3Engine::startup_commands) {
+    auto [success, err_msg] = db->raw_exec(s);
+    if (!success) {
+      log_error("Failed running startup command: ", s,
+                "; with error: ", err_msg);
+    }
+  }
+
+  return db;
 }
 
 std::tuple<bool, std::string> Sqlite3Engine::raw_exec(const std::string& query)
@@ -97,4 +128,17 @@ std::string Sqlite3Engine::id_column_type()
   return "INTEGER PRIMARY KEY AUTOINCREMENT";
 }
 
+DatabaseType Sqlite3Engine::get_type() {
+  return DatabaseType::sqlite3;
+};
+
+
+bool Sqlite3Engine::is_valid_sqlite_journal_mode(std::string &sqlite_journal_mode) {
+  // https://sqlite.org/pragma.html#pragma_journal_mode
+  for (const std::string &s : Sqlite3Engine::valid_sqlite_journal_modes) {
+    if (sqlite_journal_mode == s)
+      return true;
+  }
+  return false;
+}
 #endif