From ad14748a30a6bbd5667597cd2a295fa3af5c5842 Mon Sep 17 00:00:00 2001 From: Luca Matei Pintilie Date: Mon, 20 Jul 2026 15:12:53 +0200 Subject: [PATCH] database: various optimizations for SQLite and PostgreSQL --- CHANGELOG.rst | 3 ++ doc/admin.rst | 10 +++++++ src/config/config.cpp | 1 + src/config/config.hpp | 2 -- src/database/database_type.hpp | 9 ++++++ src/database/engine.hpp | 3 +- src/database/postgresql_engine.cpp | 3 ++ src/database/postgresql_engine.hpp | 4 ++- src/database/row.hpp | 1 + src/database/sqlite3_engine.cpp | 48 ++++++++++++++++++++++++++++-- src/database/sqlite3_engine.hpp | 21 +++++++++++++ src/database/table.hpp | 1 - src/scheduler/scheduler.cpp | 45 ++++++++++++++++++++++++++-- src/scheduler/scheduler.hpp | 1 + src/utils/timed_events.hpp | 2 +- 15 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 src/database/database_type.hpp diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5b31bac..cb1e22e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,9 @@ For admins persist option - Option archive_expires_after has been added to delete older messages from the database +- SQLite enables WAL by default now. This can be changed with the + sqlite_journal_mode option. +- Biboumi will regularely perform maintanance on the database. For packagers ------------- diff --git a/doc/admin.rst b/doc/admin.rst index fff918d..2ae86ac 100644 --- a/doc/admin.rst +++ b/doc/admin.rst @@ -287,6 +287,16 @@ can be changed by this setting # keep messages forever, the default archive_expires_after=0 +sqlite_journal_mode +~~~~~~~~~~~~~~~~~~~ + +Set the `journal_mode https://sqlite.org/pragma.html#pragma_journal_mode`_ to +be used when connecting to an SQLite database. This option can be one of +`DELETE`, `TRUNCATE`, `PERSIST`, `MEMORY`, `WAL`, `OFF`, or `none` to not set +the journal_mode at all. + +By default this option is set to `WAL` to use `Write-Ahead Logging +https://sqlite.org/wal.html`_. TLS configuration ----------------- diff --git a/src/config/config.cpp b/src/config/config.cpp index 2f64b9e..6a20357 100644 --- a/src/config/config.cpp +++ b/src/config/config.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/config/config.hpp b/src/config/config.hpp index 9c28e8c..625a749 100644 --- a/src/config/config.hpp +++ b/src/config/config.hpp @@ -16,8 +16,6 @@ #pragma once #include -#include -#include #include #include #include diff --git a/src/database/database_type.hpp b/src/database/database_type.hpp new file mode 100644 index 0000000..0fb48db --- /dev/null +++ b/src/database/database_type.hpp @@ -0,0 +1,9 @@ +#pragma once + +enum class DatabaseType +{ + none, + sqlite3, + postgresql, +}; + diff --git a/src/database/engine.hpp b/src/database/engine.hpp index 41dccf5..369b4cc 100644 --- a/src/database/engine.hpp +++ b/src/database/engine.hpp @@ -7,11 +7,11 @@ * Everything else (all portable stuf) should go outside of this class. */ +#include #include #include #include -#include #include #include @@ -36,6 +36,7 @@ class DatabaseEngine return {}; } virtual std::string id_column_type() = 0; + virtual DatabaseType get_type() { return DatabaseType::none; }; int64_t last_inserted_rowid{-1}; }; diff --git a/src/database/postgresql_engine.cpp b/src/database/postgresql_engine.cpp index 59bc885..d3773e4 100644 --- a/src/database/postgresql_engine.cpp +++ b/src/database/postgresql_engine.cpp @@ -101,4 +101,7 @@ std::string PostgresqlEngine::id_column_type() return "SERIAL"; } +DatabaseType PostgresqlEngine::get_type() { + return DatabaseType::postgresql; +}; #endif diff --git a/src/database/postgresql_engine.hpp b/src/database/postgresql_engine.hpp index 1a9c249..4153922 100644 --- a/src/database/postgresql_engine.hpp +++ b/src/database/postgresql_engine.hpp @@ -2,9 +2,9 @@ #include #include -#include #include +#include #include #include @@ -30,6 +30,8 @@ class PostgresqlEngine: public DatabaseEngine void extract_last_insert_rowid(Statement& statement) override; std::string get_returning_id_sql_string(const std::string& col_name) override; std::string id_column_type() override; + DatabaseType get_type() override final; + private: PGconn* const conn; }; diff --git a/src/database/row.hpp b/src/database/row.hpp index 4004b5d..f5c13b7 100644 --- a/src/database/row.hpp +++ b/src/database/row.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include template diff --git a/src/database/sqlite3_engine.cpp b/src/database/sqlite3_engine.cpp index 5e3bba1..7b4f541 100644 --- a/src/database/sqlite3_engine.cpp +++ b/src/database/sqlite3_engine.cpp @@ -7,9 +7,14 @@ #include +#include #include #include -#include +#include + +#include +#include + Sqlite3Engine::Sqlite3Engine(sqlite3* db): db(db) @@ -46,6 +51,8 @@ std::set Sqlite3Engine::get_all_columns_from_table(const std::strin std::unique_ptr Sqlite3Engine::open(const std::string& filename) { sqlite3* new_db; + const std::string dirname = utils::dirname(filename); + std::filesystem::create_directories(dirname); auto res = sqlite3_open_v2(filename.data(), &new_db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); if (res != SQLITE_OK) { @@ -53,7 +60,31 @@ std::unique_ptr Sqlite3Engine::open(const std::string& filename) sqlite3_close(new_db); throw std::runtime_error(""); } - return std::make_unique(new_db); + std::unique_ptr db = std::make_unique(new_db); + + // Enable Write-Ahead Logging + // https://sqlite.org/wal.html + auto sqlite_journal_mode = Config::get("sqlite_journal_mode", "WAL"); + if (sqlite_journal_mode != "none") { + if (Sqlite3Engine::is_valid_sqlite_journal_mode(sqlite_journal_mode)) { + auto [success, err_msg] = + db->raw_exec("PRAGMA journal_mode = " + sqlite_journal_mode); + if (!success) + log_error("Failed enabling SQLite WAL with error: ", err_msg); + } else { + log_error("Invalid config option sqlite_journal_mode: ", + sqlite_journal_mode); + } + } + for (const auto &s : Sqlite3Engine::startup_commands) { + auto [success, err_msg] = db->raw_exec(s); + if (!success) { + log_error("Failed running startup command: ", s, + "; with error: ", err_msg); + } + } + + return db; } std::tuple Sqlite3Engine::raw_exec(const std::string& query) @@ -97,4 +128,17 @@ std::string Sqlite3Engine::id_column_type() return "INTEGER PRIMARY KEY AUTOINCREMENT"; } +DatabaseType Sqlite3Engine::get_type() { + return DatabaseType::sqlite3; +}; + + +bool Sqlite3Engine::is_valid_sqlite_journal_mode(std::string &sqlite_journal_mode) { + // https://sqlite.org/pragma.html#pragma_journal_mode + for (const std::string &s : Sqlite3Engine::valid_sqlite_journal_modes) { + if (sqlite_journal_mode == s) + return true; + } + return false; +} #endif diff --git a/src/database/sqlite3_engine.hpp b/src/database/sqlite3_engine.hpp index a7bfcdb..9e29bed 100644 --- a/src/database/sqlite3_engine.hpp +++ b/src/database/sqlite3_engine.hpp @@ -29,8 +29,29 @@ class Sqlite3Engine: public DatabaseEngine std::unique_ptr prepare(const std::string& query) override; void extract_last_insert_rowid(Statement& statement) override; std::string id_column_type() override; + DatabaseType get_type() override final; + private: sqlite3* const db; + constexpr static const char *const startup_commands[3] = { + // Wait for a maximum of 5 seconds if a write is busy + "PRAGMA busy_timeout = 5000", + // Reduce disk syncs + // >When synchronous is NORMAL (1), the SQLite database engine will still + // >sync at the most critical moments + // https://sqlite.org/pragma.html#pragma_synchronous + "PRAGMA synchronous = NORMAL", + // Run PRAGMA command early to analyze all tables, as recommended by + // sqlite + // >Applications that use long-lived database connections should run + // >"PRAGMA optimize=0x10002;" when the connection is first opened + // https://sqlite.org/lang_analyze.html + "PRAGMA optimize=0x10002", + }; + constexpr static std::string valid_sqlite_journal_modes[6] = { + "DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF", + }; + static bool is_valid_sqlite_journal_mode(std::string &sqlite_journal_mode); }; #else diff --git a/src/database/table.hpp b/src/database/table.hpp index 0b8bfc0..2f63784 100644 --- a/src/database/table.hpp +++ b/src/database/table.hpp @@ -5,7 +5,6 @@ #include #include -#include #include #include diff --git a/src/scheduler/scheduler.cpp b/src/scheduler/scheduler.cpp index 3050396..aee2503 100644 --- a/src/scheduler/scheduler.cpp +++ b/src/scheduler/scheduler.cpp @@ -1,10 +1,12 @@ #include "biboumi.h" #include +#include #include #include #include #ifdef USE_DATABASE +#include #include #endif @@ -13,8 +15,10 @@ static const std::unordered_map< events = { #ifdef USE_DATABASE {"cleanup_old_messages", - {60s * 10 /* 10min */, &Scheduler::cleanup_old_messages}}, -#endif + {std::chrono::seconds(10min), &Scheduler::cleanup_old_messages}}, + {"optimize_db", + {std::chrono::seconds(24h), &Scheduler::optimize_db}}, +#endif // USE_DATABASE }; void Scheduler::start_scheduler() { @@ -65,4 +69,39 @@ void Scheduler::cleanup_old_messages() { "archive_expires_after is ", archive_expires_after); } -#endif + +void Scheduler::optimize_db() { + log_debug("[optimize_db] Starting. Type: ", (int)Database::db->get_type()); + switch (Database::db->get_type()) { +# ifdef SQLITE3_FOUND + case DatabaseType::sqlite3: { + log_debug("[optimize_db] Optimizing Sqlite database"); + // Run PRAGMA optimize periodically, as recommended by sqlite + // https://sqlite.org/lang_analyze.html#automatically_running_analyze + auto [success, err_msg] = Database::db->raw_exec("PRAGMA optimize"); + if (!success) { + log_error("[optimize_db] Failed optimizing Sqlite database: ", err_msg); + } + break; + } +# endif // SQLITE3_FOUND +# ifdef PQ_FOUND + case DatabaseType::postgresql: { + log_debug("[optimize_db] Optimizing PostgreSQL database"); + // Run VACUUM and ANALYZE periodically + // https://www.postgresql.org/docs/current/sql-vacuum.html + // https://www.postgresql.org/docs/current/sql-analyze.html + auto [success, err_msg] = Database::db->raw_exec("VACUUM ANALYZE"); + if (!success) { + log_error("[optimize_db] Failed optimizing PostgreSQL database: ", + err_msg); + } + break; + } +# endif // PQ_FOUND + case DatabaseType::none: + default: + break; + } +} +#endif // USE_DATABASE diff --git a/src/scheduler/scheduler.hpp b/src/scheduler/scheduler.hpp index b35aeaa..63e56cc 100644 --- a/src/scheduler/scheduler.hpp +++ b/src/scheduler/scheduler.hpp @@ -7,4 +7,5 @@ public: // Jobs static void cleanup_old_messages(); + static void optimize_db(); }; diff --git a/src/utils/timed_events.hpp b/src/utils/timed_events.hpp index fa0fc50..0ab3246 100644 --- a/src/utils/timed_events.hpp +++ b/src/utils/timed_events.hpp @@ -22,7 +22,7 @@ class TimedEvent friend class TimedEventsManager; public: /** - * An event the occurs only once, at the given time_point + * An event that occurs only once, at the given time_point */ explicit TimedEvent(std::chrono::steady_clock::time_point&& time_point, std::function callback, std::string name="");