Merge branch 'biboumi_jid_in_users_rosters' into 'master'

Use a db roster to manage biboumi’s presence with the contacts

Closes #3217

See merge request !13

CHANGELOG.rst

@@ -3,6 +3,9 @@ Version 6.0
 
  - The LiteSQL dependency was removed. Only libsqlite3 is now necessary
    to work with the database.
+ - Some JIDs can be added into users’ rosters. The component JID tells if
+   biboumi is started or not, and the IRC-server JIDs tell if the user is
+   currently connected to that server.
  - The RecordHistory option can now also be configured for each IRC channel,
    individually.
  - Add a global option to make all channels persistent.

doc/biboumi.1.rst

@@ -344,6 +344,29 @@ connect you to the IRC server without joining any channel), then send your
 authentication message to the user ``bot%irc.example.com@biboumi.example.com``
 and finally join the room ``#foo%irc.example.com@biboumi.example.com``.
 
+Roster
+------
+
+You can add some JIDs provided by biboumi into your own roster, to receive
+presence from them. Biboumi will always automatically accept your requests.
+
+Biboumi’s JID
+-------------
+
+By adding the component JID into your roster, the user will receive an available
+presence whenever it is started, and an unavailable presence whenever it is being
+shutdown.  This is useful to quickly view if that biboumi instance is started or
+not.
+
+IRC server JID
+--------------
+
+These presence will appear online in the user’s roster whenever they are
+connected to that IRC server (see *Connect to an IRC server* for more
+details). This is useful to keep track of which server an user is connected
+to: this is sometimes hard to remember, when they have many clients, or if
+they are using persistent channels.
+
 Channel messages
 ----------------
 

src/bridge/bridge.cpp

@@ -1081,6 +1081,16 @@ void Bridge::send_xmpp_invitation(const Iid& iid, const std::string& author)
     this->xmpp.send_invitation(std::to_string(iid), this->user_jid + "/" + resource, author);
 }
 
+void Bridge::on_irc_client_connected(const std::string& hostname)
+{
+  this->xmpp.on_irc_client_connected(hostname, this->user_jid);
+}
+
+void Bridge::on_irc_client_disconnected(const std::string& hostname)
+{
+  this->xmpp.on_irc_client_disconnected(hostname, this->user_jid);
+}
+
 void Bridge::set_preferred_from_jid(const std::string& nick, const std::string& full_jid)
 {
   auto it = this->preferred_user_from.find(nick);

src/bridge/bridge.hpp

@@ -201,6 +201,8 @@ public:
   void send_xmpp_ping_request(const std::string& nick, const std::string& hostname,
                               const std::string& id);
   void send_xmpp_invitation(const Iid& iid, const std::string& author);
+  void on_irc_client_connected(const std::string& hostname);
+  void on_irc_client_disconnected(const std::string& hostname);
 
   /**
    * Misc
@@ -301,8 +303,8 @@ private:
   using ChannelKey = std::tuple<ChannelName, IrcHostname>;
 public:
   std::map<ChannelKey, std::set<Resource>> resources_in_chan;
-private:
   std::map<IrcHostname, std::set<Resource>> resources_in_server;
+private:
   /**
    * Manage which resource is in which channel
    */

src/database/database.cpp

@@ -13,6 +13,8 @@ Database::MucLogLineTable Database::muc_log_lines("MucLogLine_");
 Database::GlobalOptionsTable Database::global_options("GlobalOptions_");
 Database::IrcServerOptionsTable Database::irc_server_options("IrcServerOptions_");
 Database::IrcChannelOptionsTable Database::irc_channel_options("IrcChannelOptions_");
+Database::RosterTable Database::roster("roster");
+
 
 void Database::open(const std::string& filename)
 {
@@ -36,6 +38,8 @@ void Database::open(const std::string& filename)
   Database::irc_server_options.upgrade(Database::db);
   Database::irc_channel_options.create(Database::db);
   Database::irc_channel_options.upgrade(Database::db);
+  Database::roster.create(Database::db);
+  Database::roster.upgrade(Database::db);
 }
 
 
@@ -177,6 +181,51 @@ std::vector<Database::MucLogLine> Database::get_muc_logs(const std::string& owne
   return {result.crbegin(), result.crend()};
 }
 
+void Database::add_roster_item(const std::string& local, const std::string& remote)
+{
+  auto roster_item = Database::roster.row();
+
+  roster_item.col<Database::LocalJid>() = local;
+  roster_item.col<Database::RemoteJid>() = remote;
+
+  roster_item.save(Database::db);
+}
+
+void Database::delete_roster_item(const std::string& local, const std::string& remote)
+{
+  Query query("DELETE FROM "s + Database::roster.get_name());
+  query << " WHERE " << Database::RemoteJid{} << "=" << remote << \
+           " AND " << Database::LocalJid{} << "=" << local;
+
+  query.execute(Database::db);
+}
+
+bool Database::has_roster_item(const std::string& local, const std::string& remote)
+{
+  auto query = Database::roster.select();
+  query.where() << Database::LocalJid{} << "=" << local << \
+        " and " << Database::RemoteJid{} << "=" << remote;
+
+  auto res = query.execute(Database::db);
+
+  return !res.empty();
+}
+
+std::vector<Database::RosterItem> Database::get_contact_list(const std::string& local)
+{
+  auto query = Database::roster.select();
+  query.where() << Database::LocalJid{} << "=" << local;
+
+  return query.execute(Database::db);
+}
+
+std::vector<Database::RosterItem> Database::get_full_roster()
+{
+  auto query = Database::roster.select();
+
+  return query.execute(Database::db);
+}
+
 void Database::close()
 {
   sqlite3_close_v2(Database::db);
@@ -192,4 +241,4 @@ std::string Database::gen_uuid()
   return uuid_str;
 }
 
-#endif
+#endif

src/database/database.hpp

@@ -72,6 +72,11 @@ class Database
   struct Persistent: Column<bool> { static constexpr auto name = "persistent_";
     Persistent(): Column<bool>(false) {} };
 
+  struct LocalJid: Column<std::string> { static constexpr auto name = "local"; };
+
+  struct RemoteJid: Column<std::string> { static constexpr auto name = "remote"; };
+
+
   using MucLogLineTable = Table<Id, Uuid, Owner, IrcChanName, IrcServerName, Date, Body, Nick>;
   using MucLogLine = MucLogLineTable::RowType;
 
@@ -84,6 +89,9 @@ class Database
   using IrcChannelOptionsTable = Table<Id, Owner, Server, Channel, EncodingOut, EncodingIn, MaxHistoryLength, Persistent, RecordHistoryOptional>;
   using IrcChannelOptions = IrcChannelOptionsTable::RowType;
 
+  using RosterTable = Table<LocalJid, RemoteJid>;
+  using RosterItem = RosterTable::RowType;
+
   Database() = default;
   ~Database() = default;
 
@@ -109,6 +117,12 @@ class Database
   static std::string store_muc_message(const std::string& owner, const std::string& chan_name, const std::string& server_name,
                                        time_point date, const std::string& body, const std::string& nick);
 
+  static void add_roster_item(const std::string& local, const std::string& remote);
+  static bool has_roster_item(const std::string& local, const std::string& remote);
+  static void delete_roster_item(const std::string& local, const std::string& remote);
+  static std::vector<Database::RosterItem> get_contact_list(const std::string& local);
+  static std::vector<Database::RosterItem> get_full_roster();
+
   static void close();
   static void open(const std::string& filename);
 
@@ -123,6 +137,7 @@ class Database
   static GlobalOptionsTable global_options;
   static IrcServerOptionsTable irc_server_options;
   static IrcChannelOptionsTable irc_channel_options;
+  static RosterTable roster;
   static sqlite3* db;
 
  private:

src/irc/irc_client.cpp

@@ -297,6 +297,7 @@ void IrcClient::on_connected()
 #endif
   this->send_gateway_message("Connected to IRC server"s + (this->use_tls ? " (encrypted)": "") + ".");
   this->send_pending_data();
+  this->bridge.on_irc_client_connected(this->get_hostname());
 }
 
 void IrcClient::on_connection_close(const std::string& error_msg)
@@ -309,6 +310,7 @@ void IrcClient::on_connection_close(const std::string& error_msg)
   const IrcMessage error{"ERROR", {message}};
   this->on_error(error);
   log_warning(message);
+  this->bridge.on_irc_client_disconnected(this->get_hostname());
 }
 
 IrcChannel* IrcClient::get_channel(const std::string& n)

src/xmpp/biboumi_component.cpp

@@ -83,6 +83,14 @@ void BiboumiComponent::shutdown()
 {
   for (auto& pair: this->bridges)
     pair.second->shutdown("Gateway shutdown");
+#ifdef USE_DATABASE
+  for (const Database::RosterItem& roster_item: Database::get_full_roster())
+    {
+      this->send_presence_to_contact(roster_item.col<Database::LocalJid>(),
+                                     roster_item.col<Database::RemoteJid>(),
+                                     "unavailable");
+    }
+#endif
 }
 
 void BiboumiComponent::clean()
@@ -160,10 +168,47 @@ void BiboumiComponent::handle_presence(const Stanza& stanza)
     {
       if (type == "subscribe")
         { // Auto-accept any subscription request for an IRC server
-          this->accept_subscription(to_str, from.bare());
-          this->ask_subscription(to_str, from.bare());
+          this->send_presence_to_contact(to_str, from.bare(), "subscribed", id);
+          if (iid.type == Iid::Type::None || bridge->find_irc_client(iid.get_server()))
+            this->send_presence_to_contact(to_str, from.bare(), "");
+          this->send_presence_to_contact(to_str, from.bare(), "subscribe");
+#ifdef USE_DATABASE
+          if (!Database::has_roster_item(to_str, from.bare()))
+            Database::add_roster_item(to_str, from.bare());
+#endif
+        }
+      else if (type == "unsubscribe")
+        {
+          this->send_presence_to_contact(to_str, from.bare(), "unavailable", id);
+          this->send_presence_to_contact(to_str, from.bare(), "unsubscribed");
+          this->send_presence_to_contact(to_str, from.bare(), "unsubscribe");
+#ifdef USE_DATABASE
+          const bool res = Database::has_roster_item(to_str, from.bare());
+          if (res)
+            Database::delete_roster_item(to_str, from.bare());
+#endif
+        }
+      else if (type == "probe")
+        {
+          if ((iid.type == Iid::Type::Server && bridge->find_irc_client(iid.get_server()))
+              || iid.type == Iid::Type::None)
+            {
+#ifdef USE_DATABASE
+              if (Database::has_roster_item(to_str, from.bare()))
+#endif
+                this->send_presence_to_contact(to_str, from.bare(), "");
+#ifdef USE_DATABASE
+              else // rfc 6121 4.3.2.1
+                this->send_presence_to_contact(to_str, from.bare(), "unsubscribed");
+#endif
+            }
+        }
+      else if (type.empty())
+        { // We just receive a presence from someone (as the result of a probe,
+          // or a directed presence, or a normal presence change)
+          if (iid.type == Iid::Type::None)
+            this->send_presence_to_contact(to_str, from.bare(), "");
         }
-
     }
   else
     {
@@ -979,3 +1024,55 @@ void BiboumiComponent::ask_subscription(const std::string& from, const std::stri
   presence["type"] = "subscribe";
   this->send_stanza(presence);
 }
+
+void BiboumiComponent::send_presence_to_contact(const std::string& from, const std::string& to,
+                                                const std::string& type, const std::string& id)
+{
+  Stanza presence("presence");
+  presence["from"] = from;
+  presence["to"] = to;
+  if (!type.empty())
+    presence["type"] = type;
+  if (!id.empty())
+    presence["id"] = id;
+  this->send_stanza(presence);
+}
+
+void BiboumiComponent::on_irc_client_connected(const std::string& irc_hostname, const std::string& jid)
+{
+#ifdef USE_DATABASE
+  const auto local_jid = irc_hostname + "@" + this->served_hostname;
+  if (Database::has_roster_item(local_jid, jid))
+    this->send_presence_to_contact(local_jid, jid, "");
+#endif
+}
+
+void BiboumiComponent::on_irc_client_disconnected(const std::string& irc_hostname, const std::string& jid)
+{
+#ifdef USE_DATABASE
+  const auto local_jid = irc_hostname + "@" + this->served_hostname;
+  if (Database::has_roster_item(local_jid, jid))
+    this->send_presence_to_contact(irc_hostname + "@" + this->served_hostname, jid, "unavailable");
+#endif
+}
+
+void BiboumiComponent::after_handshake()
+{
+  XmppComponent::after_handshake();
+
+#ifdef USE_DATABASE
+  const auto contacts = Database::get_contact_list(this->get_served_hostname());
+
+  for (const Database::RosterItem& roster_item: contacts)
+    {
+      const auto remote_jid = roster_item.col<Database::RemoteJid>();
+      // In response, we will receive a presence indicating the
+      // contact is online, to which we will respond with our own
+      // presence.
+      // If the contact removed us from their roster while we were
+      // offline, we will receive an unsubscribed presence, letting us
+      // stay in sync.
+      this->send_presence_to_contact(this->get_served_hostname(), remote_jid, "probe");
+    }
+#endif
+}

src/xmpp/biboumi_component.hpp

@@ -36,6 +36,8 @@ public:
   BiboumiComponent& operator=(const BiboumiComponent&) = delete;
   BiboumiComponent& operator=(BiboumiComponent&&) = delete;
 
+  void after_handshake() override final;
+
   /**
    * Returns the bridge for the given user. If it does not exist, return
    * nullptr.
@@ -87,6 +89,10 @@ public:
   void send_invitation(const std::string& room_target, const std::string& jid_to, const std::string& author_nick);
   void accept_subscription(const std::string& from, const std::string& to);
   void ask_subscription(const std::string& from, const std::string& to);
+  void send_presence_to_contact(const std::string& from, const std::string& to, const std::string& type, const std::string& id="");
+  void on_irc_client_connected(const std::string& irc_hostname, const std::string& jid);
+  void on_irc_client_disconnected(const std::string& irc_hostname, const std::string& jid);
+
   /**
    * Handle the various stanza types
    */

tests/end_to_end/__main__.py

@@ -49,6 +49,8 @@ class XMPPComponent(slixmpp.BaseXMPP):
     def __init__(self, scenario, biboumi):
         super().__init__(jid="biboumi.localhost", default_ns="jabber:component:accept")
         self.is_component = True
+        self.auto_authorize = None # Do not accept or reject subscribe requests automatically
+        self.auto_subscribe = False
         self.stream_header = '<stream:stream %s %s from="%s" id="%s">' % (
             'xmlns="jabber:component:accept"',
             'xmlns:stream="%s"' % self.stream_ns,
@@ -401,11 +403,11 @@ def handshake_sequence():
             partial(send_stanza, "<handshake xmlns='jabber:component:accept'/>"))
 
 
-def connection_begin_sequence(irc_host, jid):
+def connection_begin_sequence(irc_host, jid, expected_irc_presence=False):
     jid = jid.format_map(common_replacements)
     xpath    = "/message[@to='" + jid + "'][@from='" + irc_host + "@biboumi.localhost']/body[text()='%s']"
     xpath_re = "/message[@to='" + jid + "'][@from='" + irc_host + "@biboumi.localhost']/body[re:test(text(), '%s')]"
-    return (
+    result = (
     partial(expect_stanza,
             xpath % ('Connecting to %s:6697 (encrypted)' % irc_host)),
     partial(expect_stanza,
@@ -417,8 +419,13 @@ def connection_begin_sequence(irc_host, jid):
     partial(expect_stanza,
             xpath % ('Connecting to %s:6667 (not encrypted)' % irc_host)),
     partial(expect_stanza,
-            xpath % 'Connected to IRC server.'),
+            xpath % 'Connected to IRC server.'))
+
+    if expected_irc_presence:
+        result += (partial(expect_stanza, "/presence[@from='" + irc_host + "@biboumi.localhost']"),)
+
     # These five messages can be receive in any order
+    result += (
     partial(expect_stanza,
             xpath_re % (r'^%s: (\*\*\* Checking Ident|\*\*\* Looking up your hostname\.\.\.|\*\*\* Found your hostname: .*|ACK multi-prefix|\*\*\* Got Ident response)$' % 'irc.localhost')),
     partial(expect_stanza,
@@ -431,6 +438,8 @@ def connection_begin_sequence(irc_host, jid):
             xpath_re % (r'^%s: (\*\*\* Checking Ident|\*\*\* Looking up your hostname\.\.\.|\*\*\* Found your hostname: .*|ACK multi-prefix|\*\*\* Got Ident response)$' % 'irc.localhost')),
     )
 
+    return result
+
 def connection_tls_begin_sequence(irc_host, jid):
     jid = jid.format_map(common_replacements)
     xpath    = "/message[@to='" + jid + "'][@from='" + irc_host + "@biboumi.localhost']/body[text()='%s']"
@@ -492,8 +501,8 @@ def connection_middle_sequence(irc_host, jid):
     )
 
 
-def connection_sequence(irc_host, jid):
-    return connection_begin_sequence(irc_host, jid) +\
+def connection_sequence(irc_host, jid, expected_irc_presence=False):
+    return connection_begin_sequence(irc_host, jid, expected_irc_presence) +\
            connection_middle_sequence(irc_host, jid) +\
            connection_end_sequence(irc_host, jid)
 
@@ -2671,7 +2680,62 @@ if __name__ == '__main__':
                       partial(expect_stanza, "/message[@to='{jid_two}/{resource_two}'][@type='chat']/body[text()='irc.localhost: {nick_one}: Nickname is already in use.']"),
                       partial(expect_stanza, "/presence[@type='error']/error[@type='cancel'][@code='409']/stanza:conflict"),
                       partial(send_stanza, "<presence from='{jid_two}/{resource_two}' to='#foo%{irc_server_one}/{nick_one}' type='unavailable' />")
-         ])
+         ]),
+        Scenario("basic_subscribe_unsubscribe",
+                 [
+                     handshake_sequence(),
+
+                     # Mutual subscription exchange
+                     partial(send_stanza, "<presence from='{jid_one}' to='{biboumi_host}' type='subscribe' id='subid1' />"),
+                     partial(expect_stanza, "/presence[@type='subscribed'][@id='subid1']"),
+
+                     # Get the current presence of the biboumi gateway
+                     partial(expect_stanza, "/presence"),
+
+                     partial(expect_stanza, "/presence[@type='subscribe']"),
+                     partial(send_stanza, "<presence from='{jid_one}' to='{biboumi_host}' type='subscribed' />"),
+
+
+                     # Unsubscribe
+                     partial(send_stanza, "<presence from='{jid_one}' to='{biboumi_host}' type='unsubscribe' id='unsubid1' />"),
+                     partial(expect_stanza, "/presence[@type='unavailable']"),
+                     partial(expect_stanza, "/presence[@type='unsubscribed']"),
+                     partial(expect_stanza, "/presence[@type='unsubscribe']"),
+                     partial(send_stanza, "<presence from='{jid_one}' to='{biboumi_host}' type='unavailable' />"),
+                     partial(send_stanza, "<presence from='{jid_one}' to='{biboumi_host}' type='unsubscribed' />"),
+                 ]),
+        Scenario("irc_server_presence_in_roster",
+                 [
+                     handshake_sequence(),
+
+                     # Mutual subscription exchange
+                     partial(send_stanza, "<presence from='{jid_one}' to='{irc_server_one}' type='subscribe' id='subid1' />"),
+                     partial(expect_stanza, "/presence[@type='subscribed'][@id='subid1']"),
+
+                     partial(expect_stanza, "/presence[@type='subscribe']"),
+                     partial(send_stanza, "<presence from='{jid_one}' to='{irc_server_one}' type='subscribed' />"),
+
+                     # Join a channel on that server
+                     partial(send_stanza,
+                             "<presence from='{jid_one}/{resource_one}' to='#foo%{irc_server_one}/{nick_one}' />"),
+
+                     # We must receive the IRC server presence, in the connection sequence
+                     connection_sequence("irc.localhost", '{jid_one}/{resource_one}', True),
+                     partial(expect_stanza,
+                             "/message/body[text()='Mode #foo [+nt] by {irc_host_one}']"),
+                     partial(expect_stanza,
+                             ("/presence[@to='{jid_one}/{resource_one}'][@from='#foo%{irc_server_one}/{nick_one}']/muc_user:x/muc_user:item[@affiliation='admin'][@role='moderator']",
+                             "/presence/muc_user:x/muc_user:status[@code='110']")
+                             ),
+                     partial(expect_stanza, "/message[@from='#foo%{irc_server_one}'][@type='groupchat']/subject[not(text())]"),
+
+                     # Leave the channel, and thus the IRC server
+                     partial(send_stanza, "<presence type='unavailable' from='{jid_one}/{resource_one}' to='#foo%{irc_server_one}/{nick_one}' />"),
+                     partial(expect_stanza, "/presence[@type='unavailable'][@from='#foo%{irc_server_one}/{nick_one}']"),
+                     partial(expect_stanza, "/message[@from='{irc_server_one}']/body[text()='ERROR: Closing Link: localhost (Client Quit)']"),
+                     partial(expect_stanza, "/message[@from='{irc_server_one}']/body[text()='ERROR: Connection closed.']"),
+                     partial(expect_stanza, "/presence[@from='{irc_server_one}'][@to='{jid_one}'][@type='unavailable']"),
+                 ])
     )
 
     failures = 0