[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Freeipa-devel] [PATCHES][SSSD] SBUS Reconnection logic



The first patch implements reconnection logic in the SBUS itself. It
will keep track of any outstanding requests while the reconnection is
going on and submit them once reconnection succeeds.

The second patch enables the Data Provider backends to take advantage of
the auto-reconnection logic and can serve as a reference implementation
for doing the same in NSS and PAM (forthcoming)
-- 
Stephen Gallagher
RHCE 804006346421761

Looking to carve out IT costs?
www.redhat.com/carveoutcosts/
From 4edc97327e04d6eb7d82e0c9e2631d4522f0d31c Mon Sep 17 00:00:00 2001
From: Stephen Gallagher <sgallagh redhat com>
Date: Wed, 18 Mar 2009 12:05:14 -0400
Subject: [PATCH] Add reconnection logic to the SBUS

Any client of the SBUS that wants to implement automatic
reconnection may now call sbus_reconnect_init to set it up.
The clients will need to set up a callback to handle the result
of the reconnection and (in the case of a successful reconnection)
readd the method handlers to the connection context.
---
 server/sbus/sssd_dbus.h            |   17 +++
 server/sbus/sssd_dbus_connection.c |  190 +++++++++++++++++++++++++++++++++++-
 2 files changed, 206 insertions(+), 1 deletions(-)

diff --git a/server/sbus/sssd_dbus.h b/server/sbus/sssd_dbus.h
index 2496ae8..b9c7a26 100644
--- a/server/sbus/sssd_dbus.h
+++ b/server/sbus/sssd_dbus.h
@@ -35,6 +35,8 @@ typedef int (*sbus_msg_handler_fn)(DBusMessage *, struct sbus_conn_ctx *);
  */
 typedef int (*sbus_conn_destructor_fn)(void *);
 
+typedef void (*sbus_conn_reconn_callback_fn)(struct sbus_conn_ctx *, int, void *);
+
 /*
  * sbus_server_conn_init_fn
  * Set up function for connection-specific activities
@@ -48,6 +50,12 @@ enum {
     SBUS_CONN_TYPE_SHARED
 };
 
+enum {
+    SBUS_RECONNECT_SUCCESS = 1,
+    SBUS_RECONNECT_EXCEEDED_RETRIES,
+    SBUS_RECONNECT_ERROR
+};
+
 /* Special interface and method for D-BUS introspection */
 #define DBUS_INTROSPECT_INTERFACE "org.freedesktop.DBus.Introspectable"
 #define DBUS_INTROSPECT_METHOD "Introspect"
@@ -117,6 +125,15 @@ int sbus_conn_add_method_ctx(struct sbus_conn_ctx *conn_ctx,
                              struct sbus_method_ctx *method_ctx);
 bool sbus_conn_disconnecting(struct sbus_conn_ctx *conn_ctx);
 
+/* max_retries < 0: retry forever
+ * max_retries = 0: never retry (why are you calling this function?)
+ * max_retries > 0: obvious
+ */
+void sbus_reconnect_init(struct sbus_conn_ctx *conn_ctx,
+                         int max_retries,
+                         sbus_conn_reconn_callback_fn callback,
+                         void *pvt);
+
 /* Default message handler
  * Should be usable for most cases */
 DBusHandlerResult sbus_message_handler(DBusConnection *conn,
diff --git a/server/sbus/sssd_dbus_connection.c b/server/sbus/sssd_dbus_connection.c
index 6dc3130..3629957 100644
--- a/server/sbus/sssd_dbus_connection.c
+++ b/server/sbus/sssd_dbus_connection.c
@@ -10,11 +10,18 @@ struct dbus_ctx_list;
 struct sbus_conn_ctx {
     DBusConnection *conn;
     struct tevent_context *ev;
+    char *address;
     int connection_type;
     int disconnect;
     struct sbus_method_ctx *method_ctx_list;
     sbus_conn_destructor_fn destructor;
     void *pvt_data; /* Private data for this connection */
+
+    int retries;
+    int max_retries;
+    sbus_conn_reconn_callback_fn reconnect_callback;
+    /* Private data needed to reinit after reconnection */
+    void *reconnect_pvt;
 };
 
 struct sbus_message_handler_ctx {
@@ -39,6 +46,8 @@ static int _method_list_contains_path(struct sbus_method_ctx *list,
                                       struct sbus_method_ctx *method);
 static void sbus_unreg_object_paths(struct sbus_conn_ctx *dct_ctx);
 
+static int sbus_auto_reconnect(struct sbus_conn_ctx *conn_ctx);
+
 static void sbus_dispatch(struct tevent_context *ev,
                                struct tevent_timer *te,
                                struct timeval tv, void *data)
@@ -55,6 +64,32 @@ static void sbus_dispatch(struct tevent_context *ev,
     conn = dct_ctx->conn;
     DEBUG(6, ("conn: %lX\n", conn));
 
+    if (dct_ctx->retries > 0) {
+        DEBUG(6, ("SBUS is reconnecting. Deferring.\n"));
+        /* Currently trying to reconnect, defer dispatch */
+        new_event = tevent_add_timer(ev, dct_ctx, tv, sbus_dispatch, dct_ctx);
+        if (new_event == NULL) {
+            DEBUG(0,("Could not defer dispatch!\n"));
+        }
+        return;
+    }
+
+    if ((!dbus_connection_get_is_connected(conn)) &&
+        (dct_ctx->max_retries != 0)) {
+        /* Attempt to reconnect automatically */
+        ret = sbus_auto_reconnect(dct_ctx);
+        if (ret == EOK) {
+            DEBUG(1, ("Performing auto-reconnect\n"));
+            return;
+        }
+
+        DEBUG(0, ("Cannot start auto-reconnection.\n"));
+        dct_ctx->reconnect_callback(dct_ctx,
+                                    SBUS_RECONNECT_ERROR,
+                                    dct_ctx->reconnect_pvt);
+        return;
+    }
+
     if((dct_ctx->disconnect) || (!dbus_connection_get_is_connected(conn))) {
         DEBUG(3,("Connection is not open for dispatching.\n"));
         /*
@@ -266,6 +301,8 @@ static void sbus_conn_wakeup_main(void *data)
     }
 }
 
+static int sbus_add_connection_int(struct sbus_conn_ctx **dct_ctx);
+
 /*
  * integrate_connection_with_event_loop
  * Set up a D-BUS connection to use the libevents mainloop
@@ -277,18 +314,39 @@ int sbus_add_connection(TALLOC_CTX *ctx,
                         struct sbus_conn_ctx **dct_ctx,
                         int connection_type)
 {
-    dbus_bool_t dbret;
     struct sbus_conn_ctx *dt_ctx;
+    int ret;
 
     DEBUG(5,("Adding connection %lX\n", dbus_conn));
     dt_ctx = talloc_zero(ctx, struct sbus_conn_ctx);
+
     dt_ctx->ev = ev;
     dt_ctx->conn = dbus_conn;
     dt_ctx->connection_type = connection_type;
     dt_ctx->disconnect = 0;
+
     /* This will be replaced on the first call to sbus_conn_add_method_ctx() */
     dt_ctx->method_ctx_list = NULL;
 
+    /* This can be overridden by a call to sbus_reconnect_init() */
+    dt_ctx->retries = 0;
+    dt_ctx->max_retries = 0;
+    dt_ctx->reconnect_callback = NULL;
+
+    ret = sbus_add_connection_int(&dt_ctx);
+    if (ret != EOK) {
+        talloc_free(dt_ctx);
+        return ret;
+    }
+    *dct_ctx = dt_ctx;
+    return EOK;
+}
+
+static int sbus_add_connection_int(struct sbus_conn_ctx **dct_ctx)
+{
+    dbus_bool_t dbret;
+    struct sbus_conn_ctx *dt_ctx = *dct_ctx;
+
     /*
      * Set the default destructor
      * Connections can override this with
@@ -368,6 +426,9 @@ int sbus_new_connection(TALLOC_CTX *ctx, struct tevent_context *ev,
         /* FIXME: release resources */
     }
 
+    /* Store the address for later reconnection */
+    (*dct_ctx)->address = talloc_strdup((*dct_ctx), address);
+
     dbus_connection_set_exit_on_disconnect((*dct_ctx)->conn, FALSE);
 
     /* Set connection destructor */
@@ -659,6 +720,133 @@ void *sbus_conn_get_private_data(struct sbus_conn_ctx *conn_ctx)
     return conn_ctx->pvt_data;
 }
 
+static void sbus_reconnect(struct tevent_context *ev,
+                           struct tevent_timer *te,
+                           struct timeval tv, void *data)
+{
+    DBusConnection *dbus_conn;
+    DBusError dbus_error;
+    struct tevent_timer *event;
+    struct sbus_method_ctx *iter;
+    struct sbus_method_ctx *purge;
+    int ret;
+    struct sbus_conn_ctx *conn_ctx =
+        talloc_get_type(data, struct sbus_conn_ctx);
+
+    DEBUG(3, ("Making reconnection attempt %d to [%s]\n", conn_ctx->retries, conn_ctx->address));
+    /* Make a new connection to the D-BUS address */
+    dbus_error_init(&dbus_error);
+    dbus_conn = dbus_connection_open(conn_ctx->address, &dbus_error);
+    if (dbus_conn) {
+        /* We successfully reconnected. Set up mainloop
+         * integration.
+         */
+        DEBUG(3, ("Reconnected to [%s]\n", conn_ctx->address));
+        conn_ctx->conn = dbus_conn;
+        ret = sbus_add_connection_int(&conn_ctx);
+        if (ret != EOK) {
+            dbus_connection_unref(dbus_conn);
+            goto failed;
+        }
+
+        /* Remove object paths (the reconnection callback must re-add these */
+        iter = conn_ctx->method_ctx_list;
+        while (iter != NULL) {
+            DLIST_REMOVE(conn_ctx->method_ctx_list, iter);
+            purge = iter;
+            iter = iter->next;
+            talloc_unlink(conn_ctx, purge);
+        }
+
+        /* Reset retries to 0 to resume dispatch processing */
+        conn_ctx->retries = 0;
+
+        /* Notify the owner of this connection that the
+         * reconnection was successful
+         */
+        conn_ctx->reconnect_callback(conn_ctx,
+                                     SBUS_RECONNECT_SUCCESS,
+                                     conn_ctx->reconnect_pvt);
+        return;
+    }
+
+failed:
+    /* Reconnection failed, try again in a few seconds */
+    DEBUG(1, ("Failed to open connection: name=%s, message=%s\n",
+                dbus_error.name, dbus_error.message));
+    if (dbus_error_is_set(&dbus_error)) dbus_error_free(&dbus_error);
+
+    conn_ctx->retries++;
+
+    /* Check if we've passed our last chance or if we've lost track of
+     * our retry count somehow
+     */
+    if (((conn_ctx->max_retries > 0) &&
+         (conn_ctx->retries > conn_ctx->max_retries)) ||
+        conn_ctx->retries <= 0) {
+        conn_ctx->reconnect_callback(conn_ctx,
+                                     SBUS_RECONNECT_EXCEEDED_RETRIES,
+                                     conn_ctx->reconnect_pvt);
+    }
+
+    if (conn_ctx->retries == 2) {
+        tv.tv_sec += 3; /* Wait 3 seconds before the second reconnect attempt */
+    }
+    else if (conn_ctx->retries == 3) {
+        tv.tv_sec += 10; /* Wait 10 seconds before the third reconnect attempt */
+    }
+    else {
+        tv.tv_sec += 30; /* Wait 30 seconds before all subsequent reconnect attempts */
+    }
+    event = tevent_add_timer(conn_ctx->ev, conn_ctx, tv, sbus_reconnect, conn_ctx);
+    if (event == NULL) {
+        conn_ctx->reconnect_callback(conn_ctx,
+                                     SBUS_RECONNECT_ERROR,
+                                     conn_ctx->reconnect_pvt);
+    }
+}
+
+/* This function will free and recreate the sbus_conn_ctx,
+ * calling functions need to be aware of this (and whether
+ * they have attached a talloc destructor to the
+ * sbus_conn_ctx.
+ */
+static int sbus_auto_reconnect(struct sbus_conn_ctx *conn_ctx)
+{
+    struct tevent_timer *te = NULL;
+    struct timeval tv;
+
+    conn_ctx->retries++;
+    if ((conn_ctx->max_retries > 0) &&
+        (conn_ctx->retries >= conn_ctx->max_retries)) {
+        /* Return EAGAIN (to tell the calling process it
+         * needs to create a new connection from scratch
+         */
+        return EAGAIN;
+    }
+
+    gettimeofday(&tv, NULL);
+    tv.tv_sec += 1; /* Wait 1 second before the first reconnect attempt */
+    te = tevent_add_timer(conn_ctx->ev, conn_ctx, tv, sbus_reconnect, conn_ctx);
+    if (te == NULL) return EAGAIN;
+
+    return EOK;
+}
+
+/* Max retries */
+void sbus_reconnect_init(struct sbus_conn_ctx *conn_ctx,
+                         int max_retries,
+                         sbus_conn_reconn_callback_fn callback,
+                         void *pvt)
+{
+    if(max_retries == 0 || callback == NULL) return;
+
+    conn_ctx->retries = 0;
+    conn_ctx->max_retries = max_retries;
+    conn_ctx->reconnect_callback = callback;
+    conn_ctx->reconnect_pvt = pvt;
+}
+
 bool sbus_conn_disconnecting(struct sbus_conn_ctx *conn_ctx)
 {
     if (conn_ctx->disconnect == 1) return true;
-- 
1.6.0.6

From 1985475f0bdd0990cb7176cd6596af2b3e26506a Mon Sep 17 00:00:00 2001
From: Stephen Gallagher <sgallagh redhat com>
Date: Wed, 18 Mar 2009 12:19:23 -0400
Subject: [PATCH] Enable autoreconnection of Data Provider Backends to the Data Provider

---
 server/providers/data_provider_be.c |   87 +++++++++++++++++++++++++++++++++-
 server/providers/dp_backend.h       |    8 +++-
 server/providers/ldap_be.c          |   15 ++++--
 server/providers/proxy.c            |   13 ++++-
 4 files changed, 112 insertions(+), 11 deletions(-)

diff --git a/server/providers/data_provider_be.c b/server/providers/data_provider_be.c
index c9faff9..158f436 100644
--- a/server/providers/data_provider_be.c
+++ b/server/providers/data_provider_be.c
@@ -676,11 +676,13 @@ static int mon_cli_init(struct be_ctx *ctx)
     return EOK;
 }
 
+static void be_cli_reconnect_init(struct sbus_conn_ctx *sconn, int status, void *pvt);
+
 /* be_cli_init
  * sbus channel to the data provider daemon */
 static int be_cli_init(struct be_ctx *ctx)
 {
-    int ret;
+    int ret, max_retries;
     char *sbus_address;
     struct sbus_method_ctx *sm_ctx;
 
@@ -707,9 +709,82 @@ static int be_cli_init(struct be_ctx *ctx)
         return ret;
     }
 
+    /* Enable automatic reconnection to the Data Provider */
+    ret = confdb_get_int(ctx->cdb, ctx, ctx->conf_path,
+                         "retries", 3, &max_retries);
+    if (ret != EOK) {
+        DEBUG(0, ("Failed to set up automatic reconnection\n"));
+        return ret;
+    }
+
+    sbus_reconnect_init(ctx->dp_ctx->scon_ctx, max_retries,
+                        be_cli_reconnect_init, ctx);
+
     return EOK;
 }
 
+static void be_finalize(struct be_ctx *ctx);
+
+static void be_cli_reconnect_init(struct sbus_conn_ctx *sconn, int status, void *pvt)
+{
+    int ret;
+    struct be_ctx *be_ctx = talloc_get_type(pvt, struct be_ctx);
+
+    /* Did we reconnect successfully? */
+    if (status == SBUS_RECONNECT_SUCCESS) {
+        /* Add the methods back to the new connection */
+        ret = sbus_conn_add_method_ctx(be_ctx->dp_ctx->scon_ctx,
+                                       be_ctx->dp_ctx->sm_ctx);
+        if (ret != EOK) {
+            DEBUG(0, ("Could not re-add methods on reconnection.\n"));
+            be_finalize(be_ctx);
+        }
+
+        DEBUG(1, ("Reconnected to the Data Provider.\n"));
+        return;
+    }
+
+    /* Handle failure */
+    DEBUG(0, ("Could not reconnect to data provider.\n"));
+    /* Kill the backend and let the monitor restart it */
+    be_finalize(be_ctx);
+}
+
+static void be_shutdown(struct be_req *req, int status,
+                        const char *errstr)
+{
+    /* Nothing left to do but exit() */
+    if (status == EOK)
+        exit(0);
+
+    /* Something went wrong in finalize */
+    exit(1);
+}
+
+static void be_finalize(struct be_ctx *ctx)
+{
+    int ret;
+    struct be_req *shutdown_req = talloc_zero(ctx, struct be_req);
+    if (!shutdown_req) {
+        goto fail;
+    }
+
+    shutdown_req->be_ctx = ctx;
+    shutdown_req->fn = be_shutdown;
+
+    shutdown_req->pvt = ctx->pvt_data;
+
+    ret = be_file_request(ctx, ctx->ops->finalize, shutdown_req);
+    if (ret == EOK) return;
+
+fail:
+    /* If we got here, we couldn't shut down cleanly.
+     * So we'll just call exit(1) instead
+     */
+    DEBUG(0, ("ERROR: could not shut down cleanly.\n"));
+    exit(1);
+}
+
 static int load_backend(struct be_ctx *ctx)
 {
     TALLOC_CTX *tmp_ctx;
@@ -857,13 +932,19 @@ int main(int argc, const char *argv[])
     if (!srv_name) return 2;
 
     ret = server_setup(srv_name, 0, &main_ctx);
-    if (ret != EOK) return 2;
+    if (ret != EOK) {
+        DEBUG(0, ("Could not set up mainloop [%d]\n", ret));
+        return 2;
+    }
 
     ret = be_process_init(main_ctx,
                           be_name, be_domain,
                           main_ctx->event_ctx,
                           main_ctx->confdb_ctx);
-    if (ret != EOK) return 3;
+    if (ret != EOK) {
+        DEBUG(0, ("Could not initialize backend [%d]\n", ret));
+        return 3;
+    }
 
     DEBUG(1, ("Backend provider %s(%s) started!\n", be_name, be_domain));
 
diff --git a/server/providers/dp_backend.h b/server/providers/dp_backend.h
index df09544..12cfb3a 100644
--- a/server/providers/dp_backend.h
+++ b/server/providers/dp_backend.h
@@ -26,6 +26,10 @@
 #include "db/sysdb.h"
 #include "responder/pam/pamsrv.h"
 
+struct be_ctx;
+
+typedef void (*be_shutdown_fn)(void *);
+
 struct be_mod_ops;
 
 struct be_ctx {
@@ -40,6 +44,7 @@ struct be_ctx {
     const char *conf_path;
     struct be_mod_ops *ops;
     void *pvt_data;
+    be_shutdown_fn shutdown;
 };
 
 struct be_req;
@@ -66,7 +71,7 @@ struct be_online_req {
 };
 
 struct be_pam_handler {
-    int pam_status; 
+    int pam_status;
     const char *domain;
     struct pam_data *pd;
 };
@@ -77,6 +82,7 @@ struct be_mod_ops {
     be_req_fn_t check_online;
     be_req_fn_t get_account_info;
     be_req_fn_t pam_handler;
+    be_req_fn_t finalize;
 };
 
 #endif /* __DP_BACKEND_H___ */
diff --git a/server/providers/ldap_be.c b/server/providers/ldap_be.c
index 8967732..2626058 100644
--- a/server/providers/ldap_be.c
+++ b/server/providers/ldap_be.c
@@ -96,7 +96,7 @@ static int schedule_next_task(struct ldap_req *lr, struct timeval tv,
     timeout.tv_usec += tv.tv_usec;
 
 
-    te = tevent_add_timer(lr->req->be_ctx->ev, lr, timeout, task, lr); 
+    te = tevent_add_timer(lr->req->be_ctx->ev, lr, timeout, task, lr);
     if (te == NULL) {
         return EIO;
     }
@@ -281,7 +281,7 @@ static void ldap_be_loop(struct tevent_context *ev, struct tevent_fd *te,
     char *filter=NULL;
     char *attrs[] = { LDAP_NO_ATTRS, NULL };
 
-    lr = talloc_get_type(pvt, struct ldap_req); 
+    lr = talloc_get_type(pvt, struct ldap_req);
 
     switch (lr->next_op) {
         case LDAP_OP_INIT:
@@ -550,7 +550,7 @@ static void ldap_start(struct tevent_context *ev, struct tevent_timer *te,
     struct be_req *req;
     struct be_pam_handler *ph;
 
-    lr = talloc_get_type(pvt, struct ldap_req); 
+    lr = talloc_get_type(pvt, struct ldap_req);
 
     ret = ldap_be_init(lr);
     if (ret != EOK) {
@@ -624,10 +624,17 @@ done:
     req->fn(req, pam_status, NULL);
 }
 
+static void ldap_shutdown(struct be_req *req)
+{
+    /* TODO: Clean up any internal data */
+    req->fn(req, EOK, NULL);
+}
+
 struct be_mod_ops ldap_mod_ops = {
     .check_online = NULL,
     .get_account_info = NULL,
-    .pam_handler = ldap_pam_handler
+    .pam_handler = ldap_pam_handler,
+    .finalize = ldap_shutdown
 };
 
 
diff --git a/server/providers/proxy.c b/server/providers/proxy.c
index 30f5f5c..8eaaf66 100644
--- a/server/providers/proxy.c
+++ b/server/providers/proxy.c
@@ -85,8 +85,8 @@ static int proxy_internal_conv(int num_msg, const struct pam_message **msgm,
         switch( msgm[i]->msg_style ) {
             case PAM_PROMPT_ECHO_OFF:
                 DEBUG(4, ("Conversation message: %s.\n", msgm[i]->msg));
-                reply[i].resp_retcode = 0; 
-                reply[i].resp = strdup(auth_data->authtok); 
+                reply[i].resp_retcode = 0;
+                reply[i].resp = strdup(auth_data->authtok);
                 break;
             default:
                 DEBUG(1, ("Conversation style %d not supported.\n",
@@ -1052,10 +1052,17 @@ static void proxy_get_account_info(struct be_req *req)
     }
 }
 
+static void proxy_shutdown(struct be_req *req)
+{
+    /* TODO: Clean up any internal data */
+    req->fn(req, EOK, NULL);
+}
+
 struct be_mod_ops proxy_mod_ops = {
     .check_online = proxy_check_online,
     .get_account_info = proxy_get_account_info,
-    .pam_handler = proxy_pam_handler
+    .pam_handler = proxy_pam_handler,
+    .finalize = proxy_shutdown
 };
 
 static void *proxy_dlsym(void *handle, const char *functemp, char *libname)
-- 
1.6.0.6

Attachment: signature.asc
Description: OpenPGP digital signature


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]