This is the mail archive of the
cluster-cvs@sourceware.org
mailing list for the cluster.
cluster: RHEL53 - clogd: Fix bug 474179 (and still investigatingothers)
- From: Jonathan Brassow <jbrassow at fedoraproject dot org>
- To: cluster-cvs-relay at redhat dot com
- Date: Mon, 8 Dec 2008 23:59:01 +0000 (UTC)
- Subject: cluster: RHEL53 - clogd: Fix bug 474179 (and still investigatingothers)
Gitweb: http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=6ae48f308f488af819ae0e8b1607e81d9e9f24a7
Commit: 6ae48f308f488af819ae0e8b1607e81d9e9f24a7
Parent: 31903205785bcc174561fbccf60f485f175925d8
Author: Jonathan Brassow <jbrassow@redhat.com>
AuthorDate: Fri Dec 5 15:20:58 2008 -0600
Committer: Jonathan Brassow <jbrassow@redhat.com>
CommitterDate: Mon Dec 8 15:43:12 2008 -0600
clogd: Fix bug 474179 (and still investigating others)
Other bugs may include: 471042
and less likely: 464550
3 additions are in this checkin:
- Fixed a case where a node exiting the CPG would not
by able to send a 'checkpoint ready' notification,
because the attempt is made after leaving the CPG.
This can leave an incoming node with now way of getting
the log information necessary to get going.
- If a node quickly enters, exits, and enters a CPG. It
is possible for other nodes in the cluster to fill a
request for checkpoint data for the first 'enter', but
*after* the node has left. This information would then
collide with new checkpoints that should happen on the
next 'enter'. We fix this by clearing any checkpoint
data just before joining the CPG on entry.
- Add more debugging information that is printed out when
'clogd' is signaled with USR1.
---
cmirror/src/cluster.c | 129 ++++++++++++++++++++++++++++++++++---------------
1 files changed, 89 insertions(+), 40 deletions(-)
diff --git a/cmirror/src/cluster.c b/cmirror/src/cluster.c
index ce2cd91..cfc6d3c 100644
--- a/cmirror/src/cluster.c
+++ b/cmirror/src/cluster.c
@@ -527,10 +527,10 @@ rr_create_retry:
tfr->originator = cp->requester; /* FIXME: hack to overload meaning of originator */
strncpy(tfr->uuid, cp->uuid, CPG_MAX_NAME_LENGTH);
- /* FIXME: Clean-up and use better variable for rtn */
r = cluster_send(tfr);
if (r)
- LOG_ERROR("Failed to send checkpoint ready notice");
+ LOG_ERROR("Failed to send checkpoint ready notice: %s",
+ strerror(-r));
free(tfr);
return 0;
@@ -1246,6 +1246,55 @@ cpg_callbacks_t cpg_callbacks = {
.cpg_confchg_fn = cpg_config_callback,
};
+/*
+ * remove_checkpoint
+ * @entry
+ *
+ * Returns: 1 if checkpoint removed, 0 if no checkpoints, -EXXX on error
+ */
+int remove_checkpoint(struct clog_cpg *entry)
+{
+ int len;
+ SaNameT name;
+ SaAisErrorT rv;
+ SaCkptCheckpointHandleT h;
+
+ len = snprintf((char *)(name.value), SA_MAX_NAME_LENGTH, "bitmaps_%s_%u",
+ SHORT_UUID(entry->name.value), my_cluster_id);
+ name.length = len;
+
+open_retry:
+ rv = saCkptCheckpointOpen(ckpt_handle, &name, NULL,
+ SA_CKPT_CHECKPOINT_READ, 0, &h);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ LOG_ERROR("abort_startup: ckpt open retry");
+ usleep(1000);
+ goto open_retry;
+ }
+
+ if (rv != SA_AIS_OK)
+ return 0;
+
+ LOG_DBG("[%s] Removing checkpoint", SHORT_UUID(entry->name.value));
+unlink_retry:
+ rv = saCkptCheckpointUnlink(ckpt_handle, &name);
+ if (rv == SA_AIS_ERR_TRY_AGAIN) {
+ LOG_ERROR("abort_startup: ckpt unlink retry");
+ usleep(1000);
+ goto unlink_retry;
+ }
+
+ if (rv != SA_AIS_OK) {
+ LOG_ERROR("[%s] Failed to unlink checkpoint: %s",
+ SHORT_UUID(entry->name.value), str_ais_error(rv));
+ return -EIO;
+ }
+
+ saCkptCheckpointClose(h);
+
+ return 1;
+}
+
int create_cluster_cpg(char *str)
{
int r;
@@ -1275,6 +1324,16 @@ int create_cluster_cpg(char *str)
strncpy(new->name.value, str, size);
new->name.length = size;
+ /*
+ * Look for checkpoints before joining to see if
+ * someone wrote a checkpoint after I left a previous
+ * session.
+ */
+ if (remove_checkpoint(new) == 1)
+ LOG_COND(log_checkpoint,
+ "[%s] Removing checkpoints left from previous session",
+ SHORT_UUID(new->name.value));
+
r = cpg_initialize(&new->handle, &cpg_callbacks);
if (r != SA_AIS_OK) {
LOG_ERROR("cpg_initialize failed: Cannot join cluster");
@@ -1303,10 +1362,6 @@ int create_cluster_cpg(char *str)
static void abort_startup(struct clog_cpg *del)
{
- int len;
- SaNameT name;
- SaAisErrorT rv;
- SaCkptCheckpointHandleT h;
struct list_head *p, *n;
struct clog_tfr *tfr = NULL;
@@ -1322,38 +1377,7 @@ static void abort_startup(struct clog_cpg *del)
free(tfr);
}
- len = snprintf((char *)(name.value), SA_MAX_NAME_LENGTH, "bitmaps_%s_%u",
- SHORT_UUID(del->name.value), my_cluster_id);
- name.length = len;
-
-open_retry:
- rv = saCkptCheckpointOpen(ckpt_handle, &name, NULL,
- SA_CKPT_CHECKPOINT_READ, 0, &h);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- LOG_ERROR("abort_startup: ckpt open retry");
- usleep(1000);
- goto open_retry;
- }
-
- if (rv != SA_AIS_OK)
- return;
-
- LOG_DBG("[%s] Removing checkpoint", SHORT_UUID(del->name.value));
-unlink_retry:
- rv = saCkptCheckpointUnlink(ckpt_handle, &name);
- if (rv == SA_AIS_ERR_TRY_AGAIN) {
- LOG_ERROR("abort_startup: ckpt unlink retry");
- usleep(1000);
- goto unlink_retry;
- }
-
- if (rv != SA_AIS_OK) {
- LOG_ERROR("[%s] Failed to unlink checkpoint: %s",
- SHORT_UUID(del->name.value), str_ais_error(rv));
- return;
- }
-
- saCkptCheckpointClose(h);
+ remove_checkpoint(del);
}
static int _destroy_cluster_cpg(struct clog_cpg *del)
@@ -1363,6 +1387,13 @@ static int _destroy_cluster_cpg(struct clog_cpg *del)
LOG_COND(log_resend_requests, "[%s] I am leaving.2.....",
SHORT_UUID(del->name.value));
+ /*
+ * We must send any left over checkpoints before
+ * leaving. If we don't, an incoming node could
+ * be stuck with no checkpoint and stall.
+ */
+ do_checkpoints(del);
+
del->cpg_state = INVALID;
del->state = LEAVING;
@@ -1416,6 +1447,7 @@ void cleanup_cluster(void)
void cluster_debug(void)
{
+ struct checkpoint_data *cp;
struct clog_cpg *entry, *tmp;
struct list_head *p, *n;
struct clog_tfr *t;
@@ -1424,13 +1456,30 @@ void cluster_debug(void)
LOG_ERROR("");
LOG_ERROR("CLUSTER COMPONENT DEBUGGING::");
list_for_each_entry_safe(entry, tmp, &clog_cpg_list, list) {
- LOG_ERROR("Local working_list for %s:", SHORT_UUID(entry->name.value));
+ LOG_ERROR("%s::", SHORT_UUID(entry->name.value));
+ LOG_ERROR(" lowest_id : %u", entry->lowest_id);
+ LOG_ERROR(" state : %s", (entry->state == INVALID) ?
+ "INVALID" : (entry->state == VALID) ? "VALID" :
+ (entry->state == LEAVING) ? "LEAVING" : "UNKNOWN");
+ LOG_ERROR(" cpg_state : %d", entry->cpg_state);
+ LOG_ERROR(" free_me : %d", entry->free_me);
+ LOG_ERROR(" delay : %d", entry->delay);
+ LOG_ERROR(" resend_requests : %d", entry->resend_requests);
+ LOG_ERROR(" checkpoints_needed: %d", entry->checkpoints_needed);
+ for (i = 0, cp = entry->checkpoint_list;
+ i < MAX_CHECKPOINT_REQUESTERS; i++)
+ if (cp)
+ cp = cp->next;
+ else
+ break;
+ LOG_ERROR(" CKPTs waiting : %d", i);
+ LOG_ERROR(" Working list:");
list_for_each_safe(p, n, &entry->working_list) {
t = (struct clog_tfr *)p;
LOG_ERROR(" %s/%u", _RQ_TYPE(t->request_type), t->seq);
}
- LOG_ERROR("Local startup_list for %s:", SHORT_UUID(entry->name.value));
+ LOG_ERROR(" Startup list:");
list_for_each_safe(p, n, &entry->startup_list) {
t = (struct clog_tfr *)p;
LOG_ERROR(" %s/%u", _RQ_TYPE(t->request_type), t->seq);