This is the mail archive of the cluster-cvs@sourceware.org mailing list for the cluster.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

cluster: RHEL4 - rgmanager: Implement enforcement of timeouts on aper-resource basis


Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=c78d4a719b6264e4ff4165b6ef564de018c8418f
Commit:        c78d4a719b6264e4ff4165b6ef564de018c8418f
Parent:        0562c3607d70ee1e2a35c0e4b2c7c17bf26cface
Author:        Ryan O'Hara <rohara@redhat.com>
AuthorDate:    Fri Jan 23 13:30:14 2009 -0600
Committer:     Ryan O'Hara <rohara@redhat.com>
CommitterDate: Fri Jan 23 13:30:14 2009 -0600

rgmanager: Implement enforcement of timeouts on a per-resource basis

Set "__enforce_timeouts" to "1" in the resource tree in order to
enable this behavior (e.g. not the global resources list).

rhbz #333161
---
 rgmanager/include/reslist.h     |    2 +
 rgmanager/src/daemons/restree.c |   98 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/rgmanager/include/reslist.h b/rgmanager/include/reslist.h
index 23129ff..f78288f 100644
--- a/rgmanager/include/reslist.h
+++ b/rgmanager/include/reslist.h
@@ -47,6 +47,8 @@
 				  the configuration */
 #define RF_ROOT		(1<<9)
 
+#define RF_ENFORCE_TIMEOUTS (1<<10) /** Enforce timeouts for this node */
+
 
 
 #define RES_STOPPED	(0)
diff --git a/rgmanager/src/daemons/restree.c b/rgmanager/src/daemons/restree.c
index b4a75a9..828b985 100644
--- a/rgmanager/src/daemons/restree.c
+++ b/rgmanager/src/daemons/restree.c
@@ -334,6 +334,26 @@ restore_signals(void)
 }
 
 
+/** Find the index for a given operation / depth in a resource node */
+int
+res_act_index(resource_node_t *node, const char *op_str, int depth)
+{
+	int x = 0;
+	resource_act_t *act;
+
+	for (x = 0; node->rn_actions[x].ra_name; x++) {
+		act = &node->rn_actions[x];
+		if (depth != act->ra_depth)
+			continue;
+		if (strcasecmp(act->ra_name, op_str))
+			continue;
+		return x;
+	}
+
+	return -1;
+}
+
+
 /**
    Execute a resource-specific agent for a resource node in the tree.
 
@@ -348,6 +368,8 @@ res_exec(resource_node_t *node, int op, const char *arg, int depth)
 {
 	int childpid, pid;
 	int ret = 0;
+	int act_index;
+	time_t sleeptime = 0;
 	char **env = NULL;
 	resource_t *res = node->rn_resource;
 	const char *op_str = agent_op_str(op);
@@ -356,6 +378,17 @@ res_exec(resource_node_t *node, int op, const char *arg, int depth)
 	if (!res->r_rule->rr_agent)
 		return 0;
 
+	/* Get the action index for later */
+	act_index = res_act_index(node, op_str, depth);
+
+	/* This shouldn't happen, but execing an action for which 
+	   we have an incorrect depth or no status action does not
+	   indicate a problem.  This allows people writing resource
+	   agents to write agents which have no status/monitor function
+	   at their option, in violation of the OCF RA API specification. */
+	if (act_index < 0)
+		return 0;
+
 #ifdef DEBUG
 	env = build_env(node, depth, node->rn_resource->r_incarnations);
 	if (!env)
@@ -411,11 +444,53 @@ res_exec(resource_node_t *node, int op, const char *arg, int depth)
 	kill_env(env);
 #endif
 
-	do {
-		pid = waitpid(childpid, &ret, 0);
-		if ((pid < 0) && (errno == EINTR))
-			continue;
-	} while (0);
+	if (node->rn_flags & RF_ENFORCE_TIMEOUTS)
+		sleeptime = node->rn_actions[act_index].ra_timeout;
+
+	if (sleeptime > 0) {
+
+		/* There's a better way to do this, but this is easy and
+		   doesn't introduce signal woes */
+		while (sleeptime) {
+			pid = waitpid(childpid, &ret, WNOHANG);
+
+			if (pid == childpid)
+				break;
+			sleep(1);
+			--sleeptime;
+		}
+
+		if (pid != childpid && sleeptime == 0) {
+
+			clulog(LOG_ERR,
+			       "%s on %s:%s timed out after %d seconds\n",
+			       op_str, res->r_rule->rr_type,
+			       res->r_attrs->ra_value,
+			       node->rn_actions[act_index].ra_timeout,
+			       ocf_strerror(ret));
+			
+			/* This can't be guaranteed to kill even the child
+			   process if the child is in disk-wait :( */
+			kill(childpid, SIGKILL);
+			sleep(1);
+			pid = waitpid(childpid, &ret, WNOHANG);
+			if (pid == 0) {
+				clulog(LOG_ERR,
+				       "Task %s PID %d did not exit "
+				       "after SIGKILL\n",
+				       op_str, childpid);
+			}
+
+			/* Always an error if we time out */
+			return 1;
+		}
+	} else {
+		do {
+			pid = waitpid(childpid, &ret, 0);
+			if ((pid < 0) && (errno == EINTR))
+				continue;
+		} while (0);
+	}
 
 	if (WIFEXITED(ret)) {
 
@@ -573,6 +648,17 @@ do_load_resource(int ccsfd, char *base,
 		free(ref);
 	}
 
+	snprintf(tok, sizeof(tok), "%s/@__enforce_timeouts", base);
+#ifndef NO_CCS
+	if (ccs_get(ccsfd, tok, &ref) == 0) {
+#else
+	if (conf_get(tok, &ref) == 0) {
+#endif
+		if (atoi(ref) > 0 || strcasecmp(ref, "yes") == 0)
+			node->rn_flags |= RF_ENFORCE_TIMEOUTS;
+		free(ref);
+	}
+
 	curres->r_refs++;
 
 	*newnode = node;
@@ -850,6 +936,8 @@ _print_resource_tree(resource_node_t **tree, int level)
 				printf("COMMON ");
 			if (node->rn_flags & RF_INDEPENDENT)
 				printf("INDEPENDENT ");
+			if (node->rn_flags & RF_ENFORCE_TIMEOUTS)
+				printf("ENFORCE-TIMEOUTS ");
 			printf("]");
 		}
 		printf(" {\n");


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]