From 634d196c47b1df4ec117ad3f5201e7b5ac57207d Mon Sep 17 00:00:00 2001 From: Hans Feldt Date: Mon, 28 Aug 2006 12:44:15 +0000 Subject: [PATCH] 1. Improvement of the use case 'Amf node leave spontaneously' 2. Improvement of the use case 'Amf node join' 3. Improvement to manage more than one SG within an Application. 4. Improvement to manage an arbitrary number of Csi-assignments associated to the Csi git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1232 fd59a12c-fef9-0310-b244-a6a79926bd2f --- exec/amf.c | 188 +++++--- exec/amf.h | 100 ++++- exec/amfapp.c | 26 +- exec/amfcluster.c | 56 ++- exec/amfcomp.c | 397 +++++++++-------- exec/amfnode.c | 189 +++++++- exec/amfsg.c | 1071 +++++++++++++++++++++++++++++++++++---------- exec/amfsi.c | 121 +++-- exec/amfsu.c | 102 ++--- exec/amfutil.c | 208 ++++++--- 10 files changed, 1803 insertions(+), 655 deletions(-) diff --git a/exec/amf.c b/exec/amf.c index fb3ab067..547c6229 100644 --- a/exec/amf.c +++ b/exec/amf.c @@ -154,7 +154,7 @@ #define SYNCTRACE(format, args...) do { \ TRACE6(">%s: " format, __FUNCTION__, ##args); \ -} while(0) +} while (0) /* * The time AMF will wait to get synchronised by another node @@ -463,6 +463,7 @@ struct scsm_descriptor { /* node ID of current sync master */ unsigned int sync_master; + unsigned int *member_list; unsigned int *joined_list; unsigned int joined_list_entries; struct amf_cluster *cluster; @@ -491,9 +492,34 @@ struct scsm_descriptor { static struct scsm_descriptor scsm; /* IMPL */ +static char *hostname_get (unsigned int nodeid) +{ + struct totem_ip_address interfaces[INTERFACE_MAX]; + char **status; + unsigned int iface_count; + int res; + struct hostent *ent; + + res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); + if (res == -1) { + log_printf (LOG_LEVEL_ERROR, "totempg_ifaces_get failed for %u", nodeid); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + if (iface_count > 0) { + ent = gethostbyaddr (interfaces[0].addr, 4, interfaces[0].family); + if (ent == NULL) { + log_printf (LOG_LEVEL_ERROR, "gethostbyaddr failed: %d\n", h_errno); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + + return ent->h_name; + } + + return NULL; +} /** - * Get pointer to this node object. + * Return pointer to this node object. * * @param cluster * @@ -501,24 +527,16 @@ static struct scsm_descriptor scsm; */ static struct amf_node *get_this_node_obj (struct amf_cluster *cluster) { - char hostname[HOST_NAME_MAX + 1]; - struct amf_node *node; + SaClmClusterNodeT *clm_node = main_clm_get_by_nodeid (SA_CLM_LOCAL_NODE_ID); + char *hostname; + SaNameT name; - assert (cluster != NULL); + assert (clm_node != NULL); + hostname = hostname_get (clm_node->nodeId); + assert (hostname != NULL); + setSaNameT (&name, hostname); - if (gethostname (hostname, sizeof(hostname)) == -1) { - log_printf (LOG_LEVEL_ERROR, "gethostname failed: %d", errno); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - /* look for this node */ - for (node = cluster->node_head; node != NULL; node = node->next) { - if (strcmp(hostname, getSaNameT (&node->name)) == 0) { - return node; - } - } - - return NULL; + return amf_node_find (&name); } /** @@ -611,7 +629,8 @@ static void mcast_sync_ready (void) int res; SYNCTRACE ("state %s", scsm_state_names[scsm.state]); - + + req_exec.header.size = sizeof (struct req_exec_amf_sync_data); req_exec.header.id = SERVICE_ID_MAKE (AMF_SERVICE, MESSAGE_REQ_EXEC_AMF_SYNC_READY); @@ -733,40 +752,6 @@ static unsigned int calc_sync_master ( return master; } -/** - * Get hostname from TOTEM node ID - * @param nodeid - * - * @return char* - */ -static char *hostname_get (unsigned int nodeid) -{ - struct totem_ip_address interfaces[INTERFACE_MAX]; - char **status; - unsigned int iface_count; - int res; - struct hostent *ent; - - res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); - if (res == -1) { - log_printf (LOG_LEVEL_ERROR, "totempg_ifaces_get failed"); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - if (iface_count > 0) { - ent = gethostbyaddr (interfaces[0].addr, 4, interfaces[0].family); - if (ent == NULL) { - log_printf (LOG_LEVEL_ERROR, "gethostbyaddr failed: %d\n", h_errno); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - return ent->h_name; - } else { - assert (0); - } - - return NULL; -} static void free_synced_data (void) { @@ -1141,12 +1126,13 @@ static void joined_nodes_start (void) for (i = 0; i < scsm.joined_list_entries; i++) { SaNameT name; - struct amf_node *synced_node; + struct amf_node *node; setSaNameT (&name, hostname_get (scsm.joined_list[i])); - synced_node = amf_node_find (&name); - if (synced_node != NULL) { - amf_node_sync_ready (synced_node); + node = amf_node_find (&name); + if (node != NULL) { + node->nodeid = scsm.joined_list[i]; + amf_node_sync_ready (node); } else { log_printf (LOG_LEVEL_INFO, "Node %s is not configured as an AMF node", name.value); @@ -1154,6 +1140,26 @@ static void joined_nodes_start (void) } } +static void init_nodeids (void) +{ + int i; + + ENTER (""); + + for (i = 0; scsm.member_list[i] != 0; i++) { + SaNameT name; + struct amf_node *node; + + setSaNameT (&name, hostname_get (scsm.member_list[i])); + node = amf_node_find (&name); + + assert (node != NULL); + node->nodeid = scsm.member_list[i]; + } + + LEAVE (""); +} + /****************************************************************************** * AMF Framework callback implementation * *****************************************************************************/ @@ -1161,7 +1167,6 @@ static void joined_nodes_start (void) static void amf_sync_init (void) { SYNCTRACE ("state %s", scsm_state_names[scsm.state]); - switch (scsm.state) { case UNCONFIGURED: case PROBING_1: @@ -1276,10 +1281,22 @@ static void amf_sync_activate (void) switch (scsm.state) { case SYNCHRONIZING: sync_state_set (NORMAL_OPERATION); - if (amf_cluster->state == CLUSTER_UNINSTANTIATED) { - amf_cluster_start (amf_cluster); - } else { - joined_nodes_start (); + init_nodeids (); + /* TODO: Remove dependencies to amf_cluster->state */ + switch (amf_cluster->state) { + case CLUSTER_STARTED: { + joined_nodes_start (); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + amf_cluster_sync_ready (amf_cluster); + joined_nodes_start (); + break; + } + case CLUSTER_UNINSTANTIATED: + default: { + amf_cluster_sync_ready (amf_cluster); + } } break; case UPDATING_CLUSTER_MODEL: @@ -1288,14 +1305,26 @@ static void amf_sync_activate (void) scsm.cluster = NULL; this_amf_node = get_this_node_obj (amf_cluster); sync_state_set (NORMAL_OPERATION); + init_nodeids (); if (this_amf_node != NULL) { #ifdef AMF_DEBUG amf_runtime_attributes_print (amf_cluster); #endif - if (amf_cluster->state == CLUSTER_UNINSTANTIATED) { - amf_cluster_start (amf_cluster); - } else { - amf_node_sync_ready (this_amf_node); + /* TODO: Remove dependencies to amf_cluster->state */ + switch (amf_cluster->state) { + case CLUSTER_STARTED: { + amf_node_sync_ready (this_amf_node); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + amf_cluster_sync_ready (amf_cluster); + amf_node_sync_ready (this_amf_node); + break; + } + case CLUSTER_UNINSTANTIATED: + default: { + amf_cluster_sync_ready (amf_cluster); + } } } else { log_printf (LOG_LEVEL_INFO, @@ -1377,6 +1406,19 @@ static void amf_confchg_fn ( scsm.joined_list[i] = joined_list[i]; } + /** + * Save current members of the cluster, needed to initialize + * each node's totem node id later. + */ + if (scsm.member_list != NULL) { + free (scsm.member_list); + } + scsm.member_list = amf_malloc ((member_list_entries + 1) * sizeof (unsigned int)); + for (i = 0; i < member_list_entries; i++) { + scsm.member_list[i] = member_list[i]; + } + scsm.member_list[i] = 0; + switch (scsm.state) { case IDLE: { sync_state_set (PROBING_1); @@ -1443,10 +1485,17 @@ static void amf_confchg_fn ( if (scsm.sync_master == this_ip->nodeid) { SYNCTRACE ("I am (new) sync master"); } + } - /* - * TODO: activate standby, ... - */ + if (left_list_entries > 0) { + int i; + struct amf_node *node; + + for (i = 0; i < left_list_entries; i++) { + node = amf_node_find_by_nodeid (left_list[i]); + assert (node != NULL); + amf_node_leave(node); + } } break; } @@ -1456,6 +1505,7 @@ static void amf_confchg_fn ( } } + static int amf_lib_exit_fn (void *conn) { struct amf_comp *comp; diff --git a/exec/amf.h b/exec/amf.h index 656e3f7c..d4774ba2 100644 --- a/exec/amf.h +++ b/exec/amf.h @@ -39,6 +39,9 @@ #define AMF_H_DEFINED #include +#include +#include + #include "../include/saAis.h" #include "../include/saAmf.h" #include "../include/list.h" @@ -54,6 +57,33 @@ enum clc_component_types { clc_component_non_proxied_non_sa_aware = 3 /* non-proxied, non sa aware */ }; +/* + * Node Error Escallation State + */ +typedef enum { + NODE_EESM_IDLE, + NODE_EESM_ESCALLATION_LEVEL_2, + NODE_EESM_ESCALLATION_LEVEL_3 +} amf_node_eesm_state_t; + +typedef enum { + NODE_ACSM_REPAIR_NEEDED, + NODE_ACSM_ESCALLATION_LEVEL_0, + NODE_ACSM_ESCALLATION_LEVEL_2, + NODE_ACSM_ESCALLATION_LEVEL_3, + NODE_ACSM_FAILING_FAST_REBOOTING_NODE, + NODE_ACSM_FAILING_FAST_ACTIVATING_STANDBY_NODE, + NODE_ACSM_FAILING_GRACEFULLY_SWITCHING_OVER, + NODE_ACSM_FAILING_GRACEFULLY_FAILING_OVER, + NODE_ACSM_FAILING_GRACEFULLY_REBOOTING_NODE, + NODE_ACSM_LEAVING_SPONTANEOUSLY_FAILING_OVER, + NODE_ACSM_LEAVING_SPONTANEOUSLY_WAITING_FOR_NODE_TO_JOIN, + NODE_ACSM_JOINING_STARTING_SERVICE_UNITS, + NODE_ACSM_JOINING_ASSIGNING_ACTIVE_WORKLOAD, + NODE_ACSM_JOINING_ASSIGNING_STANDBY_WORKLOAD + +} amf_node_acsm_state_t; + typedef enum { SG_AC_Idle = 0, SG_AC_DeactivatingDependantWorkload, @@ -71,6 +101,11 @@ typedef enum { SG_AC_WaitingAfterOperationFailed } sg_avail_control_state_t; +typedef enum { + SG_RT_FailoverSU = 1, + SG_RT_FailoverNode +} sg_recovery_type_t; + typedef enum { SU_RC_ESCALATION_LEVEL_0 = 0, SU_RC_ESCALATION_LEVEL_1, @@ -114,7 +149,8 @@ struct amf_healthcheck; enum cluster_states { CLUSTER_UNINSTANTIATED, - CLUSTER_STARTING, + CLUSTER_STARTING_COMPONENTS, + CLUSTER_STARTING_WORKLOAD, CLUSTER_STARTED }; @@ -154,7 +190,10 @@ struct amf_node { struct amf_cluster *cluster; /* Implementation */ + unsigned int nodeid; struct amf_node *next; + amf_node_acsm_state_t acsm_state; + int synchronized; }; struct amf_application { @@ -174,6 +213,14 @@ struct amf_application { /* Implementation */ SaStringT clccli_path; struct amf_application *next; + struct amf_node *node_to_start; +}; + +struct sg_recovery_scope { + sg_recovery_type_t recovery_type; + struct amf_si **sis; + struct amf_su **sus; + struct amf_comp *comp; }; struct amf_sg { @@ -208,8 +255,9 @@ struct amf_sg { /* Implementation */ SaStringT clccli_path; - struct amf_sg *next; - sg_avail_control_state_t avail_state; + struct amf_sg *next; + sg_avail_control_state_t avail_state; + struct sg_recovery_scope recovery_scope; }; struct amf_su { @@ -467,6 +515,7 @@ struct req_exec_amf_healthcheck_tmo { SaAmfHealthcheckKeyT safHealthcheckKey; }; + struct req_exec_amf_cluster_start_tmo { mar_req_header_t header; }; @@ -486,7 +535,7 @@ extern const char *amf_presence_state (int state); extern const char *amf_ha_state (int state); extern const char *amf_readiness_state (int state); extern const char *amf_assignment_state (int state); - +extern struct amf_node *amf_node_find_by_nodeid (unsigned int nodeid); extern char *amf_serialize_SaNameT ( char *buf, int *size, int *offset, SaNameT *name); extern char *amf_serialize_SaStringT ( @@ -551,7 +600,7 @@ extern void *amf_cluster_serialize (struct amf_cluster *cluster, int *len); extern struct amf_cluster *amf_cluster_deserialize (char *buf, int size); /* Event methods */ -extern void amf_cluster_start (struct amf_cluster *cluster); +extern void amf_cluster_sync_ready (struct amf_cluster *cluster); extern void amf_cluster_assign_workload (struct amf_cluster *cluster); /* Response event methods */ @@ -637,9 +686,6 @@ extern void amf_sg_su_assignment_removed ( extern void amf_sg_si_activated ( struct amf_sg *sg, struct amf_si *si); -/* Timer event methods */ -//static void timer_function_auto_adjust_tmo (void *sg); - /*===========================================================================*/ /* amfsu.c */ @@ -664,7 +710,8 @@ extern int amf_su_get_saAmfSUNumCurrActiveSIs (struct amf_su *su); extern int amf_su_get_saAmfSUNumCurrStandbySIs (struct amf_su *su); extern SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState ( struct amf_su *su); - +extern int amf_su_presence_state_all_comps_in_su_are_set (struct amf_su *su, + SaAmfPresenceStateT state); /* Event methods */ extern void amf_su_instantiate (struct amf_su *su); extern void amf_su_assign_si ( @@ -684,19 +731,11 @@ extern void amf_su_remove_assignment (struct amf_su *su); /* Response event methods */ extern void amf_su_comp_state_changed ( struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state); -#if 0 -extern void amf_su_comp_hastate_changed ( - struct amf_su *su, struct amf_comp *comp, - struct amf_csi_assignment *csi_assignment); -#endif extern void amf_su_comp_error_suspected ( struct amf_su *su, struct amf_comp *comp, SaAmfRecommendedRecoveryT recommended_recovery); -/* Timer event methods */ -//static void timer_function_su_probation_period_expired(void *data); - /*===========================================================================*/ /* amfcomp.c */ @@ -722,6 +761,7 @@ extern SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState ( /* Event methods */ extern void amf_comp_instantiate (struct amf_comp *comp); extern void amf_comp_terminate (struct amf_comp *comp); +extern void amf_comp_node_left (struct amf_comp *comp); /** * Request the component to assume a HA state @@ -817,6 +857,15 @@ extern struct amf_si_assignment *amf_si_assignment_deserialize ( * @return int */ extern int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si); +/** + * Get number of active assignments for the specified SI and SU + * @param si + * @param su + * + * @return int + */ +extern int amf_si_su_get_saAmfSINumCurrActiveAssignments (struct amf_si *si, + struct amf_su *su); /** * Get number of standby assignments for the specified SI @@ -824,14 +873,25 @@ extern int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si); * * @return int */ + extern int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si); +/** + * Get number of standby assignments for the specified SI and SU + * @param si + * + * @return int + */ +extern int amf_si_su_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si, + struct amf_su *su); + /** * Get assignment state for the specified SI. * @param si * * @return SaAmfAssignmentStateT */ + extern SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState ( struct amf_si *si); @@ -915,12 +975,16 @@ extern char *amf_csi_assignment_dn_make ( struct amf_csi_assignment *csi_assignment, SaNameT *name); extern struct amf_csi_assignment *amf_csi_assignment_find ( struct amf_cluster *cluster, SaNameT *name); - extern struct amf_csi_attribute *amf_csi_attribute_new (struct amf_csi *csi); extern void *amf_csi_attribute_serialize ( struct amf_csi_attribute *csi_attribute, int *len); extern struct amf_csi_attribute *amf_csi_attribute_deserialize ( struct amf_csi *csi, char *buf, int size); +/* extern int sa_amf_grep(const char *string, char *pattern, size_t nmatch, */ +/* char** sub_match_array); */ + +extern int sa_amf_grep(const char *string, char *pattern, size_t nmatch, + SaNameT *sub_match_array); /*===========================================================================*/ extern struct amf_node *this_amf_node; diff --git a/exec/amfapp.c b/exec/amfapp.c index 7b6736ec..3661570d 100644 --- a/exec/amfapp.c +++ b/exec/amfapp.c @@ -75,7 +75,7 @@ #include "amf.h" #include "print.h" -static int all_sg_started(struct amf_application *app) +static int all_sg_started (struct amf_application *app) { struct amf_sg *sg; struct amf_su *su; @@ -102,6 +102,8 @@ void amf_application_start ( ENTER ("'%s'", app->name.value); + app->node_to_start = node; + /* TODO: Calculate and set SI dependency levels */ for (sg = app->sg_head; sg != NULL; sg = sg->next) { @@ -119,6 +121,9 @@ void amf_application_assign_workload ( * Each dependency level should be looped and amf_sg_assign_si * called several times. */ + + app->node_to_start = node; + for (sg = app->sg_head; sg != NULL; sg = sg->next) { amf_sg_assign_si (sg, 0); } @@ -133,9 +138,13 @@ void amf_application_sg_started ( struct amf_application *app, struct amf_sg *sg, struct amf_node *node) { ENTER ("'%s'", app->name.value); - + if (all_sg_started (app)) { - amf_cluster_application_started (app->cluster, app); + if (app->node_to_start == NULL) { + amf_cluster_application_started (app->cluster, app); + } else { + amf_node_application_started (app->node_to_start, app); + } } } @@ -143,7 +152,11 @@ void amf_application_sg_assigned ( struct amf_application *app, struct amf_sg *sg) { ENTER ("'%s'", app->name.value); - amf_cluster_application_workload_assigned (app->cluster, app); + if (app->node_to_start == NULL) { + amf_cluster_application_workload_assigned (app->cluster, app); + } else { + amf_node_application_workload_assigned (app->node_to_start, app); + } } struct amf_application *amf_application_new (struct amf_cluster *cluster) @@ -219,14 +232,15 @@ struct amf_application *amf_application_find ( { struct amf_application *app; - ENTER ("%s", name); - for (app = cluster->application_head; app != NULL; app = app->next) { if (strncmp (name, (char*)app->name.value, app->name.length) == 0) { break; } } + if (app == NULL) { + dprintf ("App %s not found!", name); + } return app; } diff --git a/exec/amfcluster.c b/exec/amfcluster.c index 5e1c6133..9cce4d72 100644 --- a/exec/amfcluster.c +++ b/exec/amfcluster.c @@ -108,7 +108,7 @@ static void timer_function_cluster_assign_workload_tmo (void *_cluster) { struct req_exec_amf_cluster_start_tmo req; struct iovec iovec; - + ENTER (""); req.header.size = sizeof (struct req_exec_amf_cluster_start_tmo); @@ -123,23 +123,50 @@ static void timer_function_cluster_assign_workload_tmo (void *_cluster) } -void amf_cluster_start (struct amf_cluster *cluster) +void amf_cluster_sync_ready (struct amf_cluster *cluster) { struct amf_application *app; log_printf(LOG_NOTICE, "Cluster: starting applications."); - amf_cluster->state = CLUSTER_STARTING; + switch (amf_cluster->state) { + case CLUSTER_UNINSTANTIATED: { + amf_cluster->state = CLUSTER_STARTING_COMPONENTS; + for (app = cluster->application_head; app != NULL; app = app->next) { + amf_application_start (app, NULL); + } + poll_timer_add (aisexec_poll_handle, + cluster->saAmfClusterStartupTimeout, + cluster, + timer_function_cluster_assign_workload_tmo, + &cluster->timeout_handle); - for (app = cluster->application_head; app != NULL; app = app->next) { - amf_application_start (app, NULL); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + if (cluster->timeout_handle) { + + poll_timer_delete ( + aisexec_poll_handle, cluster->timeout_handle); + + cluster->timeout_handle = 0; + } + break; + } + case CLUSTER_STARTING_WORKLOAD: { + log_printf (LOG_LEVEL_ERROR, "Sync ready not implemented in " + "cluster state: %u\n", amf_cluster->state); + assert (0); + break; + } + case CLUSTER_STARTED: { + assert (0); + break; + } + default: + assert (0); } - /* wait a while before assigning workload */ - poll_timer_add (aisexec_poll_handle, cluster->saAmfClusterStartupTimeout, - cluster, - timer_function_cluster_assign_workload_tmo, - &cluster->timeout_handle); } void amf_cluster_init (void) @@ -160,7 +187,7 @@ void amf_cluster_application_started ( poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle); cluster->timeout_handle = 0; } - + cluster->state = CLUSTER_STARTING_WORKLOAD; amf_cluster_assign_workload (cluster); } } @@ -182,7 +209,8 @@ struct amf_cluster *amf_cluster_new (void) void amf_cluster_application_workload_assigned ( struct amf_cluster *cluster, struct amf_application *app) { - log_printf(LOG_NOTICE, "Cluster: all workload assigned."); + log_printf (LOG_NOTICE, "Cluster: application %s assigned.", + app->name.value); amf_cluster->state = CLUSTER_STARTED; } @@ -226,13 +254,15 @@ void amf_cluster_assign_workload (struct amf_cluster *cluster) ENTER (""); + cluster->state = CLUSTER_STARTING_WORKLOAD; + if (cluster->timeout_handle) { poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle); cluster->timeout_handle = 0; } for (app = cluster->application_head; app != NULL; app = app->next) { - amf_application_assign_workload (app, this_amf_node); + amf_application_assign_workload (app, NULL); } } diff --git a/exec/amfcomp.c b/exec/amfcomp.c index c39254de..0c921a43 100644 --- a/exec/amfcomp.c +++ b/exec/amfcomp.c @@ -280,28 +280,10 @@ static int invocation_get_and_destroy ( *interface = invocation_entries[invocation].interface; *data = invocation_entries[invocation].data; memset (&invocation_entries[invocation], 0, sizeof (struct invocation)); - + return (0); } -#if 0 -static int invocation_get ( - SaUint64T invocation, unsigned int *interface, void **data) -{ - if (invocation > invocation_entries_size) { - return (-1); - } - if (invocation_entries[invocation].active == 0) { - return (-1); - } - - *interface = invocation_entries[invocation].interface; - *data = invocation_entries[invocation].data; - - return (0); -} -#endif - static void invocation_destroy_by_data (void *data) { int i; @@ -332,7 +314,7 @@ static void report_error_suspected ( char *amf_comp_dn_make (struct amf_comp *comp, SaNameT *name) { - int i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, + int i = snprintf ((char*) name->value, SA_MAX_NAME_LENGTH, "safComp=%s,safSu=%s,safSg=%s,safApp=%s", comp->name.value, comp->su->name.value, comp->su->sg->name.value, comp->su->sg->application->name.value); @@ -375,16 +357,16 @@ static void *clc_command_run (void *context) waitpid (pid, &status, 0); if (WIFEXITED (status) != 0 && WEXITSTATUS(status) != 0) { fprintf (stderr, "Error: CLC_CLI (%d) failed with exit status:" - " %d - %s\n", pid, WEXITSTATUS(status), - strerror (WEXITSTATUS(status))); + " %d - %s\n", pid, WEXITSTATUS(status), + strerror (WEXITSTATUS(status))); /* - * TODO: remove this and handle properly later... - */ + * TODO: remove this and handle properly later... + */ openais_exit_error (AIS_DONE_FATAL_ERR); } if (WIFSIGNALED (status) != 0) { fprintf (stderr, "Error: CLC_CLI (%d) failed with exit status:" - " %d\n", pid, WTERMSIG(status)); + " %d\n", pid, WTERMSIG(status)); /* * TODO: remove this and handle properly later... */ @@ -419,22 +401,18 @@ static void *clc_command_run (void *context) /* If command is not an absolute path, search for paths in parent objects */ if (cmd[0] != '/') { - if (clc_command_run_data->comp->clccli_path != NULL && - strlen (clc_command_run_data->comp->clccli_path)) { + if (clc_command_run_data->comp->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->clccli_path)) { + clc_command_run_data->comp->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->sg->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->sg->clccli_path)) { + clc_command_run_data->comp->su->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->sg->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->sg->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->sg->application->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->sg->application->clccli_path)) { + clc_command_run_data->comp->su->sg->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->sg->application->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->sg->application->clccli_path, cmd); + clc_command_run_data->comp->su->sg->application->clccli_path, cmd); } cmd = path; } @@ -445,13 +423,13 @@ static void *clc_command_run (void *context) { /* make a proper argv array */ i = 1; - char *ptrptr; + char *ptrptr; char *arg = strtok_r(comp_argv, " ", &ptrptr); while (arg) { argv_size++; argv = realloc (argv, sizeof (char*) * argv_size); if (argv == NULL) { - fprintf (stderr, "out-of-memory"); + fprintf (stderr, "out-of-memory"); exit (-1); } argv[i] = arg; @@ -461,20 +439,20 @@ static void *clc_command_run (void *context) } argv[i] = NULL; - i = snprintf(comp_name, SA_MAX_NAME_LENGTH, - "SA_AMF_COMPONENT_NAME=safComp=%s,safSu=%s,safSg=%s,safApp=%s", - clc_command_run_data->comp->name.value, - clc_command_run_data->comp->su->name.value, - clc_command_run_data->comp->su->sg->name.value, - clc_command_run_data->comp->su->sg->application->name.value); + i = snprintf (comp_name, SA_MAX_NAME_LENGTH, + "SA_AMF_COMPONENT_NAME=safComp=%s,safSu=%s,safSg=%s,safApp=%s", + clc_command_run_data->comp->name.value, + clc_command_run_data->comp->su->name.value, + clc_command_run_data->comp->su->sg->name.value, + clc_command_run_data->comp->su->sg->application->name.value); assert (i <= sizeof (comp_name)); - /* two is for component name and NULL termination */ + /* two is for component name and NULL termination */ envp_size = 2; envp = amf_malloc (sizeof (char*) * envp_size); envp[0] = comp_name; for (i = 1; clc_command_run_data->comp->saAmfCompCmdEnv && - clc_command_run_data->comp->saAmfCompCmdEnv[i - 1]; i++) { + clc_command_run_data->comp->saAmfCompCmdEnv[i - 1]; i++) { envp_size++; envp = realloc (envp, sizeof (char*) * envp_size); if (envp == NULL) { @@ -497,9 +475,9 @@ static void *clc_command_run (void *context) res = execve (cmd, argv, envp); if (res == -1) { fprintf (stderr, "Couldn't exec program %s (%s)\n", - cmd, strerror (errno)); + cmd, strerror (errno)); } - exit (res); /* abnormal exit of forked process */ + exit (res); /* abnormal exit of forked process */ return (0); } @@ -528,6 +506,8 @@ static int clc_cli_instantiate (struct amf_comp *comp) if (res != 0) { log_printf (LOG_LEVEL_ERROR, "pthread_create failed: %d", res); } +// clc_command_run_data->completion_callback (clc_command_run_data); + // TODO error code from pthread_create return (res); } @@ -580,7 +560,7 @@ static int lib_comp_terminate_request (struct amf_comp *comp) invocation_create ( AMF_RESPONSE_COMPONENTTERMINATECALLBACK, component_terminate_callback_data); - + openais_conn_send_response ( openais_conn_partner_get (comp->conn), &res_lib, @@ -605,11 +585,8 @@ static void mcast_cleanup_completion_event (void *context) struct req_exec_amf_clc_cleanup_completed req; struct iovec iovec; - TRACE2("CLC cleanup done for '%s'", - clc_command_run_data->comp->name.value); - req.header.size = sizeof (struct req_exec_amf_clc_cleanup_completed); - req.header.id = SERVICE_ID_MAKE (AMF_SERVICE, + req.header.id = SERVICE_ID_MAKE (AMF_SERVICE, MESSAGE_REQ_EXEC_AMF_CLC_CLEANUP_COMPLETED); amf_comp_dn_make (clc_command_run_data->comp, &req.compName); @@ -619,7 +596,7 @@ static void mcast_cleanup_completion_event (void *context) assert (totempg_groups_mcast_joined (openais_group_handle, &iovec, 1, TOTEMPG_AGREED) == 0); } - + /* * Cleanup possible operations */ @@ -684,7 +661,7 @@ struct amf_healthcheck *amf_comp_find_healthcheck ( healthcheck = healthcheck->next) { if (memcmp (key, &healthcheck->safHealthcheckKey, - sizeof (SaAmfHealthcheckKeyT)) == 0) { + sizeof (SaAmfHealthcheckKeyT)) == 0) { ret_healthcheck = healthcheck; break; } @@ -783,6 +760,8 @@ struct amf_comp *amf_comp_find (struct amf_cluster *cluster, SaNameT *name) char *ptrptr; char *buf; + assert (cluster != NULL && name != NULL); + /* malloc new buffer since strtok_r writes to its first argument */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value,name ->length + 1); @@ -815,7 +794,7 @@ struct amf_comp *amf_comp_find (struct amf_cluster *cluster, SaNameT *name) for (su = sg->su_head; su != NULL; su = su->next) { if (strncmp (su_name, (char*)su->name.value, su->name.length) == 0) { for (comp = su->comp_head; comp != NULL; comp = comp->next) { - if (strncmp (comp_name, (char*)comp->name.value, + if (strncmp (comp_name, (char*)comp->name.value, comp->name.length) == 0) { goto end; } @@ -838,8 +817,8 @@ void amf_comp_healthcheck_deactivate (struct amf_comp *comp) ENTER ("'%s'\n", getSaNameT (&comp->name)); for (healthcheck = comp->healthcheck_head; - healthcheck != NULL; - healthcheck = healthcheck->next) { + healthcheck != NULL; + healthcheck = healthcheck->next) { if (healthcheck->active) { healthcheck_deactivate (healthcheck); @@ -847,7 +826,7 @@ void amf_comp_healthcheck_deactivate (struct amf_comp *comp) } } -static void comp_ha_state_set ( struct amf_comp *comp, +static void comp_ha_state_set ( struct amf_comp *comp, struct amf_csi_assignment *csi_assignment, SaAmfHAStateT ha_state) { @@ -901,7 +880,7 @@ static void lib_csi_remove_request (struct amf_comp *comp, &csi->name, sizeof (SaNameT)); res_lib_amf_csiremovecallback.csiFlags = 0; - + openais_conn_send_response ( openais_conn_partner_get (comp->conn), &res_lib_amf_csiremovecallback, @@ -910,7 +889,8 @@ static void lib_csi_remove_request (struct amf_comp *comp, #endif struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( - struct amf_comp *component, const struct amf_csi_assignment *csi_assignment) + struct amf_comp *component, + const struct amf_csi_assignment *csi_assignment) { struct amf_si *si; struct amf_csi *csi; @@ -939,7 +919,7 @@ struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( si = csi->si; tmp_csi_assignment = csi->assigned_csis; } - } else { + } else { csi = tmp_csi_assignment->csi; si = csi->si; } @@ -971,8 +951,8 @@ struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( void amf_comp_foreach_csi_assignment ( struct amf_comp *component, - void (*foreach_fn)(struct amf_comp *component, - struct amf_csi_assignment *csi_assignment)) + void (*foreach_fn) (struct amf_comp *component, + struct amf_csi_assignment *csi_assignment)) { struct amf_csi_assignment *csi_assignment; @@ -986,7 +966,7 @@ void amf_comp_foreach_csi_assignment ( } static struct amf_csi_assignment *csi_assignment_find_in ( - struct amf_comp *component, SaNameT *csi_name) + struct amf_comp *component, SaNameT *csi_name) { struct amf_csi_assignment *csi_assignment; SaNameT dn; @@ -1008,12 +988,12 @@ static void healthcheck_deactivate ( struct amf_healthcheck *healthcheck_active) { dprintf ("deactivating healthcheck for component %s\n", - getSaNameT (&healthcheck_active->comp->name)); + getSaNameT (&healthcheck_active->comp->name)); - poll_timer_delete (aisexec_poll_handle, - healthcheck_active->timer_handle_period); poll_timer_delete (aisexec_poll_handle, - healthcheck_active->timer_handle_duration); + healthcheck_active->timer_handle_period); + poll_timer_delete (aisexec_poll_handle, + healthcheck_active->timer_handle_duration); invocation_destroy_by_data ((void *)healthcheck_active); healthcheck_active->active = 0; @@ -1054,7 +1034,7 @@ static void mcast_healthcheck_tmo_event ( amf_comp_dn_make (healthcheck->comp, &req_exec.compName); memcpy (&req_exec.safHealthcheckKey, - &healthcheck->safHealthcheckKey, sizeof (SaAmfHealthcheckKeyT)); + &healthcheck->safHealthcheckKey, sizeof (SaAmfHealthcheckKeyT)); iovec.iov_base = (char *)&req_exec; iovec.iov_len = sizeof (req_exec); @@ -1083,7 +1063,7 @@ static void lib_healthcheck_request (struct amf_healthcheck *healthcheck) { struct res_lib_amf_healthcheckcallback res_lib; - res_lib.header.id = MESSAGE_RES_AMF_HEALTHCHECKCALLBACK; + res_lib.header.id = MESSAGE_RES_AMF_HEALTHCHECKCALLBACK; res_lib.header.size = sizeof (struct res_lib_amf_healthcheckcallback); res_lib.header.error = SA_AIS_OK; res_lib.invocation = @@ -1094,7 +1074,7 @@ static void lib_healthcheck_request (struct amf_healthcheck *healthcheck) sizeof (SaAmfHealthcheckKeyT)); TRACE7 ("sending healthcheck request to component %s", - res_lib.compName.value); + res_lib.compName.value); openais_conn_send_response ( openais_conn_partner_get (healthcheck->comp->conn), &res_lib, sizeof (struct res_lib_amf_healthcheckcallback)); @@ -1120,8 +1100,9 @@ static void lib_csi_set_request ( csi = csi_assignment->csi; ENTER ("Assigning CSI '%s' state %s to comp '%s'\n", - getSaNameT (&csi->name), amf_ha_state (csi_assignment->requested_ha_state), - comp->name.value); + getSaNameT (&csi->name), + amf_ha_state (csi_assignment->requested_ha_state), + comp->name.value); for (attribute = csi->attributes_head; attribute != NULL; @@ -1134,7 +1115,7 @@ static void lib_csi_set_request ( } } p = amf_malloc(sizeof(struct res_lib_amf_csisetcallback) + - char_length_of_csi_attrs); + char_length_of_csi_attrs); res_lib = (struct res_lib_amf_csisetcallback*)p; /* Address of the buffer containing the Csi name value pair */ @@ -1162,7 +1143,7 @@ static void lib_csi_set_request ( case SA_AMF_HA_ACTIVE: { res_lib->csiStateDescriptor.activeDescriptor.activeCompName.length = 0; res_lib->csiStateDescriptor.activeDescriptor.transitionDescriptor = - SA_AMF_CSI_NEW_ASSIGN; + SA_AMF_CSI_NEW_ASSIGN; break; } case SA_AMF_HA_STANDBY: { @@ -1197,35 +1178,32 @@ static void lib_csi_set_request ( res_lib->invocation = invocation_create (AMF_RESPONSE_CSISETCALLBACK, csi_assignment); openais_conn_send_response ( - openais_conn_partner_get (comp->conn), res_lib, res_lib->header.size); - + openais_conn_partner_get (comp->conn), res_lib, res_lib->header.size); + free(p); } SaAisErrorT amf_comp_register (struct amf_comp *comp) { TRACE2("Exec comp register '%s'", comp->name.value); - + if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING) { comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED); } else if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_INSTANTIATING) { amf_comp_operational_state_set (comp, SA_AMF_OPERATIONAL_ENABLED); comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED); - } - else { + } else { assert (0); } - + return SA_AIS_OK; } -void amf_comp_error_report ( - struct amf_comp *comp, SaAmfRecommendedRecoveryT recommendedRecovery) +void amf_comp_error_report (struct amf_comp *comp, SaAmfRecommendedRecoveryT recommendedRecovery) { struct res_lib_amf_componenterrorreport res_lib; - TRACE2("Exec comp error report '%s'", comp->name.value); - + if (amf_su_is_local (comp->su)) { res_lib.header.size = sizeof (struct res_lib_amf_componenterrorreport); res_lib.header.id = MESSAGE_RES_AMF_COMPONENTERRORREPORT; @@ -1266,7 +1244,7 @@ void amf_comp_cleanup_completed (struct amf_comp *comp) { TRACE2("Exec CLC cleanup completed for '%s'", comp->name.value); - /* Set all CSI's confirmed HA state to unknown */ + /* Set all CSI's confirmed HA state to unknown */ amf_comp_foreach_csi_assignment (comp, clear_ha_state); /* clear error suspected flag, component is terminated now */ @@ -1301,13 +1279,13 @@ SaAisErrorT amf_comp_healthcheck_start ( healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == 0) { log_printf (LOG_ERR, "Healthcheckstart: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; goto error_exit; } dprintf ("Healthcheckstart: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + comp->name.value, healthcheckKey->key); /* * Determine if this healthcheck is already active @@ -1362,20 +1340,23 @@ SaAisErrorT amf_comp_healthcheck_stop ( struct amf_healthcheck *healthcheck; SaAisErrorT error = SA_AIS_OK; - dprintf ("Healthcheckstop: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + dprintf ("Healthcheckstop: '%s'", comp->name.value); + + if (!amf_su_is_local (comp->su)) { + return SA_AIS_OK; + } if (healthcheckKey == NULL) { for (healthcheck = comp->healthcheck_head; - healthcheck != NULL; - healthcheck = healthcheck->next) { + healthcheck != NULL; + healthcheck = healthcheck->next) { healthcheck_deactivate (healthcheck); } } else { healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == NULL) { log_printf (LOG_ERR, "Healthcheckstop: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; } else { healthcheck_deactivate (healthcheck); @@ -1408,7 +1389,6 @@ void amf_comp_instantiate (struct amf_comp *comp) void amf_comp_readiness_state_set (struct amf_comp *comp, SaAmfReadinessStateT state) { -// comp->saAmfCompReadinessState = state; TRACE1 ("Setting comp '%s' readiness state: %s\n", comp->name.value, amf_readiness_state (state)); } @@ -1440,31 +1420,31 @@ int amf_comp_response_1 ( switch (*interface) { case AMF_RESPONSE_HEALTHCHECKCALLBACK: { - struct amf_healthcheck *healthcheck = data; - SaNameT name; - TRACE7 ("Healthcheck response from '%s': %d", + struct amf_healthcheck *healthcheck = data; + SaNameT name; + TRACE7 ("Healthcheck response from '%s': %d", amf_comp_dn_make (healthcheck->comp, &name), error); - if (healthcheck->invocationType == SA_AMF_HEALTHCHECK_AMF_INVOKED) { + if (healthcheck->invocationType == SA_AMF_HEALTHCHECK_AMF_INVOKED) { /* the response was on time, delete supervision timer */ - poll_timer_delete (aisexec_poll_handle, - healthcheck->timer_handle_duration); - healthcheck->timer_handle_duration = 0; + poll_timer_delete (aisexec_poll_handle, + healthcheck->timer_handle_duration); + healthcheck->timer_handle_duration = 0; /* start timer to execute next healthcheck request */ - poll_timer_add (aisexec_poll_handle, - healthcheck->saAmfHealthcheckPeriod, - (void *)healthcheck, - timer_function_healthcheck_next_fn, - &healthcheck->timer_handle_period); - *retval = SA_AIS_OK; - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; - } + poll_timer_add (aisexec_poll_handle, + healthcheck->saAmfHealthcheckPeriod, + (void *)healthcheck, + timer_function_healthcheck_next_fn, + &healthcheck->timer_handle_period); + *retval = SA_AIS_OK; + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } - return 0; /* do not multicast event */ - break; - } + return 0; /* do not multicast event */ + break; + } case AMF_RESPONSE_CSISETCALLBACK: /* fall-through */ case AMF_RESPONSE_CSIREMOVECALLBACK: amf_csi_assignment_dn_make (data, dn); @@ -1472,15 +1452,15 @@ int amf_comp_response_1 ( break; #if 0 case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { - struct component_terminate_callback_data *component_terminate_callback_data; - component_terminate_callback_data = data; + struct component_terminate_callback_data *component_terminate_callback_data; + component_terminate_callback_data = data; - dprintf ("Lib component terminate callback response, error: %d", error); - amf_comp_healthcheck_deactivate (component_terminate_callback_data->comp); - escalation_policy_restart (component_terminate_callback_data->comp); - return 1; - break; - } + dprintf ("Lib component terminate callback response, error: %d", error); + amf_comp_healthcheck_deactivate (component_terminate_callback_data->comp); + escalation_policy_restart (component_terminate_callback_data->comp); + return 1; + break; + } #endif default: assert (0); @@ -1513,51 +1493,51 @@ struct amf_comp *amf_comp_response_2 ( switch (interface) { case AMF_RESPONSE_CSISETCALLBACK: { - csi_assignment = amf_csi_assignment_find (amf_cluster, dn); - assert (csi_assignment != NULL); - comp = csi_assignment->comp; - dprintf ("CSI '%s' set callback response from '%s', error: %d", - csi_assignment->csi->name.value, - csi_assignment->comp->name.value, error); - comp = csi_assignment->comp; - if (error == SA_AIS_OK) { - comp_ha_state_set ( - comp, csi_assignment, csi_assignment->requested_ha_state); - } else if (error == SA_AIS_ERR_FAILED_OPERATION) { - amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, - csi_assignment); - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; + csi_assignment = amf_csi_assignment_find (amf_cluster, dn); + assert (csi_assignment != NULL); + comp = csi_assignment->comp; + dprintf ("CSI '%s' set callback response from '%s', error: %d", + csi_assignment->csi->name.value, + csi_assignment->comp->name.value, error); + comp = csi_assignment->comp; + if (error == SA_AIS_OK) { + comp_ha_state_set ( + comp, csi_assignment, csi_assignment->requested_ha_state); + } else if (error == SA_AIS_ERR_FAILED_OPERATION) { + amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, + csi_assignment); + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } + break; } - break; - } case AMF_RESPONSE_CSIREMOVECALLBACK: { - csi_assignment = amf_csi_assignment_find (amf_cluster, dn); - assert (csi_assignment != NULL); - dprintf ("Lib csi '%s' remove callback response from '%s', error: %d", - csi_assignment->csi->name.value, - csi_assignment->comp->name.value, error); - comp = csi_assignment->comp; - if (error == SA_AIS_OK) { - comp_ha_state_set (comp, csi_assignment, - csi_assignment->requested_ha_state); - } else if (error == SA_AIS_ERR_FAILED_OPERATION) { - amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, - csi_assignment); - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; + csi_assignment = amf_csi_assignment_find (amf_cluster, dn); + assert (csi_assignment != NULL); + dprintf ("Lib csi '%s' remove callback response from '%s', error: %d", + csi_assignment->csi->name.value, + csi_assignment->comp->name.value, error); + comp = csi_assignment->comp; + if (error == SA_AIS_OK) { + comp_ha_state_set (comp, csi_assignment, + csi_assignment->requested_ha_state); + } else if (error == SA_AIS_ERR_FAILED_OPERATION) { + amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, + csi_assignment); + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } + break; } - break; - } #if 0 - case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { - struct component_terminate_callback_data *callback_data = data; - dprintf ("Lib comp '%s' terminate callback response, error: %d", - callback_data->comp->name.value, error); - comp_presence_state_set (callback_data->comp, - SA_AMF_PRESENCE_UNINSTANTIATED); - break; - } + case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { + struct component_terminate_callback_data *callback_data = data; + dprintf ("Lib comp '%s' terminate callback response, error: %d", + callback_data->comp->name.value, error); + comp_presence_state_set (callback_data->comp, + SA_AMF_PRESENCE_UNINSTANTIATED); + break; + } #endif default: assert (0); @@ -1577,9 +1557,10 @@ void amf_comp_hastate_set ( struct amf_comp *component, struct amf_csi_assignment *csi_assignment) { + ENTER ("'%s'", csi_assignment->csi->name.value); + assert (component != NULL && csi_assignment != NULL); - ENTER ("'%s'", csi_assignment->csi->name.value); if (!component->error_suspected) { lib_csi_set_request(component, csi_assignment); @@ -1590,7 +1571,7 @@ void amf_comp_hastate_set ( assert (0); } } - + LEAVE(""); } @@ -1601,10 +1582,10 @@ void amf_comp_hastate_set ( void amf_comp_terminate (struct amf_comp *comp) { dprintf ("comp terminate '%s'\n", getSaNameT (&comp->name)); - amf_comp_healthcheck_stop (comp, NULL); comp_presence_state_set (comp, SA_AMF_PRESENCE_TERMINATING); if (amf_su_is_local (comp->su)) { + amf_comp_healthcheck_stop (comp, NULL); if (comp->error_suspected) { clc_interfaces[comp->comptype]->cleanup (comp); } else { @@ -1622,9 +1603,9 @@ void amf_comp_restart (struct amf_comp *comp) dprintf ("comp restart '%s'\n", getSaNameT (&comp->name)); comp_presence_state_set (comp, SA_AMF_PRESENCE_RESTARTING); comp->saAmfCompRestartCount += 1; - amf_comp_healthcheck_stop (comp, NULL); if (amf_su_is_local (comp->su)) { + amf_comp_healthcheck_stop (comp, NULL); clc_interfaces[comp->comptype]->cleanup (comp); } } @@ -1645,7 +1626,7 @@ SaAisErrorT amf_comp_hastate_get ( assert (comp != NULL && csi_name != NULL && ha_state != NULL); dprintf ("comp ha state get from comp '%s' CSI '%s'\n", - getSaNameT (&comp->name), csi_name->value); + getSaNameT (&comp->name), csi_name->value); assignment = csi_assignment_find_in (comp, csi_name); if (assignment != NULL) { @@ -1673,12 +1654,12 @@ SaAisErrorT amf_comp_healthcheck_confirm ( SaAisErrorT error = SA_AIS_OK; dprintf ("Healthcheckconfirm: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + comp->name.value, healthcheckKey->key); healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == NULL) { log_printf (LOG_ERR, "Healthcheckstop: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; } else if (healthcheck->active) { if (healthcheckResult == SA_AIS_OK) { @@ -1767,6 +1748,72 @@ SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState ( /* XXX we fall here in case NDEBUG is set */ return -1; } +/** + * Component is informed that the node where the 'real' + * component process is executing has unexpectadly left the + * node. If there is a pending interaction between AMF + * (component) and the 'real' component process, then component + * will indicate to its subordinate objects the the interaction + * failed. Pending presence state changes is indicated by + * reporting the new state is uninstantiated while pending csi + * operations are indicated by 'operation failed'. + * @param comp + * + * @return void + */ +void amf_comp_node_left (struct amf_comp *component) +{ + int change_pending = 0; + struct amf_csi_assignment *csi_assignment; + + ENTER(""); + if (component->saAmfCompPresenceState == SA_AMF_PRESENCE_INSTANTIATING || + component->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING || + component->saAmfCompPresenceState == SA_AMF_PRESENCE_TERMINATING) { + change_pending = 1; + } + + component->saAmfCompPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED; + + if (amf_su_presence_state_all_comps_in_su_are_set (component->su, + SA_AMF_PRESENCE_UNINSTANTIATED) != 0) { + component->su->saAmfSUPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED; + } + + if (change_pending) { + change_pending =0; + amf_su_comp_state_changed ( component->su, + component, + SA_AMF_PRESENCE_STATE, + SA_AMF_PRESENCE_UNINSTANTIATED); + } + + if (component->saAmfCompOperState == SA_AMF_OPERATIONAL_ENABLED) { + change_pending = 1; + } + + component->saAmfCompOperState = SA_AMF_OPERATIONAL_DISABLED; + if (change_pending) { + change_pending =0; + amf_su_comp_state_changed ( component->su, + component, + SA_AMF_OP_STATE, + SA_AMF_OPERATIONAL_DISABLED); + } + + csi_assignment = amf_comp_get_next_csi_assignment (component, NULL); + while (csi_assignment != NULL) { + if (csi_assignment->requested_ha_state != + csi_assignment->saAmfCSICompHAState) { + amf_si_comp_set_ha_state_failed ( + csi_assignment->csi->si,csi_assignment); + } + csi_assignment = amf_comp_get_next_csi_assignment ( + component, csi_assignment); + } + + +} /** * Serialize a component including variable length arrays and @@ -1786,13 +1833,13 @@ void *amf_comp_serialize (struct amf_comp *component, int *len) buf = amf_serialize_SaNameT (buf, &size, &offset, &component->name); - /* count cstypes and write to buf */ + /* count cstypes and write to buf */ for (i = 0; component->saAmfCompCsTypes && - component->saAmfCompCsTypes[i] != NULL; i++); + component->saAmfCompCsTypes[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); for (i = 0; component->saAmfCompCsTypes && - component->saAmfCompCsTypes[i] != NULL; i++) { + component->saAmfCompCsTypes[i] != NULL; i++) { buf = amf_serialize_SaNameT ( buf, &size, &offset, component->saAmfCompCsTypes[i]); } @@ -1806,7 +1853,7 @@ void *amf_comp_serialize (struct amf_comp *component, int *len) buf = amf_serialize_SaUint32T ( buf, &size, &offset, component->saAmfCompNumMaxStandbyCsi); - /* count environment vars and write to buf */ + /* count environment vars and write to buf */ for (i = 0; component->saAmfCompCmdEnv[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); diff --git a/exec/amfnode.c b/exec/amfnode.c index 53d47bbc..370d6a76 100644 --- a/exec/amfnode.c +++ b/exec/amfnode.c @@ -113,6 +113,107 @@ #include "amf.h" #include "util.h" #include "print.h" +#include "main.h" + + +static void amf_node_acsm_enter_leaving_spontaneously(struct amf_node *node) +{ + ENTER("'%s'", node->name.value); + node->synchronized = FALSE; + node->saAmfNodeOperState = SA_AMF_OPERATIONAL_DISABLED; +} + +static void amf_node_acsm_enter_failing_over (struct amf_node *node) +{ + struct amf_application *app; + struct amf_sg *sg; + struct amf_su *su; + struct amf_comp *component = NULL; + + ENTER("'%s'", node->name.value); + node->acsm_state = NODE_ACSM_LEAVING_SPONTANEOUSLY_FAILING_OVER; + + for (app = amf_cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + for (su = sg->su_head; su != NULL; su = su->next) { + if (name_match(&node->name, &su->saAmfSUHostedByNode)) { + for (component = su->comp_head; component != NULL; + component = component->next) { + amf_comp_node_left(component); + } + } + } + } + } + + for (app = amf_cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + amf_sg_failover_node_req(sg, node); + } + } +} + +/** + * Node leave event is obtained from amf_confchg_fn + * + * @param node + */ +void amf_node_leave (struct amf_node *node) +{ + ENTER("'%s'", node->name.value); + amf_node_acsm_enter_leaving_spontaneously(node); + amf_node_acsm_enter_failing_over (node); + +} + +/** + * + * @param node + */ +void amf_node_failover (struct amf_node *node) +{ + +} + +/** + * + * @param node + */ +void amf_node_switchover (struct amf_node *node) +{ + +} + +/** + * + * @param node + */ +void amf_node_failfast (struct amf_node *node) +{ + +} + +/** + * + * @param node + * @param comp + */ +void amf_node_comp_restart_req ( + struct amf_node *node, struct amf_comp *comp) +{ + +} + +/** + * + * @param node + * @param comp + */ +void amf_node_comp_failover_req ( + struct amf_node *node, struct amf_comp *comp) +{ + +} /** * Node constructor @@ -120,7 +221,7 @@ * @param cluster * @param node */ -struct amf_node *amf_node_new (struct amf_cluster *cluster, char *name) +struct amf_node *amf_node_new (struct amf_cluster *cluster, char *name) { struct amf_node *node = calloc (1, sizeof (struct amf_node)); @@ -153,7 +254,7 @@ void *amf_node_serialize (struct amf_node *node, int *len) } struct amf_node *amf_node_deserialize ( - struct amf_cluster *cluster, char *buf, int size) + struct amf_cluster *cluster, char *buf, int size) { int objsz = sizeof (struct amf_node); @@ -180,8 +281,8 @@ void amf_node_sync_ready (struct amf_node *node) assert (node != NULL); log_printf(LOG_NOTICE, "Node %s sync ready, starting hosted SUs.", - node->name.value); - + node->name.value); + node->saAmfNodeOperState = SA_AMF_OPERATIONAL_ENABLED; for (app = amf_cluster->application_head; app != NULL; app = app->next) { amf_application_start (app, node); } @@ -196,7 +297,11 @@ struct amf_node *amf_node_find (SaNameT *name) { struct amf_node *node; - ENTER (""); + if (amf_cluster == NULL) { + return NULL; + } + + assert (name != NULL); for (node = amf_cluster->node_head; node != NULL; node = node->next) { if (name_match (&node->name, name)) { @@ -204,6 +309,80 @@ struct amf_node *amf_node_find (SaNameT *name) } } + dprintf ("node %s not found!", name->value); + return NULL; } +struct amf_node *amf_node_find_by_nodeid (unsigned int nodeid) +{ + struct amf_node *node; + + for (node = amf_cluster->node_head; node != NULL; node = node->next) { + if (node->nodeid == nodeid) { + return node; + } + } + + dprintf ("node %u not found!", nodeid); + + return NULL; +} + +static int all_applications_on_node_started (struct amf_node *node, + struct amf_cluster *cluster) +{ + int all_started = 1; + struct amf_application *app; + struct amf_sg *sg; + struct amf_su *su; + + for (app = cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + for (su = sg->su_head; su != NULL; su = su->next) { + if (su->saAmfSUPresenceState != SA_AMF_PRESENCE_INSTANTIATED && + name_match(&su->saAmfSUHostedByNode,&node->name)) { + all_started = 0; + goto done; + } + } + } + } + +done: + return all_started; + +} + +void amf_node_application_started (struct amf_node *node, + struct amf_application *_app) +{ + struct amf_application *app = _app; + + ENTER ("application '%s' started", app->name.value); + + if (all_applications_on_node_started (node, app->cluster)) { + + log_printf(LOG_NOTICE, + "Node: all applications started, assigning workload."); + } + + + for (app = _app->cluster->application_head; app != NULL; + app = app->next) { + amf_application_assign_workload (app, node); + } +} + +void amf_node_application_workload_assigned (struct amf_node *node, + struct amf_application *app) +{ + + log_printf(LOG_NOTICE, "Node: all workload assigned on node %s", + node->name.value); + + /** + * TODO: Set node acsm state + */ +} + diff --git a/exec/amfsg.c b/exec/amfsg.c index 02a4a769..5368f5b7 100644 --- a/exec/amfsg.c +++ b/exec/amfsg.c @@ -148,6 +148,526 @@ #include "main.h" #include "util.h" +static void acsm_enter_activating_standby (struct amf_sg *sg); +static void delete_si_assignments_in_scope (struct amf_sg *sg); + +static void standby_su_activated_cbfn ( + struct amf_si_assignment *si_assignment, int result); + +static void dependent_si_deactivated_cbfn ( + struct amf_si_assignment *si_assignment, int result); + +static const char *sg_recovery_type_text[] = { + "Unknown", + "FailoverSU", + "FailoverNode" +}; + +static void return_to_idle (struct amf_sg *sg) +{ + SaNameT dn; + + ENTER ("sg: %s state: %d", sg->name.value,sg->avail_state); + + sg->avail_state = SG_AC_Idle; + if (sg->recovery_scope.recovery_type != 0) { + switch (sg->recovery_scope.recovery_type) { + case SG_RT_FailoverSU: + case SG_RT_FailoverNode: + assert (sg->recovery_scope.sus[0] != NULL); + amf_su_dn_make (sg->recovery_scope.sus[0], &dn); + log_printf ( + LOG_NOTICE, "'%s' %s recovery action finished", + dn.value, + sg_recovery_type_text[sg->recovery_scope.recovery_type]); + break; + default: + log_printf ( + LOG_NOTICE, "'%s' recovery action finished", + sg_recovery_type_text[0]); + } + } + + if (sg->recovery_scope.sus != NULL) { + free ((void *)sg->recovery_scope.sus); + } + if (sg->recovery_scope.sis != NULL) { + free ((void *)sg->recovery_scope.sis); + } + memset (&sg->recovery_scope, 0, sizeof (struct sg_recovery_scope)); +} + +static int su_instantiated_count (struct amf_sg *sg) +{ + int cnt = 0; + struct amf_su *su; + + for (su = sg->su_head; su != NULL; su = su->next) { + if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) + cnt++; + } + + return cnt; +} + +static int has_any_su_in_scope_active_workload (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + struct amf_si_assignment *si_assignment; + + while (*sus != NULL) { + si_assignment = amf_su_get_next_si_assignment (*sus, NULL); + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { + break; + } + si_assignment = amf_su_get_next_si_assignment ( + *sus, si_assignment); + } + if (si_assignment != NULL) { + break; + } + sus++; + } + return(*sus == NULL); +} + +static int is_standby_for_non_active_si_in_scope (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + + /* + * Check if there is any si in the scope which has no active assignment + * and at least one standby assignment. + */ + while (*sis != NULL) { + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + break; + } + si_assignment = si_assignment->next; + } + if (si_assignment == NULL) { + /* There is no ACTIVE assignment ..*/ + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + break; + } + si_assignment = si_assignment->next; + } + if (si_assignment != NULL) { + /* .. and one STANDBY assignment*/ + break; + } + } + sis++; + } + return(*sis != NULL); +} + +static void acsm_enter_terminating_suspected (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + + sg->avail_state = SG_AC_TerminatingSuspected; + /* + * Terminate suspected SU(s) + */ + while (*sus != 0) { + amf_su_terminate (*sus); + sus++; + } +} + + +/** + * Callback function used by SI when there is no dependent SI to + * deactivate. + * @param sg + */ +static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg) +{ + struct amf_su **sus = sg->recovery_scope.sus; + ENTER("'%s'", sg->name.value); + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. + */ + while (*sus != NULL) { + ENTER("SU %s pr_state='%d'",(*sus)->name.value, + (*sus)->saAmfSUPresenceState); + if (((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_UNINSTANTIATED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_TERMINATION_FAILED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope(sg); + acsm_enter_activating_standby (sg); + } + +} + +static void timer_function_dependent_si_deactivated2 (void *sg) +{ + + ENTER (""); + dependent_si_deactivated_cbfn2 (sg); +} + +static struct amf_si *si_get_dependent (struct amf_si *si) +{ + + struct amf_si *tmp_si = NULL; + ENTER("'%p'",si->depends_on); + if (si->depends_on != NULL) { + + if (si->depends_on->name.length < SA_MAX_NAME_LENGTH) { + si->depends_on->name.value[si->depends_on->name.length] = '\0'; + } + SaNameT res_arr[2]; + int is_match; + + + is_match = sa_amf_grep ((char*)si->depends_on->name.value, + "safDepend=.*,safSi=(.*),safApp=.*", + 2, res_arr); + + if (is_match) { + tmp_si = amf_si_find (si->application, (char*)res_arr[1].value); + } else { + log_printf (LOG_LEVEL_ERROR, "distinguished name for " + "amf_si_depedency failed\n"); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + } + return tmp_si; +} + +struct amf_si *amf_dependent_get_next (struct amf_si *si, + struct amf_si *si_iter) +{ + struct amf_si *tmp_si; + struct amf_application *application; + + ENTER(""); + if (si_iter == NULL) { + assert(amf_cluster != NULL); + application = amf_cluster->application_head; + assert(application != NULL); + tmp_si = application->si_head; + } else { + tmp_si = si_iter->next; + if (tmp_si == NULL) { + application = si->application->next; + if (application == NULL) { + goto out; + } + } + } + + for (; tmp_si != NULL; tmp_si = tmp_si->next) { + struct amf_si *depends_on_si = si_get_dependent (tmp_si); + while (depends_on_si != NULL) { + if (depends_on_si == si) { + goto out; + } + depends_on_si = depends_on_si->next; + } + } + out: + return tmp_si; +} + +static void acsm_enter_deactivating_dependent_workload (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + int callback_pending = 0; + + sg->avail_state = SG_AC_DeactivatingDependantWorkload; + + ENTER("'%s'",sg->name.value); + /* + * For each SI in the recovery scope, find all active assignments + * and request them to be deactivated. + */ + while (*sis != NULL) { + struct amf_si *dependent_si; + struct amf_si *si = *sis; + si_assignment = si->assigned_sis; + dependent_si = amf_dependent_get_next (si, NULL); + + while (dependent_si != NULL) { + si_assignment = dependent_si->assigned_sis; + + while (si_assignment != NULL) { + + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + si_assignment->requested_ha_state = SA_AMF_HA_QUIESCED; + callback_pending = 1; + amf_si_ha_state_assume ( + si_assignment, dependent_si_deactivated_cbfn); + } + si_assignment = si_assignment->next; + } + dependent_si = amf_dependent_get_next (si, dependent_si); + } + sis++; + } + + if (callback_pending == 0) { + poll_timer_handle handle; + ENTER(""); + poll_timer_add (aisexec_poll_handle, 0, sg, + timer_function_dependent_si_deactivated2, &handle); + } +} + +/** + * Enter function for state SG_AC_ActivatingStandby. It activates + * one STANDBY assignment for each SI in the recovery scope. + * @param sg + */ +static void acsm_enter_activating_standby (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + ENTER("'%s'",sg->name.value); + sg->avail_state = SG_AC_ActivatingStandby; + + /* + * For each SI in the recovery scope, find one standby + * SI assignment and activate it. + */ + while (*sis != NULL) { + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + si_assignment->requested_ha_state = SA_AMF_HA_ACTIVE; + amf_si_ha_state_assume ( + si_assignment, standby_su_activated_cbfn); + break; + } + si_assignment = si_assignment->next; + } + sis++; + } +} + +static void acsm_enter_repairing_su (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + ENTER("'%s'",sg->name.value); + sg->avail_state = SG_AC_ReparingSu; + + /* + * Instantiate SUs in current recovery scope until the configured + * preference is fulfiled. + */ + while (*sus != NULL) { + if (su_instantiated_count ((*sus)->sg) < + (*sus)->sg->saAmfSGNumPrefInserviceSUs) { + struct amf_node *node = amf_node_find(&((*sus)->saAmfSUHostedByNode)); + if (node == NULL) { + log_printf (LOG_LEVEL_ERROR, "no node to hosted on su found" + "amf_si_depedency failed\n"); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + if (node->saAmfNodeOperState == SA_AMF_OPERATIONAL_ENABLED) { + amf_su_instantiate ((*sus)); + } else { + return_to_idle (sg); + } + + } + sus++; + } +} + +/** + * Checks if the si pointed out is already in the scope. + * @param sg + * @param si + */ +static int is_si_in_scope(struct amf_sg *sg, struct amf_si *si) +{ + struct amf_si **tmp_sis= sg->recovery_scope.sis; + + while (*tmp_sis != NULL) { + if (*tmp_sis == si) { + break; + } + tmp_sis++; + } + return(*tmp_sis == si); +} + +/** + * Adds the si pointed out to the scope. + * @param sg + * @param si + */ +static void add_si_to_scope ( struct amf_sg *sg, struct amf_si *si) +{ + int number_of_si = 2; /* It shall be at least two */ + struct amf_si **tmp_sis= sg->recovery_scope.sis; + + ENTER ("'%s'", si->name.value); + + while (*tmp_sis != NULL) { + number_of_si++; + tmp_sis++; + } + + sg->recovery_scope.sis = (struct amf_si **) + realloc((void *)sg->recovery_scope.sis, + sizeof (struct amf_si *)*number_of_si); + assert (sg->recovery_scope.sis != NULL); + + tmp_sis= sg->recovery_scope.sis; + while (*tmp_sis != NULL) { + tmp_sis++; + } + + *tmp_sis= si; +} + +/** + * Adds the ssu pointed out to the scope. + * @param sg + * @param su + */ +static void add_su_to_scope (struct amf_sg *sg, struct amf_su *su) +{ + int number_of_su = 2; /* It shall be at least two */ + struct amf_su **tmp_sus= sg->recovery_scope.sus; + + ENTER ("'%s'", su->name.value); + while (*tmp_sus != NULL) { + number_of_su++; + tmp_sus++; + } + sg->recovery_scope.sus = (struct amf_su **) + realloc((void *)sg->recovery_scope.sus, + sizeof (struct amf_su *)*number_of_su); + assert (sg->recovery_scope.sus != NULL); + + tmp_sus= sg->recovery_scope.sus; + while (*tmp_sus != NULL) { + tmp_sus++; + } + + *tmp_sus= su; +} + +/** + * Set recovery scope for failover SU. + * @param sg + * @param su + */ + +static void set_scope_for_failover_su (struct amf_sg *sg, struct amf_su *su) +{ + struct amf_si_assignment *si_assignment; + struct amf_si **sis; + struct amf_su **sus; + + sg->recovery_scope.recovery_type = SG_RT_FailoverSU; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.sus = (struct amf_su **) + calloc (2, sizeof (struct amf_su *)); + sg->recovery_scope.sis = (struct amf_si **) + calloc (1, sizeof (struct amf_si *)); + + assert ((sg->recovery_scope.sus != NULL) && + (sg->recovery_scope.sis != NULL)); + sg->recovery_scope.sus[0] = su; + + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (is_si_in_scope(sg, si_assignment->si) == 0) { + add_si_to_scope(sg,si_assignment->si ); + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + + sus= sg->recovery_scope.sus; + dprintf("The following sus are within the scope:\n"); + while (*sus != NULL) { + dprintf("%s\n", (*sus)->name.value); + sus++; + } + sis= sg->recovery_scope.sis; + dprintf("The following sis are within the scope:\n"); + while (*sis != NULL) { + dprintf("%s\n", (*sis)->name.value); + sis++; + } +} + +static void set_scope_for_failover_node (struct amf_sg *sg, struct amf_node *node) +{ + struct amf_si_assignment *si_assignment; + struct amf_si **sis; + struct amf_su **sus; + struct amf_su *su; + + ENTER ("'%s'", node->name.value); + sg->recovery_scope.recovery_type = SG_RT_FailoverNode; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.sus = (struct amf_su **) + calloc (1, sizeof (struct amf_su *)); + sg->recovery_scope.sis = (struct amf_si **) + calloc (1, sizeof (struct amf_si *)); + + assert ((sg->recovery_scope.sus != NULL) && + (sg->recovery_scope.sis != NULL)); + for (su = sg->su_head; su != NULL; su = su->next) { + if (name_match (&node->name, &su->saAmfSUHostedByNode)) { + add_su_to_scope (sg, su); + } + } + + sus = sg->recovery_scope.sus; + while (*sus != 0) { + su = *sus; + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (is_si_in_scope(sg, si_assignment->si) == 0) { + add_si_to_scope(sg, si_assignment->si ); + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + sus++; + } + + sus= sg->recovery_scope.sus; + dprintf("The following sus are within the scope:\n"); + while (*sus != NULL) { + dprintf("%s\n", (*sus)->name.value); + sus++; + } + sis = sg->recovery_scope.sis; + dprintf("The following sis are within the scope:\n"); + while (*sis != NULL) { + dprintf("%s\n", (*sis)->name.value); + sis++; + } +} + /** * Delete all SI assignments and all CSI assignments * by requesting all contained components. @@ -158,7 +678,7 @@ static void delete_si_assignments (struct amf_su *su) struct amf_csi *csi; struct amf_si *si; struct amf_si_assignment *si_assignment; - + ENTER ("'%s'", su->name.value); for (si = su->sg->application->si_head; si != NULL; si = si->next) { @@ -178,35 +698,29 @@ static void delete_si_assignments (struct amf_su *su) if (si->assigned_sis->su == su) { si_assignment = si->assigned_sis; si->assigned_sis = si_assignment->next; - dprintf ("first"); } else { si_assignment = si->assigned_sis->next; si->assigned_sis->next = NULL; - dprintf ("second"); } - dprintf ("%p, %d, %d", - si_assignment, si_assignment->name.length, - si->assigned_sis->name.length); assert (si_assignment != NULL); free (si_assignment); } } -static int all_si_has_hastate (struct amf_su *su, SaAmfHAStateT hastate) +/** + * Delete all SI assignments and all CSI assignments in current + * recovery scope. + * @param sg + */ +static void delete_si_assignments_in_scope (struct amf_sg *sg) { - struct amf_si_assignment *si_assignment; - int all_confirmed = 1; + struct amf_su **sus= sg->recovery_scope.sus; - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - if (si_assignment->saAmfSISUHAState != hastate) { - all_confirmed = 0; - break; - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + while (*sus != NULL) { + delete_si_assignments (*sus); + sus++; } - return all_confirmed; } /** @@ -214,62 +728,94 @@ static int all_si_has_hastate (struct amf_su *su, SaAmfHAStateT hastate) * @param si_assignment * @param result */ -static void failover_su_si_deactivated_cbfn ( +static void dependent_si_deactivated_cbfn ( struct amf_si_assignment *si_assignment, int result) { + struct amf_sg *sg = si_assignment->su->sg; + struct amf_su **sus = sg->recovery_scope.sus; + struct amf_su *su; + ENTER ("'%s', %d", si_assignment->si->name.value, result); /* - * If all SI assignments for the SU are quiesced, goto next - * state (TerminatingSuspected). + * If all SI assignments for all SUs in the SG are not pending, + * goto next state (TerminatingSuspected). */ - if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_QUIESCED)) { - si_assignment->su->sg->avail_state = SG_AC_TerminatingSuspected; - /* - * Terminate suspected SU(s) + for (su = sg->su_head ; su != NULL; su = su->next) { + struct amf_si_assignment *si_assignment; + si_assignment = amf_su_get_next_si_assignment(su, NULL); + + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState != + si_assignment->requested_ha_state) { + goto still_wating; + } + si_assignment = amf_su_get_next_si_assignment(su, si_assignment); + } + + + } + + still_wating: + + if (su == NULL) { + sus = si_assignment->su->sg->recovery_scope.sus; + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. */ - amf_su_terminate (si_assignment->su); + while (*sus != NULL) { + if (((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_UNINSTANTIATED) && + ((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_TERMINATION_FAILED) && + ((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + break; + } + sus++; + } + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope(sg); + acsm_enter_activating_standby (sg); + } } + LEAVE(""); } -static int su_instantiated_count (struct amf_sg *sg) -{ - int cnt = 0; - struct amf_su *su; - - for (su = sg->su_head; su != NULL; su = su->next) { - if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) - cnt++; - } - - return cnt; -} static void standby_su_activated_cbfn ( struct amf_si_assignment *si_assignment, int result) { - struct amf_su *su; + struct amf_su **sus= si_assignment->su->sg->recovery_scope.sus; + struct amf_si **sis= si_assignment->su->sg->recovery_scope.sis; ENTER ("'%s', %d", si_assignment->si->name.value, result); - /* - * TODO: create SI assignment to spare and assign them + /* + * If all SI assignments for all SIs in the scope are activated, goto next + * state. */ - si_assignment->su->sg->avail_state = SG_AC_AssigningStandbyToSpare; - si_assignment->su->sg->avail_state = SG_AC_ReparingSu; - - if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_ACTIVE)) { - for (su = si_assignment->su->sg->su_head; su != NULL; su = su->next) { - if ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_UNINSTANTIATED) && - (su_instantiated_count (si_assignment->su->sg) < - si_assignment->su->sg->saAmfSGNumPrefInserviceSUs)) { - - amf_su_instantiate (su); - } + while (*sis != NULL) { + if ((*sis)->assigned_sis != NULL && + (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { + break; } + sis++; + } + + if (*sis == NULL) { + /* + * TODO: create SI assignment to spare and assign them + */ + (*sus)->sg->avail_state = SG_AC_AssigningStandbyToSpare; + + acsm_enter_repairing_su ((*sus)->sg); } } @@ -289,7 +835,7 @@ static void assign_si_assumed_cbfn ( * has been assigned or go back to idle state if not cluster * start. */ - for (si = sg->application->si_head; si != NULL; si = si->next) { + for (si = sg->application->si_head; si != NULL; si = si->next) { if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) { for (tmp_si_assignment = si->assigned_sis; @@ -311,7 +857,7 @@ static void assign_si_assumed_cbfn ( switch (sg->avail_state) { case SG_AC_AssigningOnRequest: if (si_assignment_cnt == confirmed_assignments) { - sg->avail_state = SG_AC_Idle; + return_to_idle (sg); amf_application_sg_assigned (sg->application, sg); } else { dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments); @@ -319,75 +865,31 @@ static void assign_si_assumed_cbfn ( break; case SG_AC_AssigningStandBy: { - SaNameT dn; if (si_assignment_cnt == confirmed_assignments) { - sg->avail_state = SG_AC_Idle; - amf_su_dn_make (si_assignment->su, &dn); - sg->avail_state = SG_AC_Idle; - log_printf ( - LOG_NOTICE, "'%s' failover recovery action finished", - dn.value); + return_to_idle (sg); } break; } default: dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt, - confirmed_assignments); + confirmed_assignments); amf_runtime_attributes_print (amf_cluster); assert (0); } } -/** - * Find an SU assigned with standby workload and activate it. - * @param su - */ -static void standby_su_activate (struct amf_su *su) -{ - struct amf_si_assignment *su_si_assignment; - struct amf_si_assignment *si_assignment; - - ENTER ("Old SU '%s'", su->name.value); - - su->sg->avail_state = SG_AC_ActivatingStandby; - - /* - * For each (active) SI assignment on the old SU, find a standby - * SI assignment and activate it. - */ - su_si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (su_si_assignment != NULL) { - for (si_assignment = su_si_assignment->si->assigned_sis; - si_assignment != NULL; - si_assignment = si_assignment->next) { - - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { - - si_assignment->requested_ha_state = SA_AMF_HA_ACTIVE; - amf_si_ha_state_assume ( - si_assignment, standby_su_activated_cbfn); - break; /* one standby is enough */ - } - } - su_si_assignment = amf_su_get_next_si_assignment (su, su_si_assignment); - } - - delete_si_assignments (su); - - LEAVE (""); -} - static inline int div_round (int a, int b) { int res; - + + assert (b != 0); res = a / b; if ((a % b) != 0) res++; return res; } -static int all_su_has_presence_state( +static int all_su_has_presence_state ( struct amf_sg *sg, SaAmfPresenceStateT state) { struct amf_su *su; @@ -403,6 +905,19 @@ static int all_su_has_presence_state( return all_set; } +static int all_su_in_scope_has_presence_state ( + struct amf_sg *sg, SaAmfPresenceStateT state) +{ + struct amf_su **sus= sg->recovery_scope.sus; + while (*sus != NULL) { + if ((*sus)->saAmfSUPresenceState != state) { + break; + } + sus++; + } + return(*sus == NULL); +} + /** * Get number of SIs protected by the specified SG. * @param sg @@ -419,107 +934,143 @@ static int sg_si_count_get (struct amf_sg *sg) cnt += 1; } } - return (cnt); + return(cnt); } -static void sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) +static int sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) { struct amf_su *su; struct amf_si *si; int assigned = 0; - int assign_per_su = 0; + int assign_to_su = 0; int total_assigned = 0; - int si_cnt; + int si_left; + int si_total; + int su_left_to_assign = su_active_assign; - ENTER ("'%s'", sg->name.value); - - si_cnt = sg_si_count_get (sg); - assign_per_su = div_round (si_cnt, su_active_assign); - if (assign_per_su > sg->saAmfSGMaxActiveSIsperSUs) { - assign_per_su = sg->saAmfSGMaxActiveSIsperSUs; + si_total = sg_si_count_get (sg); + si_left = si_total; + assign_to_su = div_round (si_left, su_active_assign); + if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) { + assign_to_su = sg->saAmfSGMaxActiveSIsperSUs; } - si = sg->application->si_head; su = sg->su_head; - while (su != NULL) { + while (su != NULL && su_left_to_assign > 0) { if (amf_su_get_saAmfSUReadinessState (su) != - SA_AMF_READINESS_IN_SERVICE || + SA_AMF_READINESS_IN_SERVICE || amf_su_get_saAmfSUNumCurrActiveSIs (su) == - sg->saAmfSGMaxActiveSIsperSUs || + assign_to_su || amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) { su = su->next; continue; /* Not in service */ } + si = sg->application->si_head; assigned = 0; - while (si != NULL && - assigned < assign_per_su && - total_assigned < si_cnt) { + assign_to_su = div_round (si_left, su_left_to_assign); + while (si != NULL) { - if (amf_si_get_saAmfSINumCurrActiveAssignments (si) == 0) { - assigned += 1; - total_assigned += 1; - amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE); + if (name_match (&si->saAmfSIProtectedbySG, &sg->name) && + assigned < assign_to_su && + amf_si_get_saAmfSINumCurrActiveAssignments(si) < + su_active_assign) { + + if (amf_si_su_get_saAmfSINumCurrActiveAssignments (si, su) == + 0) { + + assigned += 1; + total_assigned += 1; + amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE); + } } si = si->next; } su = su->next; + su_left_to_assign -= 1; + si_left -= assigned; + dprintf (" su_left_to_assign =%d, si_left=%d\n", + su_left_to_assign, si_left); } - + + assert (total_assigned <= si_total); if (total_assigned == 0) { - dprintf ("Info: No SIs assigned!"); + dprintf ("Info: No SIs assigned"); } + LEAVE(); + + return total_assigned; } -static void sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign) +static int sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign) { struct amf_su *su; struct amf_si *si; int assigned = 0; - int assign_per_su = 0; + int assign_to_su = 0; int total_assigned = 0; - int si_cnt; + int si_left; + int si_total; + int su_left_to_assign = su_standby_assign; ENTER ("'%s'", sg->name.value); if (su_standby_assign == 0) { - return; + return 0; } - si_cnt = sg_si_count_get (sg); - assign_per_su = div_round (si_cnt, su_standby_assign); - if (assign_per_su > sg->saAmfSGMaxStandbySIsperSUs) { - assign_per_su = sg->saAmfSGMaxStandbySIsperSUs; + si_total = sg_si_count_get (sg); + si_left = si_total; + assign_to_su = div_round (si_left, su_standby_assign); + if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) { + assign_to_su = sg->saAmfSGMaxStandbySIsperSUs; } - si = sg->application->si_head; su = sg->su_head; - while (su != NULL) { + while (su != NULL && su_left_to_assign > 0) { if (amf_su_get_saAmfSUReadinessState (su) != - SA_AMF_READINESS_IN_SERVICE || + SA_AMF_READINESS_IN_SERVICE || amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 || amf_su_get_saAmfSUNumCurrStandbySIs (su) == - sg->saAmfSGMaxStandbySIsperSUs) { + assign_to_su) { su = su->next; continue; /* Not available for assignment */ } + si = sg->application->si_head; assigned = 0; - while (si != NULL && assigned < assign_per_su) { + assign_to_su = div_round (si_left, su_left_to_assign); + while (si != NULL) { + if (name_match (&si->saAmfSIProtectedbySG, &sg->name) && + assigned < assign_to_su && + amf_si_get_saAmfSINumCurrStandbyAssignments (si) < + su_standby_assign) { - if (amf_si_get_saAmfSINumCurrStandbyAssignments (si) == 0) { - assigned += 1; - total_assigned += 1; - amf_su_assign_si (su, si, SA_AMF_HA_STANDBY); + if (amf_si_su_get_saAmfSINumCurrStandbyAssignments (si, su) == + 0) { + + assigned += 1; + total_assigned += 1; + amf_su_assign_si (su, si, SA_AMF_HA_STANDBY); + } } si = si->next; } + su_left_to_assign -= 1; + si_left -= assigned; + dprintf (" su_left_to_assign =%d, si_left=%d\n", + su_left_to_assign, si_left); + su = su->next; } + + assert (total_assigned <= si_total); if (total_assigned == 0) { dprintf ("Info: No SIs assigned!"); } + + return total_assigned; } static int su_inservice_count_get (struct amf_sg *sg) @@ -534,29 +1085,27 @@ static int su_inservice_count_get (struct amf_sg *sg) answer += 1; } } - return (answer); + return(answer); } + /** * TODO: dependency_level not used, hard coded * @param sg * @param dependency_level */ -void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) +static void assign_si (struct amf_sg *sg, int dependency_level) { int active_sus_needed; int standby_sus_needed; int inservice_count; - int units_for_standby; - int units_for_active; - int ii_spare; int su_active_assign; int su_standby_assign; int su_spare_assign; + int assigned = 0; ENTER ("'%s'", sg->name.value); - sg->avail_state = SG_AC_AssigningOnRequest; /** * Phase 1: Calculate assignments and create all runtime objects in @@ -569,88 +1118,77 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) */ inservice_count = (float)su_inservice_count_get (sg); - active_sus_needed = div_round (sg_si_count_get (sg), + active_sus_needed = div_round ( + sg_si_count_get (sg) * sg->saAmfSGNumPrefActiveSUs, sg->saAmfSGMaxActiveSIsperSUs); - standby_sus_needed = div_round (sg_si_count_get (sg), + standby_sus_needed = div_round ( + sg_si_count_get (sg) * sg->saAmfSGNumPrefStandbySUs, sg->saAmfSGMaxStandbySIsperSUs); - units_for_active = inservice_count - sg->saAmfSGNumPrefStandbySUs; - if (units_for_active < 0) { - units_for_active = 0; - } + dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)" + "\n", + inservice_count, active_sus_needed, standby_sus_needed); - units_for_standby = inservice_count - sg->saAmfSGNumPrefActiveSUs; - if (units_for_standby < 0) { - units_for_standby = 0; - } - - ii_spare = inservice_count - sg->saAmfSGNumPrefActiveSUs - - sg->saAmfSGNumPrefStandbySUs; - - if (ii_spare < 0) { - ii_spare = 0; - } - - /** - * Determine number of active and standby service units + /* Determine number of active and standby service units * to assign based upon reduction procedure */ - if ((inservice_count - active_sus_needed) < 0) { + if ((inservice_count < active_sus_needed)) { dprintf ("assignment VI - partial assignment with SIs drop outs\n"); - su_active_assign = active_sus_needed; + su_active_assign = inservice_count; su_standby_assign = 0; su_spare_assign = 0; } else - if ((inservice_count - active_sus_needed - standby_sus_needed) < 0) { - dprintf ("assignment V - partial assignment with reduction " - "of standby units\n"); + if ((inservice_count < active_sus_needed + standby_sus_needed)) { + dprintf ("assignment V - partial assignment with reduction of" + " standby units\n"); su_active_assign = active_sus_needed; - if (standby_sus_needed > units_for_standby) { - su_standby_assign = units_for_standby; - } else { - su_standby_assign = standby_sus_needed; - } + su_standby_assign = inservice_count - active_sus_needed; su_spare_assign = 0; } else - if ((sg->saAmfSGMaxStandbySIsperSUs * units_for_standby) <= - sg_si_count_get (sg)) { - - dprintf ("IV: full assignment with reduction of active service units\n"); + if ((inservice_count < sg->saAmfSGNumPrefActiveSUs + standby_sus_needed)) { + dprintf ("IV: full assignment with reduction of active service" + " units\n"); su_active_assign = inservice_count - standby_sus_needed; su_standby_assign = standby_sus_needed; su_spare_assign = 0; - } else - if ((sg->saAmfSGMaxActiveSIsperSUs * units_for_active) <= - sg_si_count_get (sg)) { - - dprintf ("III: full assignment with reduction of standby " - "service units\n"); + } else + if ((inservice_count < + sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) { + dprintf ("III: full assignment with reduction of standby service" + " units\n"); su_active_assign = sg->saAmfSGNumPrefActiveSUs; - su_standby_assign = units_for_standby; + su_standby_assign = inservice_count - sg->saAmfSGNumPrefActiveSUs; su_spare_assign = 0; } else - if (ii_spare == 0) { - dprintf ("II: full assignment with spare reduction\n"); + if ((inservice_count == + sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) { + if (sg->saAmfSGNumPrefInserviceSUs > inservice_count) { + dprintf ("II: full assignment with spare reduction\n"); + } else { + dprintf ("II: full assignment without spares\n"); + } su_active_assign = sg->saAmfSGNumPrefActiveSUs; su_standby_assign = sg->saAmfSGNumPrefStandbySUs; su_spare_assign = 0; } else { dprintf ("I: full assignment with spares\n"); - su_active_assign = sg->saAmfSGNumPrefActiveSUs; su_standby_assign = sg->saAmfSGNumPrefStandbySUs; - su_spare_assign = ii_spare; + su_spare_assign = inservice_count - + sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs; } dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)" - " (assigning spares=%d)\n", + " (assigning spares=%d)\n", inservice_count, su_active_assign, su_standby_assign, su_spare_assign); - sg_assign_nm_active (sg, su_active_assign); - sg_assign_nm_standby (sg, su_standby_assign); + assigned = sg_assign_nm_active (sg, su_active_assign); + assigned += sg_assign_nm_standby (sg, su_standby_assign); + + assert (assigned > 0); sg->saAmfSGNumCurrAssignedSUs = inservice_count; @@ -665,7 +1203,7 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) for (si = sg->application->si_head; si != NULL; si = si->next) { if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) { for (si_assignment = si->assigned_sis; si_assignment != NULL; - si_assignment = si_assignment->next) { + si_assignment = si_assignment->next) { if (si_assignment->requested_ha_state != si_assignment->saAmfSISUHAState) { @@ -680,9 +1218,71 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) LEAVE ("'%s'", sg->name.value); } +void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) +{ + sg->avail_state = SG_AC_AssigningOnRequest; + assign_si (sg, dependency_level); +} + +void amf_sg_failover_node_req ( + struct amf_sg *sg, struct amf_node *node) +{ + + ENTER("'%s'",node->name.value); + + /* + * TODO: Defer all new events. Workaround is to exit. + */ + if (sg->avail_state != SG_AC_Idle) { + log_printf (LOG_LEVEL_ERROR, "To handle multiple simultaneous SG" + " recovery actions is not implemented yet:" + " SG '%s', NODE '%s', avail_state %d", + sg->name.value, node->name.value, sg->avail_state); + openais_exit_error (AIS_DONE_FATAL_ERR); + return; + } + + set_scope_for_failover_node(sg, node); + + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + struct amf_su **sus = sg->recovery_scope.sus; + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. + */ + while (*sus != NULL) { + ENTER("SU %s pr_state='%d'",(*sus)->name.value, + (*sus)->saAmfSUPresenceState); + if (((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_UNINSTANTIATED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_TERMINATION_FAILED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope (sg); + return_to_idle (sg); + } + + } + +} + void amf_sg_start (struct amf_sg *sg, struct amf_node *node) { struct amf_su *su; + sg_avail_control_state_t old_avail_state = sg->avail_state; + int instantiated_sus = 0; ENTER ("'%s'", sg->name.value); @@ -692,41 +1292,63 @@ void amf_sg_start (struct amf_sg *sg, struct amf_node *node) if (node == NULL) { /* Cluster start */ amf_su_instantiate (su); + instantiated_sus++; } else { /* Node start, match if SU is hosted on the specified node */ if (name_match (&node->name, &su->saAmfSUHostedByNode)) { amf_su_instantiate (su); + instantiated_sus++; } } } + + if (instantiated_sus == 0) { + sg->avail_state = old_avail_state; + } } -void amf_sg_su_state_changed ( - struct amf_sg *sg, struct amf_su *su, SaAmfStateT type, int state) +void amf_sg_su_state_changed (struct amf_sg *sg, + struct amf_su *su, SaAmfStateT type, int state) { ENTER ("'%s' SU '%s' state %s", - sg->name.value, su->name.value, amf_presence_state(state)); + sg->name.value, su->name.value, amf_presence_state(state)); if (type == SA_AMF_PRESENCE_STATE) { if (state == SA_AMF_PRESENCE_INSTANTIATED) { - if (all_su_has_presence_state(su->sg, - SA_AMF_PRESENCE_INSTANTIATED)) { - - if (sg->avail_state == SG_AC_InstantiatingServiceUnits) { + if (sg->avail_state == SG_AC_InstantiatingServiceUnits) { + if (all_su_has_presence_state(su->sg, + SA_AMF_PRESENCE_INSTANTIATED)) { su->sg->avail_state = SG_AC_Idle; amf_application_sg_started ( sg->application, sg, this_amf_node); - } else if (sg->avail_state == SG_AC_ReparingSu) { + } + } else if (sg->avail_state == SG_AC_ReparingSu) { + if (all_su_in_scope_has_presence_state(su->sg, + SA_AMF_PRESENCE_INSTANTIATED)) { su->sg->avail_state = SG_AC_AssigningStandBy; - amf_sg_assign_si (sg, 0); + assign_si (sg, 0); + } else { dprintf ("avail-state: %u", sg->avail_state); assert (0); } + } else { + assert (0); } } else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) { if (sg->avail_state == SG_AC_TerminatingSuspected) { - standby_su_activate (su); + if (all_su_in_scope_has_presence_state (sg, state)) { + delete_si_assignments_in_scope (sg); + if (is_standby_for_non_active_si_in_scope (sg)) { + acsm_enter_activating_standby (sg); + } else { + /* + * TODO: create SI assignment to spare and assign them + */ + sg->avail_state = SG_AC_AssigningStandbyToSpare; + acsm_enter_repairing_su (sg); + } + } } else { assert (0); } @@ -746,24 +1368,23 @@ void amf_sg_init (void) void amf_sg_failover_su_req ( struct amf_sg *sg, struct amf_su *su, struct amf_node *node) { - struct amf_si_assignment *si_assignment; - ENTER (""); - - sg->avail_state = SG_AC_DeactivatingDependantWorkload; - - /* - * Deactivate workload for SU - */ - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { - si_assignment->requested_ha_state = SA_AMF_HA_QUIESCED; - amf_si_ha_state_assume ( - si_assignment, failover_su_si_deactivated_cbfn); - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + /* + * TODO: Defer all new events. Workaround is to exit. + */ + if (sg->avail_state != SG_AC_Idle) { + log_printf (LOG_LEVEL_ERROR, "To handle multiple simultaneous SG" + " recovery actions is not implemented yet:" + " SG '%s', SU '%s', avail_state %d", + sg->name.value, su->name.value, sg->avail_state); + openais_exit_error (AIS_DONE_FATAL_ERR); + return; + } + set_scope_for_failover_su (sg, su); + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + acsm_enter_terminating_suspected (sg); } } @@ -907,6 +1528,8 @@ struct amf_sg *amf_sg_find (struct amf_application *app, char *name) { struct amf_sg *sg; + assert (app != NULL && name != NULL); + for (sg = app->sg_head; sg != NULL; sg = sg->next) { if (strncmp (name, (char*)sg->name.value, sg->name.length) == 0) { break; diff --git a/exec/amfsi.c b/exec/amfsi.c index f61974fb..22a2c0bd 100644 --- a/exec/amfsi.c +++ b/exec/amfsi.c @@ -340,7 +340,6 @@ void amf_si_ha_state_assume ( amf_ha_state (si_assignment->requested_ha_state)); si_assignment->assumed_callback_fn = assumed_ha_state_callback_fn; - for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) { for (csi_assignment = csi->assigned_csis; csi_assignment != NULL; csi_assignment = csi_assignment->next) { @@ -379,7 +378,7 @@ void amf_si_ha_state_assume ( if (csi_assignment_cnt == hastate_set_done_cnt) { poll_timer_handle handle; poll_timer_add (aisexec_poll_handle, 0, si_assignment, - timer_function_ha_state_assumed, &handle); + timer_function_ha_state_assumed, &handle); } } @@ -390,6 +389,7 @@ void amf_si_ha_state_assume ( * @return int */ int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si) + { int cnt = 0; struct amf_si_assignment *si_assignment; @@ -405,6 +405,23 @@ int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si) return cnt; } +int amf_si_su_get_saAmfSINumCurrActiveAssignments (struct amf_si *si, + struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + for (si_assignment = si->assigned_sis; si_assignment != NULL; + si_assignment = si_assignment->next) { + + if (si_assignment->su == su && + si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + cnt++; + } + } + + return cnt; +} + int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si) { int cnt = 0; @@ -421,6 +438,24 @@ int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si) return cnt; } +int amf_si_su_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si, + struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + + for (si_assignment = si->assigned_sis; si_assignment != NULL; + si_assignment = si_assignment->next) { + + if (si_assignment->su == su && + si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + cnt++; + } + } + + return cnt; +} + SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (struct amf_si *si) { if ((amf_si_get_saAmfSINumCurrActiveAssignments (si) == @@ -439,24 +474,21 @@ SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (struct amf_si *si) void amf_csi_delete_assignments (struct amf_csi *csi, struct amf_su *su) { struct amf_csi_assignment *csi_assignment; - ENTER ("'%s'", su->name.value); + struct amf_csi_assignment **prev = &csi->assigned_csis; - /* - * TODO: this only works for n+m where each CSI list has only - * two assignments, one active and one standby. - * TODO: use DN instead - */ - if (csi->assigned_csis->comp->su == su) { - csi_assignment = csi->assigned_csis; - csi->assigned_csis = csi_assignment->next; - } else { - csi_assignment = csi->assigned_csis->next; - csi->assigned_csis->next = NULL; - assert (csi_assignment != NULL && csi_assignment->comp->su == su); + + for (csi_assignment = csi->assigned_csis; csi_assignment != NULL; + csi_assignment = csi_assignment->next) { + if (csi_assignment->comp->su == su) { + struct amf_csi_assignment *tmp = csi_assignment; + *prev = csi_assignment->next; + dprintf ("CSI assignment %s unlinked", tmp->name.value); + free (tmp); + } else { + prev = &csi_assignment->next; + } } - assert (csi_assignment != NULL); - free (csi_assignment); } /** @@ -538,7 +570,8 @@ void *amf_si_serialize (struct amf_si *si, int *len) return copy; } -struct amf_si *amf_si_deserialize (struct amf_application *app, char *buf, int size) +struct amf_si *amf_si_deserialize ( + struct amf_application *app, char *buf, int size) { int objsz = sizeof (struct amf_si); @@ -550,7 +583,7 @@ struct amf_si *amf_si_deserialize (struct amf_application *app, char *buf, int s TRACE8 ("%s", si->name.value); memcpy (&si->saAmfSIProtectedbySG, &tmp->saAmfSIProtectedbySG, - sizeof (SaNameT)); + sizeof (SaNameT)); si->saAmfSIRank = tmp->saAmfSIRank; si->saAmfSINumCSIs = tmp->saAmfSINumCSIs; si->saAmfSIPrefActiveAssignments = tmp->saAmfSIPrefActiveAssignments; @@ -568,7 +601,7 @@ struct amf_si_assignment *amf_si_assignment_new (struct amf_si *si) { struct amf_si_assignment *si_assignment; - si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); + si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); si_assignment->si = si; return si_assignment; @@ -614,14 +647,16 @@ struct amf_si *amf_si_find (struct amf_application *app, char *name) { struct amf_si *si; - ENTER ("%s", name); - for (si = app->si_head; si != NULL; si = si->next) { if (strncmp (name, (char*)si->name.value, si->name.length) == 0) { break; } } + if (si == NULL) { + dprintf ("SI %s not found!", name); + } + return si; } @@ -691,14 +726,15 @@ struct amf_csi *amf_csi_find (struct amf_si *si, char *name) { struct amf_csi *csi; - ENTER ("%s", name); - for (csi = si->csi_head; csi != NULL; csi = csi->next) { if (strncmp (name, (char*)csi->name.value, csi->name.length) == 0) { break; } } + if (csi == NULL) { + dprintf ("CSI %s not found!", name); + } return csi; } @@ -730,6 +766,30 @@ void *amf_csi_assignment_serialize ( return copy; } +struct amf_si_assignment *si_assignment_find ( + struct amf_csi_assignment *csi_assignment) +{ + + struct amf_comp *component; + struct amf_si_assignment *si_assignment = NULL; + + component = amf_comp_find(csi_assignment->csi->si->application->cluster, + &csi_assignment->name); + + + for (si_assignment = csi_assignment->csi->si->assigned_sis; + si_assignment != NULL; si_assignment = si_assignment->next) { + SaNameT su_name; + amf_su_dn_make (component->su,&su_name); + + if (name_match(&su_name, &si_assignment->name)) { + break; + } + } + + return si_assignment; +} + struct amf_csi_assignment *amf_csi_assignment_deserialize ( struct amf_csi *csi, char *buf, int size) { @@ -746,8 +806,11 @@ struct amf_csi_assignment *amf_csi_assignment_deserialize ( TRACE8 ("%s", obj->name.value); obj->csi = csi; obj->comp = amf_comp_find (csi->si->application->cluster, &obj->name); + assert (obj->comp != NULL); obj->next = csi->assigned_csis; csi->assigned_csis = obj; + + obj->si_assignment = si_assignment_find(obj); return obj; } } @@ -785,9 +848,7 @@ struct amf_csi_assignment *amf_csi_assignment_find ( char *csi_assignment_name; char *buf; - ENTER ("%s", name->value); - - /* malloc new buffer since we need to write to the buffer */ + /* malloc new buffer since we need to write to the buffer */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value, name->length + 1); @@ -808,7 +869,7 @@ struct amf_csi_assignment *amf_csi_assignment_find ( *(si_name - 1) = '\0'; *(app_name - 1) = '\0'; - /* jump to value */ + /* jump to value */ csi_assignment_name += 11; csi_name += 7; si_name += 6; @@ -867,11 +928,11 @@ void *amf_csi_attribute_serialize ( /* count value and write to buf */ for (i = 0; csi_attribute->value && - csi_attribute->value[i] != NULL; i++); + csi_attribute->value[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); for (i = 0; csi_attribute->value && - csi_attribute->value[i] != NULL; i++) { + csi_attribute->value[i] != NULL; i++) { buf = amf_serialize_SaStringT ( buf, &size, &offset, csi_attribute->value[i]); } diff --git a/exec/amfsu.c b/exec/amfsu.c index 1595cfe2..89ee88b7 100644 --- a/exec/amfsu.c +++ b/exec/amfsu.c @@ -128,7 +128,7 @@ #include "print.h" #include "main.h" -static int presence_state_all_comps_in_su_are_set (struct amf_su *su, +int amf_su_presence_state_all_comps_in_su_are_set (struct amf_su *su, SaAmfPresenceStateT state) { int all_set = 1; @@ -219,8 +219,8 @@ static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi, csi_assignment->next = csi->assigned_csis; csi->assigned_csis = csi_assignment; amf_comp_dn_make (comp, &csi_assignment->name); - csi_assignment->csi = csi; csi_assignment->comp = comp; + csi_assignment->csi = csi; csi_assignment->saAmfCSICompHAState = 0; /* undefined confirmed HA state */ csi_assignment->requested_ha_state = ha_state; csi_assignment->si_assignment = si_assignment; @@ -235,8 +235,8 @@ static void su_restart (struct amf_su *su) amf_su_dn_make (su, &dn); log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tSU restart", dn.value); - + "action:\n\t\tSU restart", dn.value); + su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING; su->restart_control_state = SU_RC_RESTART_SU_INSTANTIATING; su->escalation_level_history_state = @@ -256,7 +256,7 @@ static void comp_restart (struct amf_comp *comp) ENTER ("'%s'", comp->name.value); amf_comp_dn_make (comp, &dn); log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tcomponent restart", dn.value); + "action:\n\t\tcomponent restart", dn.value); comp->su->restart_control_state = SU_RC_RESTART_COMP_DEACTIVATING; comp->su->restart_control_state = SU_RC_RESTART_COMP_RESTARTING; @@ -311,7 +311,7 @@ void amf_su_assign_si (struct amf_su *su, struct amf_si *si, for (csi = si->csi_head; csi != NULL; csi = csi->next) { if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value, - cs_type->length)) { + cs_type->length)) { comp_assign_csi (comp, csi, si_assignment, ha_state); no_of_assignments++; } @@ -406,7 +406,7 @@ static void su_comp_presence_state_changed ( * TODO: send to node */ case SU_RC_ESCALATION_LEVEL_0: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( comp->su, SA_AMF_PRESENCE_INSTANTIATED)) { su_presence_state_set ( @@ -418,7 +418,7 @@ static void su_comp_presence_state_changed ( reassign_sis (comp->su); break; case SU_RC_RESTART_SU_INSTANTIATING: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( comp->su, SA_AMF_PRESENCE_INSTANTIATED)) { su->restart_control_state = SU_RC_RESTART_SU_SETTING; @@ -433,7 +433,7 @@ static void su_comp_presence_state_changed ( } break; case SA_AMF_PRESENCE_UNINSTANTIATED: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( su, SA_AMF_PRESENCE_UNINSTANTIATED)) { su_presence_state_set (comp->su, @@ -458,25 +458,25 @@ static void su_comp_op_state_changed ( switch (state) { case SA_AMF_OPERATIONAL_ENABLED: - { - struct amf_comp *comp_compare; - int all_set = 1; - for (comp_compare = comp->su->comp_head; - comp_compare != NULL; comp_compare = comp_compare->next) { - if (comp_compare->saAmfCompOperState != + { + struct amf_comp *comp_compare; + int all_set = 1; + for (comp_compare = comp->su->comp_head; + comp_compare != NULL; comp_compare = comp_compare->next) { + if (comp_compare->saAmfCompOperState != SA_AMF_OPERATIONAL_ENABLED) { - all_set = 0; - break; + all_set = 0; + break; + } } + if (all_set) { + su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_ENABLED); + } else { + su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED); + } + break; } - if (all_set) { - su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_ENABLED); - } else { - su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED); - } - break; - } case SA_AMF_OPERATIONAL_DISABLED: break; default: @@ -534,17 +534,9 @@ void amf_su_comp_error_suspected ( { ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value); - /* - * Defer all new events. Workaround to be able to use gdb. - */ - if (su->sg->avail_state != SG_AC_Idle) { - ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value); - fprintf (stderr, "Warning Debug: event deferred!\n"); - return; - } - switch (su->restart_control_state) { case SU_RC_ESCALATION_LEVEL_0: + if (comp->saAmfCompRestartCount >= su->sg->saAmfSGCompRestartMax) { su->restart_control_state = SU_RC_ESCALATION_LEVEL_1; amf_su_comp_error_suspected (su, comp, recommended_recovery); @@ -572,29 +564,23 @@ void amf_su_comp_error_suspected ( /* * TODO: delegate to node */ - struct amf_si_assignment *si_assignment = - amf_su_get_next_si_assignment (su, NULL); - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { - SaNameT dn; + SaNameT dn; - su_operational_state_set (su, SA_AMF_OPERATIONAL_DISABLED); - amf_comp_operational_state_set ( - comp, SA_AMF_OPERATIONAL_DISABLED); - amf_comp_dn_make (comp, &dn); - log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tSU failover", dn.value); - amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node); - return; - } else { - su_restart (comp->su); - } + su_operational_state_set (su, SA_AMF_OPERATIONAL_DISABLED); + amf_comp_operational_state_set ( + comp, SA_AMF_OPERATIONAL_DISABLED); + amf_comp_dn_make (comp, &dn); + log_printf (LOG_NOTICE, "Error detected for '%s', recovery " + "action:\n\t\tSU failover", dn.value); + amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node); + return; } else { su_restart (comp->su); } break; default: - assert (0); + break; } } @@ -620,9 +606,13 @@ void amf_su_terminate (struct amf_su *su) char *amf_su_dn_make (struct amf_su *su, SaNameT *name) { - int i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, + int i; + + assert (su != NULL); + + i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, "safSu=%s,safSg=%s,safApp=%s", - su->name.value, su->sg->name.value, su->sg->application->name.value); + su->name.value, su->sg->name.value, su->sg->application->name.value); assert (i <= SA_MAX_NAME_LENGTH); name->length = i; return (char *)name->value; @@ -676,7 +666,7 @@ struct amf_si_assignment *amf_su_get_next_si_assignment ( void amf_su_foreach_si_assignment ( struct amf_su *su, void (*foreach_fn)(struct amf_su *su, - struct amf_si_assignment *si_assignment)) + struct amf_si_assignment *si_assignment)) { struct amf_si_assignment *si_assignment; @@ -715,7 +705,7 @@ int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su) { int cnt = 0; struct amf_si_assignment *si_assignment; - + si_assignment = amf_su_get_next_si_assignment (su, NULL); while (si_assignment != NULL) { if (su->sg->avail_state == SG_AC_AssigningOnRequest && @@ -736,7 +726,7 @@ SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su) { if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) && ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) || - (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { + (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { return SA_AMF_READINESS_IN_SERVICE; } else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) { @@ -773,7 +763,7 @@ struct amf_su *amf_su_new (struct amf_sg *sg, char *name) } if (tail == NULL) { - sg->su_head = su; + sg->su_head = su; } else { tail->next = su; } @@ -883,6 +873,8 @@ struct amf_su *amf_su_find (struct amf_cluster *cluster, SaNameT *name) char *ptrptr; char *buf; + assert (cluster != NULL && name != NULL); + /* malloc new buffer since strtok_r writes to its first argument */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value, name->length + 1); diff --git a/exec/amfutil.c b/exec/amfutil.c index 655caf0f..82a9f8d5 100644 --- a/exec/amfutil.c +++ b/exec/amfutil.c @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include "../include/saAis.h" #include "../include/saAmf.h" @@ -113,21 +115,21 @@ static const char *assignment_state_text[] = { static int init_category (struct amf_comp *comp, char *loc) { - if (strcmp (loc, "sa_aware") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_SA_AWARE; - } else if (strcmp (loc, "proxy") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_PROXY; - } else if (strcmp (loc, "proxied") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_PROXIED; - } else if (strcmp (loc, "local") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_LOCAL; - } else { - return -1; - } - - return 0; -} - + if (strcmp (loc, "sa_aware") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_SA_AWARE; + } else if (strcmp (loc, "proxy") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_PROXY; + } else if (strcmp (loc, "proxied") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_PROXIED; + } else if (strcmp (loc, "local") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_LOCAL; + } else { + return -1; + } + + return 0; +} + static int init_capability (struct amf_comp *comp, char *loc) { if (strcmp (loc, "x_active_and_y_standby") == 0) { @@ -147,10 +149,10 @@ static int init_capability (struct amf_comp *comp, char *loc) } else { return -1; } - + return 0; } - + static int init_recovery_on_error (struct amf_comp *comp, char *loc) { if (strcmp (loc, "component_restart") == 0) { @@ -258,8 +260,7 @@ struct amf_cluster *amf_config_read (char **error_string) int csi_dependencies_cnt = 0; char *error_reason = NULL; char *value; - - filename = getenv("OPENAIS_AMF_CONFIG_FILE"); + filename = getenv ("OPENAIS_AMF_CONFIG_FILE"); if (!filename) { filename = "/etc/ais/amf.conf"; } @@ -267,7 +268,7 @@ struct amf_cluster *amf_config_read (char **error_string) fp = fopen (filename, "r"); if (fp == 0) { sprintf (buf, "Can't read %s file reason = (%s).\n", - filename, strerror (errno)); + filename, strerror (errno)); *error_string = buf; return NULL; } @@ -395,6 +396,10 @@ struct amf_cluster *amf_config_read (char **error_string) } else if ((loc = strstr_rs (line, "safSg=")) != 0) { sg = amf_sg_new (app, trim_str (loc)); sg_cnt++; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.recovery_type = 0; + sg->recovery_scope.sis = NULL; + sg->recovery_scope.sus = NULL; current_parse = AMF_SG; su_cnt = 0; } else if ((loc = strstr_rs (line, "safSi=")) != 0) { @@ -406,7 +411,7 @@ struct amf_cluster *amf_config_read (char **error_string) if (sg_cnt == 1) { for (si = app->si_head; si != NULL; si = si->next) { memcpy (&si->saAmfSIProtectedbySG, &sg->name, - sizeof (SaNameT)); + sizeof (SaNameT)); } } else { for (si = app->si_head; si != NULL; si = si->next) { @@ -689,7 +694,7 @@ struct amf_cluster *amf_config_read (char **error_string) } else { comp_cs_type_cnt++; comp->saAmfCompCsTypes = realloc (comp->saAmfCompCsTypes, - (comp_cs_type_cnt + 1) * sizeof(SaNameT)); + (comp_cs_type_cnt + 1) * sizeof(SaNameT)); comp->saAmfCompCsTypes[comp_cs_type_cnt] = NULL; comp->saAmfCompCsTypes[comp_cs_type_cnt - 1] = amf_malloc (sizeof(SaNameT)); setSaNameT (comp->saAmfCompCsTypes[comp_cs_type_cnt - 1], line); @@ -702,7 +707,7 @@ struct amf_cluster *amf_config_read (char **error_string) } else if ((loc = strchr (line, '=')) != 0) { comp_env_var_cnt++; comp->saAmfCompCmdEnv = realloc (comp->saAmfCompCmdEnv, - (comp_env_var_cnt + 1) * sizeof(SaStringT)); + (comp_env_var_cnt + 1) * sizeof(SaStringT)); comp->saAmfCompCmdEnv[comp_env_var_cnt] = NULL; env_var = comp->saAmfCompCmdEnv[comp_env_var_cnt - 1] = amf_malloc (strlen (line) + 1); strcpy (env_var, line); @@ -744,9 +749,9 @@ struct amf_cluster *amf_config_read (char **error_string) csi->si = si; setSaNameT (&csi->name, trim_str (loc)); current_parse = AMF_CSI; - } else if ((loc = strstr_rs (line, "saAmfSIProtectedbySG{")) != 0) { + } else if ((loc = strstr_rs (line, "saAmfSIProtectedbySG=")) != 0) { setSaNameT (&si->saAmfSIProtectedbySG, loc); - } else if ((loc = strstr_rs (line, "saAmfSIRank{")) != 0) { + } else if ((loc = strstr_rs (line, "saAmfSIRank=")) != 0) { si->saAmfSIRank = atoi (loc); } else if ((loc = strstr_rs (line, "saAmfSINumCSIs=")) != 0) { si->saAmfSINumCSIs = atoi (loc); @@ -860,7 +865,7 @@ struct amf_cluster *amf_config_read (char **error_string) parse_error: sprintf (buf, "parse error at %s: %d: %s\n", - filename, line_number, error_reason); + filename, line_number, error_reason); *error_string = buf; fclose (fp); return NULL; @@ -879,7 +884,7 @@ static void print_si_assignment (struct amf_su *su, { log_printf (LOG_INFO, " safSi=%s\n", si_assignment->si->name.value); log_printf (LOG_INFO, " HA state: %s\n", - ha_state_text[si_assignment->saAmfSISUHAState]); + ha_state_text[si_assignment->saAmfSISUHAState]); } void amf_runtime_attributes_print (struct amf_cluster *cluster) @@ -896,49 +901,49 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) log_printf (LOG_INFO, "==================================================="); log_printf (LOG_INFO, "safCluster=%s", getSaNameT(&cluster->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[cluster->saAmfClusterAdminState]); + admin_state_text[cluster->saAmfClusterAdminState]); for (node = cluster->node_head; node != NULL; node = node->next) { log_printf (LOG_INFO, " safNode=%s\n", getSaNameT (&node->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[node->saAmfNodeAdminState]); + admin_state_text[node->saAmfNodeAdminState]); log_printf (LOG_INFO, " oper state: %s\n", - oper_state_text[node->saAmfNodeOperState]); + oper_state_text[node->saAmfNodeOperState]); } for (app = cluster->application_head; app != NULL; app = app->next) { log_printf (LOG_INFO, " safApp=%s\n", getSaNameT(&app->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[app->saAmfApplicationAdminState]); + admin_state_text[app->saAmfApplicationAdminState]); log_printf (LOG_INFO, " num_sg: %d\n", app->saAmfApplicationCurrNumSG); - for (sg = app->sg_head; sg != NULL; sg = sg->next) { - log_printf (LOG_INFO, " safSG=%s\n", getSaNameT(&sg->name)); + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + log_printf (LOG_INFO, " safSg=%s\n", getSaNameT(&sg->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[sg->saAmfSGAdminState]); + admin_state_text[sg->saAmfSGAdminState]); log_printf (LOG_INFO, " assigned SUs %d\n", - sg->saAmfSGNumCurrAssignedSUs); + sg->saAmfSGNumCurrAssignedSUs); log_printf (LOG_INFO, " non inst. spare SUs %d\n", - sg->saAmfSGNumCurrNonInstantiatedSpareSUs); + sg->saAmfSGNumCurrNonInstantiatedSpareSUs); log_printf (LOG_INFO, " inst. spare SUs %d\n", - sg->saAmfSGNumCurrInstantiatedSpareSUs); + sg->saAmfSGNumCurrInstantiatedSpareSUs); for (su = sg->su_head; su != NULL; su = su->next) { log_printf (LOG_INFO, " safSU=%s\n", getSaNameT(&su->name)); log_printf (LOG_INFO, " oper state: %s\n", - oper_state_text[su->saAmfSUOperState]); + oper_state_text[su->saAmfSUOperState]); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[su->saAmfSUAdminState]); + admin_state_text[su->saAmfSUAdminState]); log_printf (LOG_INFO, " readiness state: %s\n", readiness_state_text[amf_su_get_saAmfSUReadinessState (su)]); log_printf (LOG_INFO, " presence state: %s\n", - presence_state_text[su->saAmfSUPresenceState]); + presence_state_text[su->saAmfSUPresenceState]); log_printf (LOG_INFO, " hosted by node %s\n", - su->saAmfSUHostedByNode.value); + su->saAmfSUHostedByNode.value); log_printf (LOG_INFO, " num active SIs %d\n", - amf_su_get_saAmfSUNumCurrActiveSIs (su)); + amf_su_get_saAmfSUNumCurrActiveSIs (su)); log_printf (LOG_INFO, " num standby SIs %d\n", - amf_su_get_saAmfSUNumCurrStandbySIs (su)); + amf_su_get_saAmfSUNumCurrStandbySIs (su)); log_printf (LOG_INFO, " restart count %d\n", - su->saAmfSURestartCount); + su->saAmfSURestartCount); log_printf (LOG_INFO, " restart control state %d\n", - su->restart_control_state); + su->restart_control_state); log_printf (LOG_INFO, " SU failover cnt %d\n", su->su_failover_cnt); log_printf (LOG_INFO, " assigned SIs:"); amf_su_foreach_si_assignment (su, print_si_assignment); @@ -956,7 +961,7 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) log_printf (LOG_INFO, " num standby CSIs %d\n", amf_comp_get_saAmfCompNumCurrStandbyCsi (comp)); log_printf (LOG_INFO, " restart count %d\n", - comp->saAmfCompRestartCount); + comp->saAmfCompRestartCount); log_printf (LOG_INFO, " assigned CSIs:"); amf_comp_foreach_csi_assignment ( comp, print_csi_assignment); @@ -969,11 +974,11 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) admin_state_text[si->saAmfSIAdminState]); log_printf (LOG_INFO, " assignm. state: %s\n", assignment_state_text[ - amf_si_get_saAmfSIAssignmentState (si)]); + amf_si_get_saAmfSIAssignmentState (si)]); log_printf (LOG_INFO, " active assignments: %d\n", - amf_si_get_saAmfSINumCurrActiveAssignments (si)); + amf_si_get_saAmfSINumCurrActiveAssignments (si)); log_printf (LOG_INFO, " standby assignments: %d\n", - amf_si_get_saAmfSINumCurrStandbyAssignments (si)); + amf_si_get_saAmfSINumCurrStandbyAssignments (si)); for (csi = si->csi_head; csi != NULL; csi = csi->next) { log_printf (LOG_INFO, " safCsi=%s\n", getSaNameT(&csi->name)); } @@ -991,22 +996,22 @@ int amf_enabled (struct objdb_iface_ver0 *objdb) objdb->object_find_reset (OBJECT_PARENT_HANDLE); if (objdb->object_find ( - OBJECT_PARENT_HANDLE, - "amf", - strlen ("amf"), - &object_service_handle) == 0) { + OBJECT_PARENT_HANDLE, + "amf", + strlen ("amf"), + &object_service_handle) == 0) { value = NULL; - if ( !objdb->object_key_get (object_service_handle, - "mode", - strlen ("mode"), - (void *)&value, - NULL) && value) { + if (!objdb->object_key_get (object_service_handle, + "mode", + strlen ("mode"), + (void *)&value, + NULL) && value) { if (strcmp (value, "enabled") == 0) { enabled = 1; } else - if (strcmp (value, "disabled") == 0) { + if (strcmp (value, "disabled") == 0) { enabled = 0; } } @@ -1146,7 +1151,7 @@ char *amf_deserialize_SaStringT (char *buf, SaStringT *str) memcpy (tmp_str, tmp, len); tmp_str[len] = '\0'; *str = tmp_str; - } else { + } else { *str = NULL; } @@ -1185,3 +1190,86 @@ void *_amf_malloc (size_t size, char *file, unsigned int line) return tmp; } + +int sa_amf_grep_one_sub_match(const char *string, char *pattern, + SaNameT *matches_arr) +{ + int status; + regex_t re; + size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + + int i; + + ENTER("'%s %s'",string, pattern); + + if (regcomp(&re, pattern, REG_EXTENDED) != 0) { + status = 0; + goto out; + } + status = regexec(&re, string, nmatch, pmatch, 0); + if (status != 0) { + regfree(&re); + status = 0; + goto out; + } else { + + + for (i = 0; i < nmatch; i++) { + int sub_string_len; + sub_string_len = pmatch[i].rm_eo - pmatch[i].rm_so; + if (i==1) { + + memcpy(matches_arr[i].value, string + pmatch[i].rm_so, + sub_string_len); + matches_arr[i].value[sub_string_len] = '\0'; + } + } + status = 1; + regfree(&re); + } + out: + return status; + +} + + +int sa_amf_grep(const char *string, char *pattern, size_t nmatch, + SaNameT *matches_arr) +{ + int status; + regex_t re; + regmatch_t pmatch[nmatch]; + + int i; + + ENTER("'%s %s'",string, pattern); + if (regcomp(&re, pattern, REG_EXTENDED) != 0) { + status = 0; + goto out; + } + status = regexec(&re, string, nmatch, pmatch, 0); + if (status != 0) { + regfree(&re); + status = 0; + goto out; + } else { + + + for (i = 0; i < nmatch; i++) { + int sub_string_len; + sub_string_len = pmatch[i].rm_eo - pmatch[i].rm_so; + memcpy(matches_arr[i].value, string + pmatch[i].rm_so, + sub_string_len); + matches_arr[i].value[sub_string_len] = '\0'; + matches_arr[i].length = sub_string_len; + } + status = 1; + regfree(&re); + } +out: + return status; + +} + +