diff --git a/exec/amf.c b/exec/amf.c index fb3ab067..547c6229 100644 --- a/exec/amf.c +++ b/exec/amf.c @@ -154,7 +154,7 @@ #define SYNCTRACE(format, args...) do { \ TRACE6(">%s: " format, __FUNCTION__, ##args); \ -} while(0) +} while (0) /* * The time AMF will wait to get synchronised by another node @@ -463,6 +463,7 @@ struct scsm_descriptor { /* node ID of current sync master */ unsigned int sync_master; + unsigned int *member_list; unsigned int *joined_list; unsigned int joined_list_entries; struct amf_cluster *cluster; @@ -491,9 +492,34 @@ struct scsm_descriptor { static struct scsm_descriptor scsm; /* IMPL */ +static char *hostname_get (unsigned int nodeid) +{ + struct totem_ip_address interfaces[INTERFACE_MAX]; + char **status; + unsigned int iface_count; + int res; + struct hostent *ent; + + res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); + if (res == -1) { + log_printf (LOG_LEVEL_ERROR, "totempg_ifaces_get failed for %u", nodeid); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + if (iface_count > 0) { + ent = gethostbyaddr (interfaces[0].addr, 4, interfaces[0].family); + if (ent == NULL) { + log_printf (LOG_LEVEL_ERROR, "gethostbyaddr failed: %d\n", h_errno); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + + return ent->h_name; + } + + return NULL; +} /** - * Get pointer to this node object. + * Return pointer to this node object. * * @param cluster * @@ -501,24 +527,16 @@ static struct scsm_descriptor scsm; */ static struct amf_node *get_this_node_obj (struct amf_cluster *cluster) { - char hostname[HOST_NAME_MAX + 1]; - struct amf_node *node; + SaClmClusterNodeT *clm_node = main_clm_get_by_nodeid (SA_CLM_LOCAL_NODE_ID); + char *hostname; + SaNameT name; - assert (cluster != NULL); + assert (clm_node != NULL); + hostname = hostname_get (clm_node->nodeId); + assert (hostname != NULL); + setSaNameT (&name, hostname); - if (gethostname (hostname, sizeof(hostname)) == -1) { - log_printf (LOG_LEVEL_ERROR, "gethostname failed: %d", errno); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - /* look for this node */ - for (node = cluster->node_head; node != NULL; node = node->next) { - if (strcmp(hostname, getSaNameT (&node->name)) == 0) { - return node; - } - } - - return NULL; + return amf_node_find (&name); } /** @@ -611,7 +629,8 @@ static void mcast_sync_ready (void) int res; SYNCTRACE ("state %s", scsm_state_names[scsm.state]); - + + req_exec.header.size = sizeof (struct req_exec_amf_sync_data); req_exec.header.id = SERVICE_ID_MAKE (AMF_SERVICE, MESSAGE_REQ_EXEC_AMF_SYNC_READY); @@ -733,40 +752,6 @@ static unsigned int calc_sync_master ( return master; } -/** - * Get hostname from TOTEM node ID - * @param nodeid - * - * @return char* - */ -static char *hostname_get (unsigned int nodeid) -{ - struct totem_ip_address interfaces[INTERFACE_MAX]; - char **status; - unsigned int iface_count; - int res; - struct hostent *ent; - - res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); - if (res == -1) { - log_printf (LOG_LEVEL_ERROR, "totempg_ifaces_get failed"); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - if (iface_count > 0) { - ent = gethostbyaddr (interfaces[0].addr, 4, interfaces[0].family); - if (ent == NULL) { - log_printf (LOG_LEVEL_ERROR, "gethostbyaddr failed: %d\n", h_errno); - openais_exit_error (AIS_DONE_FATAL_ERR); - } - - return ent->h_name; - } else { - assert (0); - } - - return NULL; -} static void free_synced_data (void) { @@ -1141,12 +1126,13 @@ static void joined_nodes_start (void) for (i = 0; i < scsm.joined_list_entries; i++) { SaNameT name; - struct amf_node *synced_node; + struct amf_node *node; setSaNameT (&name, hostname_get (scsm.joined_list[i])); - synced_node = amf_node_find (&name); - if (synced_node != NULL) { - amf_node_sync_ready (synced_node); + node = amf_node_find (&name); + if (node != NULL) { + node->nodeid = scsm.joined_list[i]; + amf_node_sync_ready (node); } else { log_printf (LOG_LEVEL_INFO, "Node %s is not configured as an AMF node", name.value); @@ -1154,6 +1140,26 @@ static void joined_nodes_start (void) } } +static void init_nodeids (void) +{ + int i; + + ENTER (""); + + for (i = 0; scsm.member_list[i] != 0; i++) { + SaNameT name; + struct amf_node *node; + + setSaNameT (&name, hostname_get (scsm.member_list[i])); + node = amf_node_find (&name); + + assert (node != NULL); + node->nodeid = scsm.member_list[i]; + } + + LEAVE (""); +} + /****************************************************************************** * AMF Framework callback implementation * *****************************************************************************/ @@ -1161,7 +1167,6 @@ static void joined_nodes_start (void) static void amf_sync_init (void) { SYNCTRACE ("state %s", scsm_state_names[scsm.state]); - switch (scsm.state) { case UNCONFIGURED: case PROBING_1: @@ -1276,10 +1281,22 @@ static void amf_sync_activate (void) switch (scsm.state) { case SYNCHRONIZING: sync_state_set (NORMAL_OPERATION); - if (amf_cluster->state == CLUSTER_UNINSTANTIATED) { - amf_cluster_start (amf_cluster); - } else { - joined_nodes_start (); + init_nodeids (); + /* TODO: Remove dependencies to amf_cluster->state */ + switch (amf_cluster->state) { + case CLUSTER_STARTED: { + joined_nodes_start (); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + amf_cluster_sync_ready (amf_cluster); + joined_nodes_start (); + break; + } + case CLUSTER_UNINSTANTIATED: + default: { + amf_cluster_sync_ready (amf_cluster); + } } break; case UPDATING_CLUSTER_MODEL: @@ -1288,14 +1305,26 @@ static void amf_sync_activate (void) scsm.cluster = NULL; this_amf_node = get_this_node_obj (amf_cluster); sync_state_set (NORMAL_OPERATION); + init_nodeids (); if (this_amf_node != NULL) { #ifdef AMF_DEBUG amf_runtime_attributes_print (amf_cluster); #endif - if (amf_cluster->state == CLUSTER_UNINSTANTIATED) { - amf_cluster_start (amf_cluster); - } else { - amf_node_sync_ready (this_amf_node); + /* TODO: Remove dependencies to amf_cluster->state */ + switch (amf_cluster->state) { + case CLUSTER_STARTED: { + amf_node_sync_ready (this_amf_node); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + amf_cluster_sync_ready (amf_cluster); + amf_node_sync_ready (this_amf_node); + break; + } + case CLUSTER_UNINSTANTIATED: + default: { + amf_cluster_sync_ready (amf_cluster); + } } } else { log_printf (LOG_LEVEL_INFO, @@ -1377,6 +1406,19 @@ static void amf_confchg_fn ( scsm.joined_list[i] = joined_list[i]; } + /** + * Save current members of the cluster, needed to initialize + * each node's totem node id later. + */ + if (scsm.member_list != NULL) { + free (scsm.member_list); + } + scsm.member_list = amf_malloc ((member_list_entries + 1) * sizeof (unsigned int)); + for (i = 0; i < member_list_entries; i++) { + scsm.member_list[i] = member_list[i]; + } + scsm.member_list[i] = 0; + switch (scsm.state) { case IDLE: { sync_state_set (PROBING_1); @@ -1443,10 +1485,17 @@ static void amf_confchg_fn ( if (scsm.sync_master == this_ip->nodeid) { SYNCTRACE ("I am (new) sync master"); } + } - /* - * TODO: activate standby, ... - */ + if (left_list_entries > 0) { + int i; + struct amf_node *node; + + for (i = 0; i < left_list_entries; i++) { + node = amf_node_find_by_nodeid (left_list[i]); + assert (node != NULL); + amf_node_leave(node); + } } break; } @@ -1456,6 +1505,7 @@ static void amf_confchg_fn ( } } + static int amf_lib_exit_fn (void *conn) { struct amf_comp *comp; diff --git a/exec/amf.h b/exec/amf.h index 656e3f7c..d4774ba2 100644 --- a/exec/amf.h +++ b/exec/amf.h @@ -39,6 +39,9 @@ #define AMF_H_DEFINED #include +#include +#include + #include "../include/saAis.h" #include "../include/saAmf.h" #include "../include/list.h" @@ -54,6 +57,33 @@ enum clc_component_types { clc_component_non_proxied_non_sa_aware = 3 /* non-proxied, non sa aware */ }; +/* + * Node Error Escallation State + */ +typedef enum { + NODE_EESM_IDLE, + NODE_EESM_ESCALLATION_LEVEL_2, + NODE_EESM_ESCALLATION_LEVEL_3 +} amf_node_eesm_state_t; + +typedef enum { + NODE_ACSM_REPAIR_NEEDED, + NODE_ACSM_ESCALLATION_LEVEL_0, + NODE_ACSM_ESCALLATION_LEVEL_2, + NODE_ACSM_ESCALLATION_LEVEL_3, + NODE_ACSM_FAILING_FAST_REBOOTING_NODE, + NODE_ACSM_FAILING_FAST_ACTIVATING_STANDBY_NODE, + NODE_ACSM_FAILING_GRACEFULLY_SWITCHING_OVER, + NODE_ACSM_FAILING_GRACEFULLY_FAILING_OVER, + NODE_ACSM_FAILING_GRACEFULLY_REBOOTING_NODE, + NODE_ACSM_LEAVING_SPONTANEOUSLY_FAILING_OVER, + NODE_ACSM_LEAVING_SPONTANEOUSLY_WAITING_FOR_NODE_TO_JOIN, + NODE_ACSM_JOINING_STARTING_SERVICE_UNITS, + NODE_ACSM_JOINING_ASSIGNING_ACTIVE_WORKLOAD, + NODE_ACSM_JOINING_ASSIGNING_STANDBY_WORKLOAD + +} amf_node_acsm_state_t; + typedef enum { SG_AC_Idle = 0, SG_AC_DeactivatingDependantWorkload, @@ -71,6 +101,11 @@ typedef enum { SG_AC_WaitingAfterOperationFailed } sg_avail_control_state_t; +typedef enum { + SG_RT_FailoverSU = 1, + SG_RT_FailoverNode +} sg_recovery_type_t; + typedef enum { SU_RC_ESCALATION_LEVEL_0 = 0, SU_RC_ESCALATION_LEVEL_1, @@ -114,7 +149,8 @@ struct amf_healthcheck; enum cluster_states { CLUSTER_UNINSTANTIATED, - CLUSTER_STARTING, + CLUSTER_STARTING_COMPONENTS, + CLUSTER_STARTING_WORKLOAD, CLUSTER_STARTED }; @@ -154,7 +190,10 @@ struct amf_node { struct amf_cluster *cluster; /* Implementation */ + unsigned int nodeid; struct amf_node *next; + amf_node_acsm_state_t acsm_state; + int synchronized; }; struct amf_application { @@ -174,6 +213,14 @@ struct amf_application { /* Implementation */ SaStringT clccli_path; struct amf_application *next; + struct amf_node *node_to_start; +}; + +struct sg_recovery_scope { + sg_recovery_type_t recovery_type; + struct amf_si **sis; + struct amf_su **sus; + struct amf_comp *comp; }; struct amf_sg { @@ -208,8 +255,9 @@ struct amf_sg { /* Implementation */ SaStringT clccli_path; - struct amf_sg *next; - sg_avail_control_state_t avail_state; + struct amf_sg *next; + sg_avail_control_state_t avail_state; + struct sg_recovery_scope recovery_scope; }; struct amf_su { @@ -467,6 +515,7 @@ struct req_exec_amf_healthcheck_tmo { SaAmfHealthcheckKeyT safHealthcheckKey; }; + struct req_exec_amf_cluster_start_tmo { mar_req_header_t header; }; @@ -486,7 +535,7 @@ extern const char *amf_presence_state (int state); extern const char *amf_ha_state (int state); extern const char *amf_readiness_state (int state); extern const char *amf_assignment_state (int state); - +extern struct amf_node *amf_node_find_by_nodeid (unsigned int nodeid); extern char *amf_serialize_SaNameT ( char *buf, int *size, int *offset, SaNameT *name); extern char *amf_serialize_SaStringT ( @@ -551,7 +600,7 @@ extern void *amf_cluster_serialize (struct amf_cluster *cluster, int *len); extern struct amf_cluster *amf_cluster_deserialize (char *buf, int size); /* Event methods */ -extern void amf_cluster_start (struct amf_cluster *cluster); +extern void amf_cluster_sync_ready (struct amf_cluster *cluster); extern void amf_cluster_assign_workload (struct amf_cluster *cluster); /* Response event methods */ @@ -637,9 +686,6 @@ extern void amf_sg_su_assignment_removed ( extern void amf_sg_si_activated ( struct amf_sg *sg, struct amf_si *si); -/* Timer event methods */ -//static void timer_function_auto_adjust_tmo (void *sg); - /*===========================================================================*/ /* amfsu.c */ @@ -664,7 +710,8 @@ extern int amf_su_get_saAmfSUNumCurrActiveSIs (struct amf_su *su); extern int amf_su_get_saAmfSUNumCurrStandbySIs (struct amf_su *su); extern SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState ( struct amf_su *su); - +extern int amf_su_presence_state_all_comps_in_su_are_set (struct amf_su *su, + SaAmfPresenceStateT state); /* Event methods */ extern void amf_su_instantiate (struct amf_su *su); extern void amf_su_assign_si ( @@ -684,19 +731,11 @@ extern void amf_su_remove_assignment (struct amf_su *su); /* Response event methods */ extern void amf_su_comp_state_changed ( struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state); -#if 0 -extern void amf_su_comp_hastate_changed ( - struct amf_su *su, struct amf_comp *comp, - struct amf_csi_assignment *csi_assignment); -#endif extern void amf_su_comp_error_suspected ( struct amf_su *su, struct amf_comp *comp, SaAmfRecommendedRecoveryT recommended_recovery); -/* Timer event methods */ -//static void timer_function_su_probation_period_expired(void *data); - /*===========================================================================*/ /* amfcomp.c */ @@ -722,6 +761,7 @@ extern SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState ( /* Event methods */ extern void amf_comp_instantiate (struct amf_comp *comp); extern void amf_comp_terminate (struct amf_comp *comp); +extern void amf_comp_node_left (struct amf_comp *comp); /** * Request the component to assume a HA state @@ -817,6 +857,15 @@ extern struct amf_si_assignment *amf_si_assignment_deserialize ( * @return int */ extern int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si); +/** + * Get number of active assignments for the specified SI and SU + * @param si + * @param su + * + * @return int + */ +extern int amf_si_su_get_saAmfSINumCurrActiveAssignments (struct amf_si *si, + struct amf_su *su); /** * Get number of standby assignments for the specified SI @@ -824,14 +873,25 @@ extern int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si); * * @return int */ + extern int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si); +/** + * Get number of standby assignments for the specified SI and SU + * @param si + * + * @return int + */ +extern int amf_si_su_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si, + struct amf_su *su); + /** * Get assignment state for the specified SI. * @param si * * @return SaAmfAssignmentStateT */ + extern SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState ( struct amf_si *si); @@ -915,12 +975,16 @@ extern char *amf_csi_assignment_dn_make ( struct amf_csi_assignment *csi_assignment, SaNameT *name); extern struct amf_csi_assignment *amf_csi_assignment_find ( struct amf_cluster *cluster, SaNameT *name); - extern struct amf_csi_attribute *amf_csi_attribute_new (struct amf_csi *csi); extern void *amf_csi_attribute_serialize ( struct amf_csi_attribute *csi_attribute, int *len); extern struct amf_csi_attribute *amf_csi_attribute_deserialize ( struct amf_csi *csi, char *buf, int size); +/* extern int sa_amf_grep(const char *string, char *pattern, size_t nmatch, */ +/* char** sub_match_array); */ + +extern int sa_amf_grep(const char *string, char *pattern, size_t nmatch, + SaNameT *sub_match_array); /*===========================================================================*/ extern struct amf_node *this_amf_node; diff --git a/exec/amfapp.c b/exec/amfapp.c index 7b6736ec..3661570d 100644 --- a/exec/amfapp.c +++ b/exec/amfapp.c @@ -75,7 +75,7 @@ #include "amf.h" #include "print.h" -static int all_sg_started(struct amf_application *app) +static int all_sg_started (struct amf_application *app) { struct amf_sg *sg; struct amf_su *su; @@ -102,6 +102,8 @@ void amf_application_start ( ENTER ("'%s'", app->name.value); + app->node_to_start = node; + /* TODO: Calculate and set SI dependency levels */ for (sg = app->sg_head; sg != NULL; sg = sg->next) { @@ -119,6 +121,9 @@ void amf_application_assign_workload ( * Each dependency level should be looped and amf_sg_assign_si * called several times. */ + + app->node_to_start = node; + for (sg = app->sg_head; sg != NULL; sg = sg->next) { amf_sg_assign_si (sg, 0); } @@ -133,9 +138,13 @@ void amf_application_sg_started ( struct amf_application *app, struct amf_sg *sg, struct amf_node *node) { ENTER ("'%s'", app->name.value); - + if (all_sg_started (app)) { - amf_cluster_application_started (app->cluster, app); + if (app->node_to_start == NULL) { + amf_cluster_application_started (app->cluster, app); + } else { + amf_node_application_started (app->node_to_start, app); + } } } @@ -143,7 +152,11 @@ void amf_application_sg_assigned ( struct amf_application *app, struct amf_sg *sg) { ENTER ("'%s'", app->name.value); - amf_cluster_application_workload_assigned (app->cluster, app); + if (app->node_to_start == NULL) { + amf_cluster_application_workload_assigned (app->cluster, app); + } else { + amf_node_application_workload_assigned (app->node_to_start, app); + } } struct amf_application *amf_application_new (struct amf_cluster *cluster) @@ -219,14 +232,15 @@ struct amf_application *amf_application_find ( { struct amf_application *app; - ENTER ("%s", name); - for (app = cluster->application_head; app != NULL; app = app->next) { if (strncmp (name, (char*)app->name.value, app->name.length) == 0) { break; } } + if (app == NULL) { + dprintf ("App %s not found!", name); + } return app; } diff --git a/exec/amfcluster.c b/exec/amfcluster.c index 5e1c6133..9cce4d72 100644 --- a/exec/amfcluster.c +++ b/exec/amfcluster.c @@ -108,7 +108,7 @@ static void timer_function_cluster_assign_workload_tmo (void *_cluster) { struct req_exec_amf_cluster_start_tmo req; struct iovec iovec; - + ENTER (""); req.header.size = sizeof (struct req_exec_amf_cluster_start_tmo); @@ -123,23 +123,50 @@ static void timer_function_cluster_assign_workload_tmo (void *_cluster) } -void amf_cluster_start (struct amf_cluster *cluster) +void amf_cluster_sync_ready (struct amf_cluster *cluster) { struct amf_application *app; log_printf(LOG_NOTICE, "Cluster: starting applications."); - amf_cluster->state = CLUSTER_STARTING; + switch (amf_cluster->state) { + case CLUSTER_UNINSTANTIATED: { + amf_cluster->state = CLUSTER_STARTING_COMPONENTS; + for (app = cluster->application_head; app != NULL; app = app->next) { + amf_application_start (app, NULL); + } + poll_timer_add (aisexec_poll_handle, + cluster->saAmfClusterStartupTimeout, + cluster, + timer_function_cluster_assign_workload_tmo, + &cluster->timeout_handle); - for (app = cluster->application_head; app != NULL; app = app->next) { - amf_application_start (app, NULL); + break; + } + case CLUSTER_STARTING_COMPONENTS: { + if (cluster->timeout_handle) { + + poll_timer_delete ( + aisexec_poll_handle, cluster->timeout_handle); + + cluster->timeout_handle = 0; + } + break; + } + case CLUSTER_STARTING_WORKLOAD: { + log_printf (LOG_LEVEL_ERROR, "Sync ready not implemented in " + "cluster state: %u\n", amf_cluster->state); + assert (0); + break; + } + case CLUSTER_STARTED: { + assert (0); + break; + } + default: + assert (0); } - /* wait a while before assigning workload */ - poll_timer_add (aisexec_poll_handle, cluster->saAmfClusterStartupTimeout, - cluster, - timer_function_cluster_assign_workload_tmo, - &cluster->timeout_handle); } void amf_cluster_init (void) @@ -160,7 +187,7 @@ void amf_cluster_application_started ( poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle); cluster->timeout_handle = 0; } - + cluster->state = CLUSTER_STARTING_WORKLOAD; amf_cluster_assign_workload (cluster); } } @@ -182,7 +209,8 @@ struct amf_cluster *amf_cluster_new (void) void amf_cluster_application_workload_assigned ( struct amf_cluster *cluster, struct amf_application *app) { - log_printf(LOG_NOTICE, "Cluster: all workload assigned."); + log_printf (LOG_NOTICE, "Cluster: application %s assigned.", + app->name.value); amf_cluster->state = CLUSTER_STARTED; } @@ -226,13 +254,15 @@ void amf_cluster_assign_workload (struct amf_cluster *cluster) ENTER (""); + cluster->state = CLUSTER_STARTING_WORKLOAD; + if (cluster->timeout_handle) { poll_timer_delete (aisexec_poll_handle, cluster->timeout_handle); cluster->timeout_handle = 0; } for (app = cluster->application_head; app != NULL; app = app->next) { - amf_application_assign_workload (app, this_amf_node); + amf_application_assign_workload (app, NULL); } } diff --git a/exec/amfcomp.c b/exec/amfcomp.c index c39254de..0c921a43 100644 --- a/exec/amfcomp.c +++ b/exec/amfcomp.c @@ -280,28 +280,10 @@ static int invocation_get_and_destroy ( *interface = invocation_entries[invocation].interface; *data = invocation_entries[invocation].data; memset (&invocation_entries[invocation], 0, sizeof (struct invocation)); - + return (0); } -#if 0 -static int invocation_get ( - SaUint64T invocation, unsigned int *interface, void **data) -{ - if (invocation > invocation_entries_size) { - return (-1); - } - if (invocation_entries[invocation].active == 0) { - return (-1); - } - - *interface = invocation_entries[invocation].interface; - *data = invocation_entries[invocation].data; - - return (0); -} -#endif - static void invocation_destroy_by_data (void *data) { int i; @@ -332,7 +314,7 @@ static void report_error_suspected ( char *amf_comp_dn_make (struct amf_comp *comp, SaNameT *name) { - int i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, + int i = snprintf ((char*) name->value, SA_MAX_NAME_LENGTH, "safComp=%s,safSu=%s,safSg=%s,safApp=%s", comp->name.value, comp->su->name.value, comp->su->sg->name.value, comp->su->sg->application->name.value); @@ -375,16 +357,16 @@ static void *clc_command_run (void *context) waitpid (pid, &status, 0); if (WIFEXITED (status) != 0 && WEXITSTATUS(status) != 0) { fprintf (stderr, "Error: CLC_CLI (%d) failed with exit status:" - " %d - %s\n", pid, WEXITSTATUS(status), - strerror (WEXITSTATUS(status))); + " %d - %s\n", pid, WEXITSTATUS(status), + strerror (WEXITSTATUS(status))); /* - * TODO: remove this and handle properly later... - */ + * TODO: remove this and handle properly later... + */ openais_exit_error (AIS_DONE_FATAL_ERR); } if (WIFSIGNALED (status) != 0) { fprintf (stderr, "Error: CLC_CLI (%d) failed with exit status:" - " %d\n", pid, WTERMSIG(status)); + " %d\n", pid, WTERMSIG(status)); /* * TODO: remove this and handle properly later... */ @@ -419,22 +401,18 @@ static void *clc_command_run (void *context) /* If command is not an absolute path, search for paths in parent objects */ if (cmd[0] != '/') { - if (clc_command_run_data->comp->clccli_path != NULL && - strlen (clc_command_run_data->comp->clccli_path)) { + if (clc_command_run_data->comp->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->clccli_path)) { + clc_command_run_data->comp->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->sg->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->sg->clccli_path)) { + clc_command_run_data->comp->su->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->sg->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->sg->clccli_path, cmd); - } else if (clc_command_run_data->comp->su->sg->application->clccli_path != NULL && - strlen (clc_command_run_data->comp->su->sg->application->clccli_path)) { + clc_command_run_data->comp->su->sg->clccli_path, cmd); + } else if (clc_command_run_data->comp->su->sg->application->clccli_path != NULL) { sprintf (path, "%s/%s", - clc_command_run_data->comp->su->sg->application->clccli_path, cmd); + clc_command_run_data->comp->su->sg->application->clccli_path, cmd); } cmd = path; } @@ -445,13 +423,13 @@ static void *clc_command_run (void *context) { /* make a proper argv array */ i = 1; - char *ptrptr; + char *ptrptr; char *arg = strtok_r(comp_argv, " ", &ptrptr); while (arg) { argv_size++; argv = realloc (argv, sizeof (char*) * argv_size); if (argv == NULL) { - fprintf (stderr, "out-of-memory"); + fprintf (stderr, "out-of-memory"); exit (-1); } argv[i] = arg; @@ -461,20 +439,20 @@ static void *clc_command_run (void *context) } argv[i] = NULL; - i = snprintf(comp_name, SA_MAX_NAME_LENGTH, - "SA_AMF_COMPONENT_NAME=safComp=%s,safSu=%s,safSg=%s,safApp=%s", - clc_command_run_data->comp->name.value, - clc_command_run_data->comp->su->name.value, - clc_command_run_data->comp->su->sg->name.value, - clc_command_run_data->comp->su->sg->application->name.value); + i = snprintf (comp_name, SA_MAX_NAME_LENGTH, + "SA_AMF_COMPONENT_NAME=safComp=%s,safSu=%s,safSg=%s,safApp=%s", + clc_command_run_data->comp->name.value, + clc_command_run_data->comp->su->name.value, + clc_command_run_data->comp->su->sg->name.value, + clc_command_run_data->comp->su->sg->application->name.value); assert (i <= sizeof (comp_name)); - /* two is for component name and NULL termination */ + /* two is for component name and NULL termination */ envp_size = 2; envp = amf_malloc (sizeof (char*) * envp_size); envp[0] = comp_name; for (i = 1; clc_command_run_data->comp->saAmfCompCmdEnv && - clc_command_run_data->comp->saAmfCompCmdEnv[i - 1]; i++) { + clc_command_run_data->comp->saAmfCompCmdEnv[i - 1]; i++) { envp_size++; envp = realloc (envp, sizeof (char*) * envp_size); if (envp == NULL) { @@ -497,9 +475,9 @@ static void *clc_command_run (void *context) res = execve (cmd, argv, envp); if (res == -1) { fprintf (stderr, "Couldn't exec program %s (%s)\n", - cmd, strerror (errno)); + cmd, strerror (errno)); } - exit (res); /* abnormal exit of forked process */ + exit (res); /* abnormal exit of forked process */ return (0); } @@ -528,6 +506,8 @@ static int clc_cli_instantiate (struct amf_comp *comp) if (res != 0) { log_printf (LOG_LEVEL_ERROR, "pthread_create failed: %d", res); } +// clc_command_run_data->completion_callback (clc_command_run_data); + // TODO error code from pthread_create return (res); } @@ -580,7 +560,7 @@ static int lib_comp_terminate_request (struct amf_comp *comp) invocation_create ( AMF_RESPONSE_COMPONENTTERMINATECALLBACK, component_terminate_callback_data); - + openais_conn_send_response ( openais_conn_partner_get (comp->conn), &res_lib, @@ -605,11 +585,8 @@ static void mcast_cleanup_completion_event (void *context) struct req_exec_amf_clc_cleanup_completed req; struct iovec iovec; - TRACE2("CLC cleanup done for '%s'", - clc_command_run_data->comp->name.value); - req.header.size = sizeof (struct req_exec_amf_clc_cleanup_completed); - req.header.id = SERVICE_ID_MAKE (AMF_SERVICE, + req.header.id = SERVICE_ID_MAKE (AMF_SERVICE, MESSAGE_REQ_EXEC_AMF_CLC_CLEANUP_COMPLETED); amf_comp_dn_make (clc_command_run_data->comp, &req.compName); @@ -619,7 +596,7 @@ static void mcast_cleanup_completion_event (void *context) assert (totempg_groups_mcast_joined (openais_group_handle, &iovec, 1, TOTEMPG_AGREED) == 0); } - + /* * Cleanup possible operations */ @@ -684,7 +661,7 @@ struct amf_healthcheck *amf_comp_find_healthcheck ( healthcheck = healthcheck->next) { if (memcmp (key, &healthcheck->safHealthcheckKey, - sizeof (SaAmfHealthcheckKeyT)) == 0) { + sizeof (SaAmfHealthcheckKeyT)) == 0) { ret_healthcheck = healthcheck; break; } @@ -783,6 +760,8 @@ struct amf_comp *amf_comp_find (struct amf_cluster *cluster, SaNameT *name) char *ptrptr; char *buf; + assert (cluster != NULL && name != NULL); + /* malloc new buffer since strtok_r writes to its first argument */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value,name ->length + 1); @@ -815,7 +794,7 @@ struct amf_comp *amf_comp_find (struct amf_cluster *cluster, SaNameT *name) for (su = sg->su_head; su != NULL; su = su->next) { if (strncmp (su_name, (char*)su->name.value, su->name.length) == 0) { for (comp = su->comp_head; comp != NULL; comp = comp->next) { - if (strncmp (comp_name, (char*)comp->name.value, + if (strncmp (comp_name, (char*)comp->name.value, comp->name.length) == 0) { goto end; } @@ -838,8 +817,8 @@ void amf_comp_healthcheck_deactivate (struct amf_comp *comp) ENTER ("'%s'\n", getSaNameT (&comp->name)); for (healthcheck = comp->healthcheck_head; - healthcheck != NULL; - healthcheck = healthcheck->next) { + healthcheck != NULL; + healthcheck = healthcheck->next) { if (healthcheck->active) { healthcheck_deactivate (healthcheck); @@ -847,7 +826,7 @@ void amf_comp_healthcheck_deactivate (struct amf_comp *comp) } } -static void comp_ha_state_set ( struct amf_comp *comp, +static void comp_ha_state_set ( struct amf_comp *comp, struct amf_csi_assignment *csi_assignment, SaAmfHAStateT ha_state) { @@ -901,7 +880,7 @@ static void lib_csi_remove_request (struct amf_comp *comp, &csi->name, sizeof (SaNameT)); res_lib_amf_csiremovecallback.csiFlags = 0; - + openais_conn_send_response ( openais_conn_partner_get (comp->conn), &res_lib_amf_csiremovecallback, @@ -910,7 +889,8 @@ static void lib_csi_remove_request (struct amf_comp *comp, #endif struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( - struct amf_comp *component, const struct amf_csi_assignment *csi_assignment) + struct amf_comp *component, + const struct amf_csi_assignment *csi_assignment) { struct amf_si *si; struct amf_csi *csi; @@ -939,7 +919,7 @@ struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( si = csi->si; tmp_csi_assignment = csi->assigned_csis; } - } else { + } else { csi = tmp_csi_assignment->csi; si = csi->si; } @@ -971,8 +951,8 @@ struct amf_csi_assignment *amf_comp_get_next_csi_assignment ( void amf_comp_foreach_csi_assignment ( struct amf_comp *component, - void (*foreach_fn)(struct amf_comp *component, - struct amf_csi_assignment *csi_assignment)) + void (*foreach_fn) (struct amf_comp *component, + struct amf_csi_assignment *csi_assignment)) { struct amf_csi_assignment *csi_assignment; @@ -986,7 +966,7 @@ void amf_comp_foreach_csi_assignment ( } static struct amf_csi_assignment *csi_assignment_find_in ( - struct amf_comp *component, SaNameT *csi_name) + struct amf_comp *component, SaNameT *csi_name) { struct amf_csi_assignment *csi_assignment; SaNameT dn; @@ -1008,12 +988,12 @@ static void healthcheck_deactivate ( struct amf_healthcheck *healthcheck_active) { dprintf ("deactivating healthcheck for component %s\n", - getSaNameT (&healthcheck_active->comp->name)); + getSaNameT (&healthcheck_active->comp->name)); - poll_timer_delete (aisexec_poll_handle, - healthcheck_active->timer_handle_period); poll_timer_delete (aisexec_poll_handle, - healthcheck_active->timer_handle_duration); + healthcheck_active->timer_handle_period); + poll_timer_delete (aisexec_poll_handle, + healthcheck_active->timer_handle_duration); invocation_destroy_by_data ((void *)healthcheck_active); healthcheck_active->active = 0; @@ -1054,7 +1034,7 @@ static void mcast_healthcheck_tmo_event ( amf_comp_dn_make (healthcheck->comp, &req_exec.compName); memcpy (&req_exec.safHealthcheckKey, - &healthcheck->safHealthcheckKey, sizeof (SaAmfHealthcheckKeyT)); + &healthcheck->safHealthcheckKey, sizeof (SaAmfHealthcheckKeyT)); iovec.iov_base = (char *)&req_exec; iovec.iov_len = sizeof (req_exec); @@ -1083,7 +1063,7 @@ static void lib_healthcheck_request (struct amf_healthcheck *healthcheck) { struct res_lib_amf_healthcheckcallback res_lib; - res_lib.header.id = MESSAGE_RES_AMF_HEALTHCHECKCALLBACK; + res_lib.header.id = MESSAGE_RES_AMF_HEALTHCHECKCALLBACK; res_lib.header.size = sizeof (struct res_lib_amf_healthcheckcallback); res_lib.header.error = SA_AIS_OK; res_lib.invocation = @@ -1094,7 +1074,7 @@ static void lib_healthcheck_request (struct amf_healthcheck *healthcheck) sizeof (SaAmfHealthcheckKeyT)); TRACE7 ("sending healthcheck request to component %s", - res_lib.compName.value); + res_lib.compName.value); openais_conn_send_response ( openais_conn_partner_get (healthcheck->comp->conn), &res_lib, sizeof (struct res_lib_amf_healthcheckcallback)); @@ -1120,8 +1100,9 @@ static void lib_csi_set_request ( csi = csi_assignment->csi; ENTER ("Assigning CSI '%s' state %s to comp '%s'\n", - getSaNameT (&csi->name), amf_ha_state (csi_assignment->requested_ha_state), - comp->name.value); + getSaNameT (&csi->name), + amf_ha_state (csi_assignment->requested_ha_state), + comp->name.value); for (attribute = csi->attributes_head; attribute != NULL; @@ -1134,7 +1115,7 @@ static void lib_csi_set_request ( } } p = amf_malloc(sizeof(struct res_lib_amf_csisetcallback) + - char_length_of_csi_attrs); + char_length_of_csi_attrs); res_lib = (struct res_lib_amf_csisetcallback*)p; /* Address of the buffer containing the Csi name value pair */ @@ -1162,7 +1143,7 @@ static void lib_csi_set_request ( case SA_AMF_HA_ACTIVE: { res_lib->csiStateDescriptor.activeDescriptor.activeCompName.length = 0; res_lib->csiStateDescriptor.activeDescriptor.transitionDescriptor = - SA_AMF_CSI_NEW_ASSIGN; + SA_AMF_CSI_NEW_ASSIGN; break; } case SA_AMF_HA_STANDBY: { @@ -1197,35 +1178,32 @@ static void lib_csi_set_request ( res_lib->invocation = invocation_create (AMF_RESPONSE_CSISETCALLBACK, csi_assignment); openais_conn_send_response ( - openais_conn_partner_get (comp->conn), res_lib, res_lib->header.size); - + openais_conn_partner_get (comp->conn), res_lib, res_lib->header.size); + free(p); } SaAisErrorT amf_comp_register (struct amf_comp *comp) { TRACE2("Exec comp register '%s'", comp->name.value); - + if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING) { comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED); } else if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_INSTANTIATING) { amf_comp_operational_state_set (comp, SA_AMF_OPERATIONAL_ENABLED); comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATED); - } - else { + } else { assert (0); } - + return SA_AIS_OK; } -void amf_comp_error_report ( - struct amf_comp *comp, SaAmfRecommendedRecoveryT recommendedRecovery) +void amf_comp_error_report (struct amf_comp *comp, SaAmfRecommendedRecoveryT recommendedRecovery) { struct res_lib_amf_componenterrorreport res_lib; - TRACE2("Exec comp error report '%s'", comp->name.value); - + if (amf_su_is_local (comp->su)) { res_lib.header.size = sizeof (struct res_lib_amf_componenterrorreport); res_lib.header.id = MESSAGE_RES_AMF_COMPONENTERRORREPORT; @@ -1266,7 +1244,7 @@ void amf_comp_cleanup_completed (struct amf_comp *comp) { TRACE2("Exec CLC cleanup completed for '%s'", comp->name.value); - /* Set all CSI's confirmed HA state to unknown */ + /* Set all CSI's confirmed HA state to unknown */ amf_comp_foreach_csi_assignment (comp, clear_ha_state); /* clear error suspected flag, component is terminated now */ @@ -1301,13 +1279,13 @@ SaAisErrorT amf_comp_healthcheck_start ( healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == 0) { log_printf (LOG_ERR, "Healthcheckstart: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; goto error_exit; } dprintf ("Healthcheckstart: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + comp->name.value, healthcheckKey->key); /* * Determine if this healthcheck is already active @@ -1362,20 +1340,23 @@ SaAisErrorT amf_comp_healthcheck_stop ( struct amf_healthcheck *healthcheck; SaAisErrorT error = SA_AIS_OK; - dprintf ("Healthcheckstop: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + dprintf ("Healthcheckstop: '%s'", comp->name.value); + + if (!amf_su_is_local (comp->su)) { + return SA_AIS_OK; + } if (healthcheckKey == NULL) { for (healthcheck = comp->healthcheck_head; - healthcheck != NULL; - healthcheck = healthcheck->next) { + healthcheck != NULL; + healthcheck = healthcheck->next) { healthcheck_deactivate (healthcheck); } } else { healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == NULL) { log_printf (LOG_ERR, "Healthcheckstop: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; } else { healthcheck_deactivate (healthcheck); @@ -1408,7 +1389,6 @@ void amf_comp_instantiate (struct amf_comp *comp) void amf_comp_readiness_state_set (struct amf_comp *comp, SaAmfReadinessStateT state) { -// comp->saAmfCompReadinessState = state; TRACE1 ("Setting comp '%s' readiness state: %s\n", comp->name.value, amf_readiness_state (state)); } @@ -1440,31 +1420,31 @@ int amf_comp_response_1 ( switch (*interface) { case AMF_RESPONSE_HEALTHCHECKCALLBACK: { - struct amf_healthcheck *healthcheck = data; - SaNameT name; - TRACE7 ("Healthcheck response from '%s': %d", + struct amf_healthcheck *healthcheck = data; + SaNameT name; + TRACE7 ("Healthcheck response from '%s': %d", amf_comp_dn_make (healthcheck->comp, &name), error); - if (healthcheck->invocationType == SA_AMF_HEALTHCHECK_AMF_INVOKED) { + if (healthcheck->invocationType == SA_AMF_HEALTHCHECK_AMF_INVOKED) { /* the response was on time, delete supervision timer */ - poll_timer_delete (aisexec_poll_handle, - healthcheck->timer_handle_duration); - healthcheck->timer_handle_duration = 0; + poll_timer_delete (aisexec_poll_handle, + healthcheck->timer_handle_duration); + healthcheck->timer_handle_duration = 0; /* start timer to execute next healthcheck request */ - poll_timer_add (aisexec_poll_handle, - healthcheck->saAmfHealthcheckPeriod, - (void *)healthcheck, - timer_function_healthcheck_next_fn, - &healthcheck->timer_handle_period); - *retval = SA_AIS_OK; - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; - } + poll_timer_add (aisexec_poll_handle, + healthcheck->saAmfHealthcheckPeriod, + (void *)healthcheck, + timer_function_healthcheck_next_fn, + &healthcheck->timer_handle_period); + *retval = SA_AIS_OK; + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } - return 0; /* do not multicast event */ - break; - } + return 0; /* do not multicast event */ + break; + } case AMF_RESPONSE_CSISETCALLBACK: /* fall-through */ case AMF_RESPONSE_CSIREMOVECALLBACK: amf_csi_assignment_dn_make (data, dn); @@ -1472,15 +1452,15 @@ int amf_comp_response_1 ( break; #if 0 case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { - struct component_terminate_callback_data *component_terminate_callback_data; - component_terminate_callback_data = data; + struct component_terminate_callback_data *component_terminate_callback_data; + component_terminate_callback_data = data; - dprintf ("Lib component terminate callback response, error: %d", error); - amf_comp_healthcheck_deactivate (component_terminate_callback_data->comp); - escalation_policy_restart (component_terminate_callback_data->comp); - return 1; - break; - } + dprintf ("Lib component terminate callback response, error: %d", error); + amf_comp_healthcheck_deactivate (component_terminate_callback_data->comp); + escalation_policy_restart (component_terminate_callback_data->comp); + return 1; + break; + } #endif default: assert (0); @@ -1513,51 +1493,51 @@ struct amf_comp *amf_comp_response_2 ( switch (interface) { case AMF_RESPONSE_CSISETCALLBACK: { - csi_assignment = amf_csi_assignment_find (amf_cluster, dn); - assert (csi_assignment != NULL); - comp = csi_assignment->comp; - dprintf ("CSI '%s' set callback response from '%s', error: %d", - csi_assignment->csi->name.value, - csi_assignment->comp->name.value, error); - comp = csi_assignment->comp; - if (error == SA_AIS_OK) { - comp_ha_state_set ( - comp, csi_assignment, csi_assignment->requested_ha_state); - } else if (error == SA_AIS_ERR_FAILED_OPERATION) { - amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, - csi_assignment); - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; + csi_assignment = amf_csi_assignment_find (amf_cluster, dn); + assert (csi_assignment != NULL); + comp = csi_assignment->comp; + dprintf ("CSI '%s' set callback response from '%s', error: %d", + csi_assignment->csi->name.value, + csi_assignment->comp->name.value, error); + comp = csi_assignment->comp; + if (error == SA_AIS_OK) { + comp_ha_state_set ( + comp, csi_assignment, csi_assignment->requested_ha_state); + } else if (error == SA_AIS_ERR_FAILED_OPERATION) { + amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, + csi_assignment); + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } + break; } - break; - } case AMF_RESPONSE_CSIREMOVECALLBACK: { - csi_assignment = amf_csi_assignment_find (amf_cluster, dn); - assert (csi_assignment != NULL); - dprintf ("Lib csi '%s' remove callback response from '%s', error: %d", - csi_assignment->csi->name.value, - csi_assignment->comp->name.value, error); - comp = csi_assignment->comp; - if (error == SA_AIS_OK) { - comp_ha_state_set (comp, csi_assignment, - csi_assignment->requested_ha_state); - } else if (error == SA_AIS_ERR_FAILED_OPERATION) { - amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, - csi_assignment); - } else { - *retval = SA_AIS_ERR_INVALID_PARAM; + csi_assignment = amf_csi_assignment_find (amf_cluster, dn); + assert (csi_assignment != NULL); + dprintf ("Lib csi '%s' remove callback response from '%s', error: %d", + csi_assignment->csi->name.value, + csi_assignment->comp->name.value, error); + comp = csi_assignment->comp; + if (error == SA_AIS_OK) { + comp_ha_state_set (comp, csi_assignment, + csi_assignment->requested_ha_state); + } else if (error == SA_AIS_ERR_FAILED_OPERATION) { + amf_si_comp_set_ha_state_failed (csi_assignment->csi->si, + csi_assignment); + } else { + *retval = SA_AIS_ERR_INVALID_PARAM; + } + break; } - break; - } #if 0 - case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { - struct component_terminate_callback_data *callback_data = data; - dprintf ("Lib comp '%s' terminate callback response, error: %d", - callback_data->comp->name.value, error); - comp_presence_state_set (callback_data->comp, - SA_AMF_PRESENCE_UNINSTANTIATED); - break; - } + case AMF_RESPONSE_COMPONENTTERMINATECALLBACK: { + struct component_terminate_callback_data *callback_data = data; + dprintf ("Lib comp '%s' terminate callback response, error: %d", + callback_data->comp->name.value, error); + comp_presence_state_set (callback_data->comp, + SA_AMF_PRESENCE_UNINSTANTIATED); + break; + } #endif default: assert (0); @@ -1577,9 +1557,10 @@ void amf_comp_hastate_set ( struct amf_comp *component, struct amf_csi_assignment *csi_assignment) { + ENTER ("'%s'", csi_assignment->csi->name.value); + assert (component != NULL && csi_assignment != NULL); - ENTER ("'%s'", csi_assignment->csi->name.value); if (!component->error_suspected) { lib_csi_set_request(component, csi_assignment); @@ -1590,7 +1571,7 @@ void amf_comp_hastate_set ( assert (0); } } - + LEAVE(""); } @@ -1601,10 +1582,10 @@ void amf_comp_hastate_set ( void amf_comp_terminate (struct amf_comp *comp) { dprintf ("comp terminate '%s'\n", getSaNameT (&comp->name)); - amf_comp_healthcheck_stop (comp, NULL); comp_presence_state_set (comp, SA_AMF_PRESENCE_TERMINATING); if (amf_su_is_local (comp->su)) { + amf_comp_healthcheck_stop (comp, NULL); if (comp->error_suspected) { clc_interfaces[comp->comptype]->cleanup (comp); } else { @@ -1622,9 +1603,9 @@ void amf_comp_restart (struct amf_comp *comp) dprintf ("comp restart '%s'\n", getSaNameT (&comp->name)); comp_presence_state_set (comp, SA_AMF_PRESENCE_RESTARTING); comp->saAmfCompRestartCount += 1; - amf_comp_healthcheck_stop (comp, NULL); if (amf_su_is_local (comp->su)) { + amf_comp_healthcheck_stop (comp, NULL); clc_interfaces[comp->comptype]->cleanup (comp); } } @@ -1645,7 +1626,7 @@ SaAisErrorT amf_comp_hastate_get ( assert (comp != NULL && csi_name != NULL && ha_state != NULL); dprintf ("comp ha state get from comp '%s' CSI '%s'\n", - getSaNameT (&comp->name), csi_name->value); + getSaNameT (&comp->name), csi_name->value); assignment = csi_assignment_find_in (comp, csi_name); if (assignment != NULL) { @@ -1673,12 +1654,12 @@ SaAisErrorT amf_comp_healthcheck_confirm ( SaAisErrorT error = SA_AIS_OK; dprintf ("Healthcheckconfirm: '%s', key '%s'", - comp->name.value, healthcheckKey->key); + comp->name.value, healthcheckKey->key); healthcheck = amf_comp_find_healthcheck (comp, healthcheckKey); if (healthcheck == NULL) { log_printf (LOG_ERR, "Healthcheckstop: Healthcheck '%s' not found", - healthcheckKey->key); + healthcheckKey->key); error = SA_AIS_ERR_NOT_EXIST; } else if (healthcheck->active) { if (healthcheckResult == SA_AIS_OK) { @@ -1767,6 +1748,72 @@ SaAmfReadinessStateT amf_comp_get_saAmfCompReadinessState ( /* XXX we fall here in case NDEBUG is set */ return -1; } +/** + * Component is informed that the node where the 'real' + * component process is executing has unexpectadly left the + * node. If there is a pending interaction between AMF + * (component) and the 'real' component process, then component + * will indicate to its subordinate objects the the interaction + * failed. Pending presence state changes is indicated by + * reporting the new state is uninstantiated while pending csi + * operations are indicated by 'operation failed'. + * @param comp + * + * @return void + */ +void amf_comp_node_left (struct amf_comp *component) +{ + int change_pending = 0; + struct amf_csi_assignment *csi_assignment; + + ENTER(""); + if (component->saAmfCompPresenceState == SA_AMF_PRESENCE_INSTANTIATING || + component->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING || + component->saAmfCompPresenceState == SA_AMF_PRESENCE_TERMINATING) { + change_pending = 1; + } + + component->saAmfCompPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED; + + if (amf_su_presence_state_all_comps_in_su_are_set (component->su, + SA_AMF_PRESENCE_UNINSTANTIATED) != 0) { + component->su->saAmfSUPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED; + } + + if (change_pending) { + change_pending =0; + amf_su_comp_state_changed ( component->su, + component, + SA_AMF_PRESENCE_STATE, + SA_AMF_PRESENCE_UNINSTANTIATED); + } + + if (component->saAmfCompOperState == SA_AMF_OPERATIONAL_ENABLED) { + change_pending = 1; + } + + component->saAmfCompOperState = SA_AMF_OPERATIONAL_DISABLED; + if (change_pending) { + change_pending =0; + amf_su_comp_state_changed ( component->su, + component, + SA_AMF_OP_STATE, + SA_AMF_OPERATIONAL_DISABLED); + } + + csi_assignment = amf_comp_get_next_csi_assignment (component, NULL); + while (csi_assignment != NULL) { + if (csi_assignment->requested_ha_state != + csi_assignment->saAmfCSICompHAState) { + amf_si_comp_set_ha_state_failed ( + csi_assignment->csi->si,csi_assignment); + } + csi_assignment = amf_comp_get_next_csi_assignment ( + component, csi_assignment); + } + + +} /** * Serialize a component including variable length arrays and @@ -1786,13 +1833,13 @@ void *amf_comp_serialize (struct amf_comp *component, int *len) buf = amf_serialize_SaNameT (buf, &size, &offset, &component->name); - /* count cstypes and write to buf */ + /* count cstypes and write to buf */ for (i = 0; component->saAmfCompCsTypes && - component->saAmfCompCsTypes[i] != NULL; i++); + component->saAmfCompCsTypes[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); for (i = 0; component->saAmfCompCsTypes && - component->saAmfCompCsTypes[i] != NULL; i++) { + component->saAmfCompCsTypes[i] != NULL; i++) { buf = amf_serialize_SaNameT ( buf, &size, &offset, component->saAmfCompCsTypes[i]); } @@ -1806,7 +1853,7 @@ void *amf_comp_serialize (struct amf_comp *component, int *len) buf = amf_serialize_SaUint32T ( buf, &size, &offset, component->saAmfCompNumMaxStandbyCsi); - /* count environment vars and write to buf */ + /* count environment vars and write to buf */ for (i = 0; component->saAmfCompCmdEnv[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); diff --git a/exec/amfnode.c b/exec/amfnode.c index 53d47bbc..370d6a76 100644 --- a/exec/amfnode.c +++ b/exec/amfnode.c @@ -113,6 +113,107 @@ #include "amf.h" #include "util.h" #include "print.h" +#include "main.h" + + +static void amf_node_acsm_enter_leaving_spontaneously(struct amf_node *node) +{ + ENTER("'%s'", node->name.value); + node->synchronized = FALSE; + node->saAmfNodeOperState = SA_AMF_OPERATIONAL_DISABLED; +} + +static void amf_node_acsm_enter_failing_over (struct amf_node *node) +{ + struct amf_application *app; + struct amf_sg *sg; + struct amf_su *su; + struct amf_comp *component = NULL; + + ENTER("'%s'", node->name.value); + node->acsm_state = NODE_ACSM_LEAVING_SPONTANEOUSLY_FAILING_OVER; + + for (app = amf_cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + for (su = sg->su_head; su != NULL; su = su->next) { + if (name_match(&node->name, &su->saAmfSUHostedByNode)) { + for (component = su->comp_head; component != NULL; + component = component->next) { + amf_comp_node_left(component); + } + } + } + } + } + + for (app = amf_cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + amf_sg_failover_node_req(sg, node); + } + } +} + +/** + * Node leave event is obtained from amf_confchg_fn + * + * @param node + */ +void amf_node_leave (struct amf_node *node) +{ + ENTER("'%s'", node->name.value); + amf_node_acsm_enter_leaving_spontaneously(node); + amf_node_acsm_enter_failing_over (node); + +} + +/** + * + * @param node + */ +void amf_node_failover (struct amf_node *node) +{ + +} + +/** + * + * @param node + */ +void amf_node_switchover (struct amf_node *node) +{ + +} + +/** + * + * @param node + */ +void amf_node_failfast (struct amf_node *node) +{ + +} + +/** + * + * @param node + * @param comp + */ +void amf_node_comp_restart_req ( + struct amf_node *node, struct amf_comp *comp) +{ + +} + +/** + * + * @param node + * @param comp + */ +void amf_node_comp_failover_req ( + struct amf_node *node, struct amf_comp *comp) +{ + +} /** * Node constructor @@ -120,7 +221,7 @@ * @param cluster * @param node */ -struct amf_node *amf_node_new (struct amf_cluster *cluster, char *name) +struct amf_node *amf_node_new (struct amf_cluster *cluster, char *name) { struct amf_node *node = calloc (1, sizeof (struct amf_node)); @@ -153,7 +254,7 @@ void *amf_node_serialize (struct amf_node *node, int *len) } struct amf_node *amf_node_deserialize ( - struct amf_cluster *cluster, char *buf, int size) + struct amf_cluster *cluster, char *buf, int size) { int objsz = sizeof (struct amf_node); @@ -180,8 +281,8 @@ void amf_node_sync_ready (struct amf_node *node) assert (node != NULL); log_printf(LOG_NOTICE, "Node %s sync ready, starting hosted SUs.", - node->name.value); - + node->name.value); + node->saAmfNodeOperState = SA_AMF_OPERATIONAL_ENABLED; for (app = amf_cluster->application_head; app != NULL; app = app->next) { amf_application_start (app, node); } @@ -196,7 +297,11 @@ struct amf_node *amf_node_find (SaNameT *name) { struct amf_node *node; - ENTER (""); + if (amf_cluster == NULL) { + return NULL; + } + + assert (name != NULL); for (node = amf_cluster->node_head; node != NULL; node = node->next) { if (name_match (&node->name, name)) { @@ -204,6 +309,80 @@ struct amf_node *amf_node_find (SaNameT *name) } } + dprintf ("node %s not found!", name->value); + return NULL; } +struct amf_node *amf_node_find_by_nodeid (unsigned int nodeid) +{ + struct amf_node *node; + + for (node = amf_cluster->node_head; node != NULL; node = node->next) { + if (node->nodeid == nodeid) { + return node; + } + } + + dprintf ("node %u not found!", nodeid); + + return NULL; +} + +static int all_applications_on_node_started (struct amf_node *node, + struct amf_cluster *cluster) +{ + int all_started = 1; + struct amf_application *app; + struct amf_sg *sg; + struct amf_su *su; + + for (app = cluster->application_head; app != NULL; app = app->next) { + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + for (su = sg->su_head; su != NULL; su = su->next) { + if (su->saAmfSUPresenceState != SA_AMF_PRESENCE_INSTANTIATED && + name_match(&su->saAmfSUHostedByNode,&node->name)) { + all_started = 0; + goto done; + } + } + } + } + +done: + return all_started; + +} + +void amf_node_application_started (struct amf_node *node, + struct amf_application *_app) +{ + struct amf_application *app = _app; + + ENTER ("application '%s' started", app->name.value); + + if (all_applications_on_node_started (node, app->cluster)) { + + log_printf(LOG_NOTICE, + "Node: all applications started, assigning workload."); + } + + + for (app = _app->cluster->application_head; app != NULL; + app = app->next) { + amf_application_assign_workload (app, node); + } +} + +void amf_node_application_workload_assigned (struct amf_node *node, + struct amf_application *app) +{ + + log_printf(LOG_NOTICE, "Node: all workload assigned on node %s", + node->name.value); + + /** + * TODO: Set node acsm state + */ +} + diff --git a/exec/amfsg.c b/exec/amfsg.c index 02a4a769..5368f5b7 100644 --- a/exec/amfsg.c +++ b/exec/amfsg.c @@ -148,6 +148,526 @@ #include "main.h" #include "util.h" +static void acsm_enter_activating_standby (struct amf_sg *sg); +static void delete_si_assignments_in_scope (struct amf_sg *sg); + +static void standby_su_activated_cbfn ( + struct amf_si_assignment *si_assignment, int result); + +static void dependent_si_deactivated_cbfn ( + struct amf_si_assignment *si_assignment, int result); + +static const char *sg_recovery_type_text[] = { + "Unknown", + "FailoverSU", + "FailoverNode" +}; + +static void return_to_idle (struct amf_sg *sg) +{ + SaNameT dn; + + ENTER ("sg: %s state: %d", sg->name.value,sg->avail_state); + + sg->avail_state = SG_AC_Idle; + if (sg->recovery_scope.recovery_type != 0) { + switch (sg->recovery_scope.recovery_type) { + case SG_RT_FailoverSU: + case SG_RT_FailoverNode: + assert (sg->recovery_scope.sus[0] != NULL); + amf_su_dn_make (sg->recovery_scope.sus[0], &dn); + log_printf ( + LOG_NOTICE, "'%s' %s recovery action finished", + dn.value, + sg_recovery_type_text[sg->recovery_scope.recovery_type]); + break; + default: + log_printf ( + LOG_NOTICE, "'%s' recovery action finished", + sg_recovery_type_text[0]); + } + } + + if (sg->recovery_scope.sus != NULL) { + free ((void *)sg->recovery_scope.sus); + } + if (sg->recovery_scope.sis != NULL) { + free ((void *)sg->recovery_scope.sis); + } + memset (&sg->recovery_scope, 0, sizeof (struct sg_recovery_scope)); +} + +static int su_instantiated_count (struct amf_sg *sg) +{ + int cnt = 0; + struct amf_su *su; + + for (su = sg->su_head; su != NULL; su = su->next) { + if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) + cnt++; + } + + return cnt; +} + +static int has_any_su_in_scope_active_workload (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + struct amf_si_assignment *si_assignment; + + while (*sus != NULL) { + si_assignment = amf_su_get_next_si_assignment (*sus, NULL); + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { + break; + } + si_assignment = amf_su_get_next_si_assignment ( + *sus, si_assignment); + } + if (si_assignment != NULL) { + break; + } + sus++; + } + return(*sus == NULL); +} + +static int is_standby_for_non_active_si_in_scope (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + + /* + * Check if there is any si in the scope which has no active assignment + * and at least one standby assignment. + */ + while (*sis != NULL) { + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + break; + } + si_assignment = si_assignment->next; + } + if (si_assignment == NULL) { + /* There is no ACTIVE assignment ..*/ + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + break; + } + si_assignment = si_assignment->next; + } + if (si_assignment != NULL) { + /* .. and one STANDBY assignment*/ + break; + } + } + sis++; + } + return(*sis != NULL); +} + +static void acsm_enter_terminating_suspected (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + + sg->avail_state = SG_AC_TerminatingSuspected; + /* + * Terminate suspected SU(s) + */ + while (*sus != 0) { + amf_su_terminate (*sus); + sus++; + } +} + + +/** + * Callback function used by SI when there is no dependent SI to + * deactivate. + * @param sg + */ +static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg) +{ + struct amf_su **sus = sg->recovery_scope.sus; + ENTER("'%s'", sg->name.value); + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. + */ + while (*sus != NULL) { + ENTER("SU %s pr_state='%d'",(*sus)->name.value, + (*sus)->saAmfSUPresenceState); + if (((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_UNINSTANTIATED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_TERMINATION_FAILED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope(sg); + acsm_enter_activating_standby (sg); + } + +} + +static void timer_function_dependent_si_deactivated2 (void *sg) +{ + + ENTER (""); + dependent_si_deactivated_cbfn2 (sg); +} + +static struct amf_si *si_get_dependent (struct amf_si *si) +{ + + struct amf_si *tmp_si = NULL; + ENTER("'%p'",si->depends_on); + if (si->depends_on != NULL) { + + if (si->depends_on->name.length < SA_MAX_NAME_LENGTH) { + si->depends_on->name.value[si->depends_on->name.length] = '\0'; + } + SaNameT res_arr[2]; + int is_match; + + + is_match = sa_amf_grep ((char*)si->depends_on->name.value, + "safDepend=.*,safSi=(.*),safApp=.*", + 2, res_arr); + + if (is_match) { + tmp_si = amf_si_find (si->application, (char*)res_arr[1].value); + } else { + log_printf (LOG_LEVEL_ERROR, "distinguished name for " + "amf_si_depedency failed\n"); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + } + return tmp_si; +} + +struct amf_si *amf_dependent_get_next (struct amf_si *si, + struct amf_si *si_iter) +{ + struct amf_si *tmp_si; + struct amf_application *application; + + ENTER(""); + if (si_iter == NULL) { + assert(amf_cluster != NULL); + application = amf_cluster->application_head; + assert(application != NULL); + tmp_si = application->si_head; + } else { + tmp_si = si_iter->next; + if (tmp_si == NULL) { + application = si->application->next; + if (application == NULL) { + goto out; + } + } + } + + for (; tmp_si != NULL; tmp_si = tmp_si->next) { + struct amf_si *depends_on_si = si_get_dependent (tmp_si); + while (depends_on_si != NULL) { + if (depends_on_si == si) { + goto out; + } + depends_on_si = depends_on_si->next; + } + } + out: + return tmp_si; +} + +static void acsm_enter_deactivating_dependent_workload (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + int callback_pending = 0; + + sg->avail_state = SG_AC_DeactivatingDependantWorkload; + + ENTER("'%s'",sg->name.value); + /* + * For each SI in the recovery scope, find all active assignments + * and request them to be deactivated. + */ + while (*sis != NULL) { + struct amf_si *dependent_si; + struct amf_si *si = *sis; + si_assignment = si->assigned_sis; + dependent_si = amf_dependent_get_next (si, NULL); + + while (dependent_si != NULL) { + si_assignment = dependent_si->assigned_sis; + + while (si_assignment != NULL) { + + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + si_assignment->requested_ha_state = SA_AMF_HA_QUIESCED; + callback_pending = 1; + amf_si_ha_state_assume ( + si_assignment, dependent_si_deactivated_cbfn); + } + si_assignment = si_assignment->next; + } + dependent_si = amf_dependent_get_next (si, dependent_si); + } + sis++; + } + + if (callback_pending == 0) { + poll_timer_handle handle; + ENTER(""); + poll_timer_add (aisexec_poll_handle, 0, sg, + timer_function_dependent_si_deactivated2, &handle); + } +} + +/** + * Enter function for state SG_AC_ActivatingStandby. It activates + * one STANDBY assignment for each SI in the recovery scope. + * @param sg + */ +static void acsm_enter_activating_standby (struct amf_sg *sg) +{ + struct amf_si **sis= sg->recovery_scope.sis; + struct amf_si_assignment *si_assignment; + ENTER("'%s'",sg->name.value); + sg->avail_state = SG_AC_ActivatingStandby; + + /* + * For each SI in the recovery scope, find one standby + * SI assignment and activate it. + */ + while (*sis != NULL) { + si_assignment = (*sis)->assigned_sis; + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + si_assignment->requested_ha_state = SA_AMF_HA_ACTIVE; + amf_si_ha_state_assume ( + si_assignment, standby_su_activated_cbfn); + break; + } + si_assignment = si_assignment->next; + } + sis++; + } +} + +static void acsm_enter_repairing_su (struct amf_sg *sg) +{ + struct amf_su **sus= sg->recovery_scope.sus; + ENTER("'%s'",sg->name.value); + sg->avail_state = SG_AC_ReparingSu; + + /* + * Instantiate SUs in current recovery scope until the configured + * preference is fulfiled. + */ + while (*sus != NULL) { + if (su_instantiated_count ((*sus)->sg) < + (*sus)->sg->saAmfSGNumPrefInserviceSUs) { + struct amf_node *node = amf_node_find(&((*sus)->saAmfSUHostedByNode)); + if (node == NULL) { + log_printf (LOG_LEVEL_ERROR, "no node to hosted on su found" + "amf_si_depedency failed\n"); + openais_exit_error (AIS_DONE_FATAL_ERR); + } + if (node->saAmfNodeOperState == SA_AMF_OPERATIONAL_ENABLED) { + amf_su_instantiate ((*sus)); + } else { + return_to_idle (sg); + } + + } + sus++; + } +} + +/** + * Checks if the si pointed out is already in the scope. + * @param sg + * @param si + */ +static int is_si_in_scope(struct amf_sg *sg, struct amf_si *si) +{ + struct amf_si **tmp_sis= sg->recovery_scope.sis; + + while (*tmp_sis != NULL) { + if (*tmp_sis == si) { + break; + } + tmp_sis++; + } + return(*tmp_sis == si); +} + +/** + * Adds the si pointed out to the scope. + * @param sg + * @param si + */ +static void add_si_to_scope ( struct amf_sg *sg, struct amf_si *si) +{ + int number_of_si = 2; /* It shall be at least two */ + struct amf_si **tmp_sis= sg->recovery_scope.sis; + + ENTER ("'%s'", si->name.value); + + while (*tmp_sis != NULL) { + number_of_si++; + tmp_sis++; + } + + sg->recovery_scope.sis = (struct amf_si **) + realloc((void *)sg->recovery_scope.sis, + sizeof (struct amf_si *)*number_of_si); + assert (sg->recovery_scope.sis != NULL); + + tmp_sis= sg->recovery_scope.sis; + while (*tmp_sis != NULL) { + tmp_sis++; + } + + *tmp_sis= si; +} + +/** + * Adds the ssu pointed out to the scope. + * @param sg + * @param su + */ +static void add_su_to_scope (struct amf_sg *sg, struct amf_su *su) +{ + int number_of_su = 2; /* It shall be at least two */ + struct amf_su **tmp_sus= sg->recovery_scope.sus; + + ENTER ("'%s'", su->name.value); + while (*tmp_sus != NULL) { + number_of_su++; + tmp_sus++; + } + sg->recovery_scope.sus = (struct amf_su **) + realloc((void *)sg->recovery_scope.sus, + sizeof (struct amf_su *)*number_of_su); + assert (sg->recovery_scope.sus != NULL); + + tmp_sus= sg->recovery_scope.sus; + while (*tmp_sus != NULL) { + tmp_sus++; + } + + *tmp_sus= su; +} + +/** + * Set recovery scope for failover SU. + * @param sg + * @param su + */ + +static void set_scope_for_failover_su (struct amf_sg *sg, struct amf_su *su) +{ + struct amf_si_assignment *si_assignment; + struct amf_si **sis; + struct amf_su **sus; + + sg->recovery_scope.recovery_type = SG_RT_FailoverSU; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.sus = (struct amf_su **) + calloc (2, sizeof (struct amf_su *)); + sg->recovery_scope.sis = (struct amf_si **) + calloc (1, sizeof (struct amf_si *)); + + assert ((sg->recovery_scope.sus != NULL) && + (sg->recovery_scope.sis != NULL)); + sg->recovery_scope.sus[0] = su; + + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (is_si_in_scope(sg, si_assignment->si) == 0) { + add_si_to_scope(sg,si_assignment->si ); + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + + sus= sg->recovery_scope.sus; + dprintf("The following sus are within the scope:\n"); + while (*sus != NULL) { + dprintf("%s\n", (*sus)->name.value); + sus++; + } + sis= sg->recovery_scope.sis; + dprintf("The following sis are within the scope:\n"); + while (*sis != NULL) { + dprintf("%s\n", (*sis)->name.value); + sis++; + } +} + +static void set_scope_for_failover_node (struct amf_sg *sg, struct amf_node *node) +{ + struct amf_si_assignment *si_assignment; + struct amf_si **sis; + struct amf_su **sus; + struct amf_su *su; + + ENTER ("'%s'", node->name.value); + sg->recovery_scope.recovery_type = SG_RT_FailoverNode; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.sus = (struct amf_su **) + calloc (1, sizeof (struct amf_su *)); + sg->recovery_scope.sis = (struct amf_si **) + calloc (1, sizeof (struct amf_si *)); + + assert ((sg->recovery_scope.sus != NULL) && + (sg->recovery_scope.sis != NULL)); + for (su = sg->su_head; su != NULL; su = su->next) { + if (name_match (&node->name, &su->saAmfSUHostedByNode)) { + add_su_to_scope (sg, su); + } + } + + sus = sg->recovery_scope.sus; + while (*sus != 0) { + su = *sus; + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (is_si_in_scope(sg, si_assignment->si) == 0) { + add_si_to_scope(sg, si_assignment->si ); + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + sus++; + } + + sus= sg->recovery_scope.sus; + dprintf("The following sus are within the scope:\n"); + while (*sus != NULL) { + dprintf("%s\n", (*sus)->name.value); + sus++; + } + sis = sg->recovery_scope.sis; + dprintf("The following sis are within the scope:\n"); + while (*sis != NULL) { + dprintf("%s\n", (*sis)->name.value); + sis++; + } +} + /** * Delete all SI assignments and all CSI assignments * by requesting all contained components. @@ -158,7 +678,7 @@ static void delete_si_assignments (struct amf_su *su) struct amf_csi *csi; struct amf_si *si; struct amf_si_assignment *si_assignment; - + ENTER ("'%s'", su->name.value); for (si = su->sg->application->si_head; si != NULL; si = si->next) { @@ -178,35 +698,29 @@ static void delete_si_assignments (struct amf_su *su) if (si->assigned_sis->su == su) { si_assignment = si->assigned_sis; si->assigned_sis = si_assignment->next; - dprintf ("first"); } else { si_assignment = si->assigned_sis->next; si->assigned_sis->next = NULL; - dprintf ("second"); } - dprintf ("%p, %d, %d", - si_assignment, si_assignment->name.length, - si->assigned_sis->name.length); assert (si_assignment != NULL); free (si_assignment); } } -static int all_si_has_hastate (struct amf_su *su, SaAmfHAStateT hastate) +/** + * Delete all SI assignments and all CSI assignments in current + * recovery scope. + * @param sg + */ +static void delete_si_assignments_in_scope (struct amf_sg *sg) { - struct amf_si_assignment *si_assignment; - int all_confirmed = 1; + struct amf_su **sus= sg->recovery_scope.sus; - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - if (si_assignment->saAmfSISUHAState != hastate) { - all_confirmed = 0; - break; - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + while (*sus != NULL) { + delete_si_assignments (*sus); + sus++; } - return all_confirmed; } /** @@ -214,62 +728,94 @@ static int all_si_has_hastate (struct amf_su *su, SaAmfHAStateT hastate) * @param si_assignment * @param result */ -static void failover_su_si_deactivated_cbfn ( +static void dependent_si_deactivated_cbfn ( struct amf_si_assignment *si_assignment, int result) { + struct amf_sg *sg = si_assignment->su->sg; + struct amf_su **sus = sg->recovery_scope.sus; + struct amf_su *su; + ENTER ("'%s', %d", si_assignment->si->name.value, result); /* - * If all SI assignments for the SU are quiesced, goto next - * state (TerminatingSuspected). + * If all SI assignments for all SUs in the SG are not pending, + * goto next state (TerminatingSuspected). */ - if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_QUIESCED)) { - si_assignment->su->sg->avail_state = SG_AC_TerminatingSuspected; - /* - * Terminate suspected SU(s) + for (su = sg->su_head ; su != NULL; su = su->next) { + struct amf_si_assignment *si_assignment; + si_assignment = amf_su_get_next_si_assignment(su, NULL); + + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState != + si_assignment->requested_ha_state) { + goto still_wating; + } + si_assignment = amf_su_get_next_si_assignment(su, si_assignment); + } + + + } + + still_wating: + + if (su == NULL) { + sus = si_assignment->su->sg->recovery_scope.sus; + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. */ - amf_su_terminate (si_assignment->su); + while (*sus != NULL) { + if (((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_UNINSTANTIATED) && + ((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_TERMINATION_FAILED) && + ((*sus)->saAmfSUPresenceState != + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + break; + } + sus++; + } + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope(sg); + acsm_enter_activating_standby (sg); + } } + LEAVE(""); } -static int su_instantiated_count (struct amf_sg *sg) -{ - int cnt = 0; - struct amf_su *su; - - for (su = sg->su_head; su != NULL; su = su->next) { - if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) - cnt++; - } - - return cnt; -} static void standby_su_activated_cbfn ( struct amf_si_assignment *si_assignment, int result) { - struct amf_su *su; + struct amf_su **sus= si_assignment->su->sg->recovery_scope.sus; + struct amf_si **sis= si_assignment->su->sg->recovery_scope.sis; ENTER ("'%s', %d", si_assignment->si->name.value, result); - /* - * TODO: create SI assignment to spare and assign them + /* + * If all SI assignments for all SIs in the scope are activated, goto next + * state. */ - si_assignment->su->sg->avail_state = SG_AC_AssigningStandbyToSpare; - si_assignment->su->sg->avail_state = SG_AC_ReparingSu; - - if (all_si_has_hastate (si_assignment->su, SA_AMF_HA_ACTIVE)) { - for (su = si_assignment->su->sg->su_head; su != NULL; su = su->next) { - if ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_UNINSTANTIATED) && - (su_instantiated_count (si_assignment->su->sg) < - si_assignment->su->sg->saAmfSGNumPrefInserviceSUs)) { - - amf_su_instantiate (su); - } + while (*sis != NULL) { + if ((*sis)->assigned_sis != NULL && + (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { + break; } + sis++; + } + + if (*sis == NULL) { + /* + * TODO: create SI assignment to spare and assign them + */ + (*sus)->sg->avail_state = SG_AC_AssigningStandbyToSpare; + + acsm_enter_repairing_su ((*sus)->sg); } } @@ -289,7 +835,7 @@ static void assign_si_assumed_cbfn ( * has been assigned or go back to idle state if not cluster * start. */ - for (si = sg->application->si_head; si != NULL; si = si->next) { + for (si = sg->application->si_head; si != NULL; si = si->next) { if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) { for (tmp_si_assignment = si->assigned_sis; @@ -311,7 +857,7 @@ static void assign_si_assumed_cbfn ( switch (sg->avail_state) { case SG_AC_AssigningOnRequest: if (si_assignment_cnt == confirmed_assignments) { - sg->avail_state = SG_AC_Idle; + return_to_idle (sg); amf_application_sg_assigned (sg->application, sg); } else { dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments); @@ -319,75 +865,31 @@ static void assign_si_assumed_cbfn ( break; case SG_AC_AssigningStandBy: { - SaNameT dn; if (si_assignment_cnt == confirmed_assignments) { - sg->avail_state = SG_AC_Idle; - amf_su_dn_make (si_assignment->su, &dn); - sg->avail_state = SG_AC_Idle; - log_printf ( - LOG_NOTICE, "'%s' failover recovery action finished", - dn.value); + return_to_idle (sg); } break; } default: dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt, - confirmed_assignments); + confirmed_assignments); amf_runtime_attributes_print (amf_cluster); assert (0); } } -/** - * Find an SU assigned with standby workload and activate it. - * @param su - */ -static void standby_su_activate (struct amf_su *su) -{ - struct amf_si_assignment *su_si_assignment; - struct amf_si_assignment *si_assignment; - - ENTER ("Old SU '%s'", su->name.value); - - su->sg->avail_state = SG_AC_ActivatingStandby; - - /* - * For each (active) SI assignment on the old SU, find a standby - * SI assignment and activate it. - */ - su_si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (su_si_assignment != NULL) { - for (si_assignment = su_si_assignment->si->assigned_sis; - si_assignment != NULL; - si_assignment = si_assignment->next) { - - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { - - si_assignment->requested_ha_state = SA_AMF_HA_ACTIVE; - amf_si_ha_state_assume ( - si_assignment, standby_su_activated_cbfn); - break; /* one standby is enough */ - } - } - su_si_assignment = amf_su_get_next_si_assignment (su, su_si_assignment); - } - - delete_si_assignments (su); - - LEAVE (""); -} - static inline int div_round (int a, int b) { int res; - + + assert (b != 0); res = a / b; if ((a % b) != 0) res++; return res; } -static int all_su_has_presence_state( +static int all_su_has_presence_state ( struct amf_sg *sg, SaAmfPresenceStateT state) { struct amf_su *su; @@ -403,6 +905,19 @@ static int all_su_has_presence_state( return all_set; } +static int all_su_in_scope_has_presence_state ( + struct amf_sg *sg, SaAmfPresenceStateT state) +{ + struct amf_su **sus= sg->recovery_scope.sus; + while (*sus != NULL) { + if ((*sus)->saAmfSUPresenceState != state) { + break; + } + sus++; + } + return(*sus == NULL); +} + /** * Get number of SIs protected by the specified SG. * @param sg @@ -419,107 +934,143 @@ static int sg_si_count_get (struct amf_sg *sg) cnt += 1; } } - return (cnt); + return(cnt); } -static void sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) +static int sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) { struct amf_su *su; struct amf_si *si; int assigned = 0; - int assign_per_su = 0; + int assign_to_su = 0; int total_assigned = 0; - int si_cnt; + int si_left; + int si_total; + int su_left_to_assign = su_active_assign; - ENTER ("'%s'", sg->name.value); - - si_cnt = sg_si_count_get (sg); - assign_per_su = div_round (si_cnt, su_active_assign); - if (assign_per_su > sg->saAmfSGMaxActiveSIsperSUs) { - assign_per_su = sg->saAmfSGMaxActiveSIsperSUs; + si_total = sg_si_count_get (sg); + si_left = si_total; + assign_to_su = div_round (si_left, su_active_assign); + if (assign_to_su > sg->saAmfSGMaxActiveSIsperSUs) { + assign_to_su = sg->saAmfSGMaxActiveSIsperSUs; } - si = sg->application->si_head; su = sg->su_head; - while (su != NULL) { + while (su != NULL && su_left_to_assign > 0) { if (amf_su_get_saAmfSUReadinessState (su) != - SA_AMF_READINESS_IN_SERVICE || + SA_AMF_READINESS_IN_SERVICE || amf_su_get_saAmfSUNumCurrActiveSIs (su) == - sg->saAmfSGMaxActiveSIsperSUs || + assign_to_su || amf_su_get_saAmfSUNumCurrStandbySIs (su) > 0) { su = su->next; continue; /* Not in service */ } + si = sg->application->si_head; assigned = 0; - while (si != NULL && - assigned < assign_per_su && - total_assigned < si_cnt) { + assign_to_su = div_round (si_left, su_left_to_assign); + while (si != NULL) { - if (amf_si_get_saAmfSINumCurrActiveAssignments (si) == 0) { - assigned += 1; - total_assigned += 1; - amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE); + if (name_match (&si->saAmfSIProtectedbySG, &sg->name) && + assigned < assign_to_su && + amf_si_get_saAmfSINumCurrActiveAssignments(si) < + su_active_assign) { + + if (amf_si_su_get_saAmfSINumCurrActiveAssignments (si, su) == + 0) { + + assigned += 1; + total_assigned += 1; + amf_su_assign_si (su, si, SA_AMF_HA_ACTIVE); + } } si = si->next; } su = su->next; + su_left_to_assign -= 1; + si_left -= assigned; + dprintf (" su_left_to_assign =%d, si_left=%d\n", + su_left_to_assign, si_left); } - + + assert (total_assigned <= si_total); if (total_assigned == 0) { - dprintf ("Info: No SIs assigned!"); + dprintf ("Info: No SIs assigned"); } + LEAVE(); + + return total_assigned; } -static void sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign) +static int sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign) { struct amf_su *su; struct amf_si *si; int assigned = 0; - int assign_per_su = 0; + int assign_to_su = 0; int total_assigned = 0; - int si_cnt; + int si_left; + int si_total; + int su_left_to_assign = su_standby_assign; ENTER ("'%s'", sg->name.value); if (su_standby_assign == 0) { - return; + return 0; } - si_cnt = sg_si_count_get (sg); - assign_per_su = div_round (si_cnt, su_standby_assign); - if (assign_per_su > sg->saAmfSGMaxStandbySIsperSUs) { - assign_per_su = sg->saAmfSGMaxStandbySIsperSUs; + si_total = sg_si_count_get (sg); + si_left = si_total; + assign_to_su = div_round (si_left, su_standby_assign); + if (assign_to_su > sg->saAmfSGMaxStandbySIsperSUs) { + assign_to_su = sg->saAmfSGMaxStandbySIsperSUs; } - si = sg->application->si_head; su = sg->su_head; - while (su != NULL) { + while (su != NULL && su_left_to_assign > 0) { if (amf_su_get_saAmfSUReadinessState (su) != - SA_AMF_READINESS_IN_SERVICE || + SA_AMF_READINESS_IN_SERVICE || amf_su_get_saAmfSUNumCurrActiveSIs (su) > 0 || amf_su_get_saAmfSUNumCurrStandbySIs (su) == - sg->saAmfSGMaxStandbySIsperSUs) { + assign_to_su) { su = su->next; continue; /* Not available for assignment */ } + si = sg->application->si_head; assigned = 0; - while (si != NULL && assigned < assign_per_su) { + assign_to_su = div_round (si_left, su_left_to_assign); + while (si != NULL) { + if (name_match (&si->saAmfSIProtectedbySG, &sg->name) && + assigned < assign_to_su && + amf_si_get_saAmfSINumCurrStandbyAssignments (si) < + su_standby_assign) { - if (amf_si_get_saAmfSINumCurrStandbyAssignments (si) == 0) { - assigned += 1; - total_assigned += 1; - amf_su_assign_si (su, si, SA_AMF_HA_STANDBY); + if (amf_si_su_get_saAmfSINumCurrStandbyAssignments (si, su) == + 0) { + + assigned += 1; + total_assigned += 1; + amf_su_assign_si (su, si, SA_AMF_HA_STANDBY); + } } si = si->next; } + su_left_to_assign -= 1; + si_left -= assigned; + dprintf (" su_left_to_assign =%d, si_left=%d\n", + su_left_to_assign, si_left); + su = su->next; } + + assert (total_assigned <= si_total); if (total_assigned == 0) { dprintf ("Info: No SIs assigned!"); } + + return total_assigned; } static int su_inservice_count_get (struct amf_sg *sg) @@ -534,29 +1085,27 @@ static int su_inservice_count_get (struct amf_sg *sg) answer += 1; } } - return (answer); + return(answer); } + /** * TODO: dependency_level not used, hard coded * @param sg * @param dependency_level */ -void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) +static void assign_si (struct amf_sg *sg, int dependency_level) { int active_sus_needed; int standby_sus_needed; int inservice_count; - int units_for_standby; - int units_for_active; - int ii_spare; int su_active_assign; int su_standby_assign; int su_spare_assign; + int assigned = 0; ENTER ("'%s'", sg->name.value); - sg->avail_state = SG_AC_AssigningOnRequest; /** * Phase 1: Calculate assignments and create all runtime objects in @@ -569,88 +1118,77 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) */ inservice_count = (float)su_inservice_count_get (sg); - active_sus_needed = div_round (sg_si_count_get (sg), + active_sus_needed = div_round ( + sg_si_count_get (sg) * sg->saAmfSGNumPrefActiveSUs, sg->saAmfSGMaxActiveSIsperSUs); - standby_sus_needed = div_round (sg_si_count_get (sg), + standby_sus_needed = div_round ( + sg_si_count_get (sg) * sg->saAmfSGNumPrefStandbySUs, sg->saAmfSGMaxStandbySIsperSUs); - units_for_active = inservice_count - sg->saAmfSGNumPrefStandbySUs; - if (units_for_active < 0) { - units_for_active = 0; - } + dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)" + "\n", + inservice_count, active_sus_needed, standby_sus_needed); - units_for_standby = inservice_count - sg->saAmfSGNumPrefActiveSUs; - if (units_for_standby < 0) { - units_for_standby = 0; - } - - ii_spare = inservice_count - sg->saAmfSGNumPrefActiveSUs - - sg->saAmfSGNumPrefStandbySUs; - - if (ii_spare < 0) { - ii_spare = 0; - } - - /** - * Determine number of active and standby service units + /* Determine number of active and standby service units * to assign based upon reduction procedure */ - if ((inservice_count - active_sus_needed) < 0) { + if ((inservice_count < active_sus_needed)) { dprintf ("assignment VI - partial assignment with SIs drop outs\n"); - su_active_assign = active_sus_needed; + su_active_assign = inservice_count; su_standby_assign = 0; su_spare_assign = 0; } else - if ((inservice_count - active_sus_needed - standby_sus_needed) < 0) { - dprintf ("assignment V - partial assignment with reduction " - "of standby units\n"); + if ((inservice_count < active_sus_needed + standby_sus_needed)) { + dprintf ("assignment V - partial assignment with reduction of" + " standby units\n"); su_active_assign = active_sus_needed; - if (standby_sus_needed > units_for_standby) { - su_standby_assign = units_for_standby; - } else { - su_standby_assign = standby_sus_needed; - } + su_standby_assign = inservice_count - active_sus_needed; su_spare_assign = 0; } else - if ((sg->saAmfSGMaxStandbySIsperSUs * units_for_standby) <= - sg_si_count_get (sg)) { - - dprintf ("IV: full assignment with reduction of active service units\n"); + if ((inservice_count < sg->saAmfSGNumPrefActiveSUs + standby_sus_needed)) { + dprintf ("IV: full assignment with reduction of active service" + " units\n"); su_active_assign = inservice_count - standby_sus_needed; su_standby_assign = standby_sus_needed; su_spare_assign = 0; - } else - if ((sg->saAmfSGMaxActiveSIsperSUs * units_for_active) <= - sg_si_count_get (sg)) { - - dprintf ("III: full assignment with reduction of standby " - "service units\n"); + } else + if ((inservice_count < + sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) { + dprintf ("III: full assignment with reduction of standby service" + " units\n"); su_active_assign = sg->saAmfSGNumPrefActiveSUs; - su_standby_assign = units_for_standby; + su_standby_assign = inservice_count - sg->saAmfSGNumPrefActiveSUs; su_spare_assign = 0; } else - if (ii_spare == 0) { - dprintf ("II: full assignment with spare reduction\n"); + if ((inservice_count == + sg->saAmfSGNumPrefActiveSUs + sg->saAmfSGNumPrefStandbySUs)) { + if (sg->saAmfSGNumPrefInserviceSUs > inservice_count) { + dprintf ("II: full assignment with spare reduction\n"); + } else { + dprintf ("II: full assignment without spares\n"); + } su_active_assign = sg->saAmfSGNumPrefActiveSUs; su_standby_assign = sg->saAmfSGNumPrefStandbySUs; su_spare_assign = 0; } else { dprintf ("I: full assignment with spares\n"); - su_active_assign = sg->saAmfSGNumPrefActiveSUs; su_standby_assign = sg->saAmfSGNumPrefStandbySUs; - su_spare_assign = ii_spare; + su_spare_assign = inservice_count - + sg->saAmfSGNumPrefActiveSUs - sg->saAmfSGNumPrefStandbySUs; } dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)" - " (assigning spares=%d)\n", + " (assigning spares=%d)\n", inservice_count, su_active_assign, su_standby_assign, su_spare_assign); - sg_assign_nm_active (sg, su_active_assign); - sg_assign_nm_standby (sg, su_standby_assign); + assigned = sg_assign_nm_active (sg, su_active_assign); + assigned += sg_assign_nm_standby (sg, su_standby_assign); + + assert (assigned > 0); sg->saAmfSGNumCurrAssignedSUs = inservice_count; @@ -665,7 +1203,7 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) for (si = sg->application->si_head; si != NULL; si = si->next) { if (name_match (&si->saAmfSIProtectedbySG, &sg->name)) { for (si_assignment = si->assigned_sis; si_assignment != NULL; - si_assignment = si_assignment->next) { + si_assignment = si_assignment->next) { if (si_assignment->requested_ha_state != si_assignment->saAmfSISUHAState) { @@ -680,9 +1218,71 @@ void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) LEAVE ("'%s'", sg->name.value); } +void amf_sg_assign_si (struct amf_sg *sg, int dependency_level) +{ + sg->avail_state = SG_AC_AssigningOnRequest; + assign_si (sg, dependency_level); +} + +void amf_sg_failover_node_req ( + struct amf_sg *sg, struct amf_node *node) +{ + + ENTER("'%s'",node->name.value); + + /* + * TODO: Defer all new events. Workaround is to exit. + */ + if (sg->avail_state != SG_AC_Idle) { + log_printf (LOG_LEVEL_ERROR, "To handle multiple simultaneous SG" + " recovery actions is not implemented yet:" + " SG '%s', NODE '%s', avail_state %d", + sg->name.value, node->name.value, sg->avail_state); + openais_exit_error (AIS_DONE_FATAL_ERR); + return; + } + + set_scope_for_failover_node(sg, node); + + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + struct amf_su **sus = sg->recovery_scope.sus; + + /* Select next state depending on if some SU in the scope is + * needs to be terminated. + */ + while (*sus != NULL) { + ENTER("SU %s pr_state='%d'",(*sus)->name.value, + (*sus)->saAmfSUPresenceState); + if (((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_UNINSTANTIATED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_TERMINATION_FAILED) || + ((*sus)->saAmfSUPresenceState == + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope (sg); + return_to_idle (sg); + } + + } + +} + void amf_sg_start (struct amf_sg *sg, struct amf_node *node) { struct amf_su *su; + sg_avail_control_state_t old_avail_state = sg->avail_state; + int instantiated_sus = 0; ENTER ("'%s'", sg->name.value); @@ -692,41 +1292,63 @@ void amf_sg_start (struct amf_sg *sg, struct amf_node *node) if (node == NULL) { /* Cluster start */ amf_su_instantiate (su); + instantiated_sus++; } else { /* Node start, match if SU is hosted on the specified node */ if (name_match (&node->name, &su->saAmfSUHostedByNode)) { amf_su_instantiate (su); + instantiated_sus++; } } } + + if (instantiated_sus == 0) { + sg->avail_state = old_avail_state; + } } -void amf_sg_su_state_changed ( - struct amf_sg *sg, struct amf_su *su, SaAmfStateT type, int state) +void amf_sg_su_state_changed (struct amf_sg *sg, + struct amf_su *su, SaAmfStateT type, int state) { ENTER ("'%s' SU '%s' state %s", - sg->name.value, su->name.value, amf_presence_state(state)); + sg->name.value, su->name.value, amf_presence_state(state)); if (type == SA_AMF_PRESENCE_STATE) { if (state == SA_AMF_PRESENCE_INSTANTIATED) { - if (all_su_has_presence_state(su->sg, - SA_AMF_PRESENCE_INSTANTIATED)) { - - if (sg->avail_state == SG_AC_InstantiatingServiceUnits) { + if (sg->avail_state == SG_AC_InstantiatingServiceUnits) { + if (all_su_has_presence_state(su->sg, + SA_AMF_PRESENCE_INSTANTIATED)) { su->sg->avail_state = SG_AC_Idle; amf_application_sg_started ( sg->application, sg, this_amf_node); - } else if (sg->avail_state == SG_AC_ReparingSu) { + } + } else if (sg->avail_state == SG_AC_ReparingSu) { + if (all_su_in_scope_has_presence_state(su->sg, + SA_AMF_PRESENCE_INSTANTIATED)) { su->sg->avail_state = SG_AC_AssigningStandBy; - amf_sg_assign_si (sg, 0); + assign_si (sg, 0); + } else { dprintf ("avail-state: %u", sg->avail_state); assert (0); } + } else { + assert (0); } } else if (state == SA_AMF_PRESENCE_UNINSTANTIATED) { if (sg->avail_state == SG_AC_TerminatingSuspected) { - standby_su_activate (su); + if (all_su_in_scope_has_presence_state (sg, state)) { + delete_si_assignments_in_scope (sg); + if (is_standby_for_non_active_si_in_scope (sg)) { + acsm_enter_activating_standby (sg); + } else { + /* + * TODO: create SI assignment to spare and assign them + */ + sg->avail_state = SG_AC_AssigningStandbyToSpare; + acsm_enter_repairing_su (sg); + } + } } else { assert (0); } @@ -746,24 +1368,23 @@ void amf_sg_init (void) void amf_sg_failover_su_req ( struct amf_sg *sg, struct amf_su *su, struct amf_node *node) { - struct amf_si_assignment *si_assignment; - ENTER (""); - - sg->avail_state = SG_AC_DeactivatingDependantWorkload; - - /* - * Deactivate workload for SU - */ - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { - si_assignment->requested_ha_state = SA_AMF_HA_QUIESCED; - amf_si_ha_state_assume ( - si_assignment, failover_su_si_deactivated_cbfn); - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + /* + * TODO: Defer all new events. Workaround is to exit. + */ + if (sg->avail_state != SG_AC_Idle) { + log_printf (LOG_LEVEL_ERROR, "To handle multiple simultaneous SG" + " recovery actions is not implemented yet:" + " SG '%s', SU '%s', avail_state %d", + sg->name.value, su->name.value, sg->avail_state); + openais_exit_error (AIS_DONE_FATAL_ERR); + return; + } + set_scope_for_failover_su (sg, su); + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + acsm_enter_terminating_suspected (sg); } } @@ -907,6 +1528,8 @@ struct amf_sg *amf_sg_find (struct amf_application *app, char *name) { struct amf_sg *sg; + assert (app != NULL && name != NULL); + for (sg = app->sg_head; sg != NULL; sg = sg->next) { if (strncmp (name, (char*)sg->name.value, sg->name.length) == 0) { break; diff --git a/exec/amfsi.c b/exec/amfsi.c index f61974fb..22a2c0bd 100644 --- a/exec/amfsi.c +++ b/exec/amfsi.c @@ -340,7 +340,6 @@ void amf_si_ha_state_assume ( amf_ha_state (si_assignment->requested_ha_state)); si_assignment->assumed_callback_fn = assumed_ha_state_callback_fn; - for (csi = si_assignment->si->csi_head; csi != NULL; csi = csi->next) { for (csi_assignment = csi->assigned_csis; csi_assignment != NULL; csi_assignment = csi_assignment->next) { @@ -379,7 +378,7 @@ void amf_si_ha_state_assume ( if (csi_assignment_cnt == hastate_set_done_cnt) { poll_timer_handle handle; poll_timer_add (aisexec_poll_handle, 0, si_assignment, - timer_function_ha_state_assumed, &handle); + timer_function_ha_state_assumed, &handle); } } @@ -390,6 +389,7 @@ void amf_si_ha_state_assume ( * @return int */ int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si) + { int cnt = 0; struct amf_si_assignment *si_assignment; @@ -405,6 +405,23 @@ int amf_si_get_saAmfSINumCurrActiveAssignments (struct amf_si *si) return cnt; } +int amf_si_su_get_saAmfSINumCurrActiveAssignments (struct amf_si *si, + struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + for (si_assignment = si->assigned_sis; si_assignment != NULL; + si_assignment = si_assignment->next) { + + if (si_assignment->su == su && + si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + cnt++; + } + } + + return cnt; +} + int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si) { int cnt = 0; @@ -421,6 +438,24 @@ int amf_si_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si) return cnt; } +int amf_si_su_get_saAmfSINumCurrStandbyAssignments (struct amf_si *si, + struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + + for (si_assignment = si->assigned_sis; si_assignment != NULL; + si_assignment = si_assignment->next) { + + if (si_assignment->su == su && + si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + cnt++; + } + } + + return cnt; +} + SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (struct amf_si *si) { if ((amf_si_get_saAmfSINumCurrActiveAssignments (si) == @@ -439,24 +474,21 @@ SaAmfAssignmentStateT amf_si_get_saAmfSIAssignmentState (struct amf_si *si) void amf_csi_delete_assignments (struct amf_csi *csi, struct amf_su *su) { struct amf_csi_assignment *csi_assignment; - ENTER ("'%s'", su->name.value); + struct amf_csi_assignment **prev = &csi->assigned_csis; - /* - * TODO: this only works for n+m where each CSI list has only - * two assignments, one active and one standby. - * TODO: use DN instead - */ - if (csi->assigned_csis->comp->su == su) { - csi_assignment = csi->assigned_csis; - csi->assigned_csis = csi_assignment->next; - } else { - csi_assignment = csi->assigned_csis->next; - csi->assigned_csis->next = NULL; - assert (csi_assignment != NULL && csi_assignment->comp->su == su); + + for (csi_assignment = csi->assigned_csis; csi_assignment != NULL; + csi_assignment = csi_assignment->next) { + if (csi_assignment->comp->su == su) { + struct amf_csi_assignment *tmp = csi_assignment; + *prev = csi_assignment->next; + dprintf ("CSI assignment %s unlinked", tmp->name.value); + free (tmp); + } else { + prev = &csi_assignment->next; + } } - assert (csi_assignment != NULL); - free (csi_assignment); } /** @@ -538,7 +570,8 @@ void *amf_si_serialize (struct amf_si *si, int *len) return copy; } -struct amf_si *amf_si_deserialize (struct amf_application *app, char *buf, int size) +struct amf_si *amf_si_deserialize ( + struct amf_application *app, char *buf, int size) { int objsz = sizeof (struct amf_si); @@ -550,7 +583,7 @@ struct amf_si *amf_si_deserialize (struct amf_application *app, char *buf, int s TRACE8 ("%s", si->name.value); memcpy (&si->saAmfSIProtectedbySG, &tmp->saAmfSIProtectedbySG, - sizeof (SaNameT)); + sizeof (SaNameT)); si->saAmfSIRank = tmp->saAmfSIRank; si->saAmfSINumCSIs = tmp->saAmfSINumCSIs; si->saAmfSIPrefActiveAssignments = tmp->saAmfSIPrefActiveAssignments; @@ -568,7 +601,7 @@ struct amf_si_assignment *amf_si_assignment_new (struct amf_si *si) { struct amf_si_assignment *si_assignment; - si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); + si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); si_assignment->si = si; return si_assignment; @@ -614,14 +647,16 @@ struct amf_si *amf_si_find (struct amf_application *app, char *name) { struct amf_si *si; - ENTER ("%s", name); - for (si = app->si_head; si != NULL; si = si->next) { if (strncmp (name, (char*)si->name.value, si->name.length) == 0) { break; } } + if (si == NULL) { + dprintf ("SI %s not found!", name); + } + return si; } @@ -691,14 +726,15 @@ struct amf_csi *amf_csi_find (struct amf_si *si, char *name) { struct amf_csi *csi; - ENTER ("%s", name); - for (csi = si->csi_head; csi != NULL; csi = csi->next) { if (strncmp (name, (char*)csi->name.value, csi->name.length) == 0) { break; } } + if (csi == NULL) { + dprintf ("CSI %s not found!", name); + } return csi; } @@ -730,6 +766,30 @@ void *amf_csi_assignment_serialize ( return copy; } +struct amf_si_assignment *si_assignment_find ( + struct amf_csi_assignment *csi_assignment) +{ + + struct amf_comp *component; + struct amf_si_assignment *si_assignment = NULL; + + component = amf_comp_find(csi_assignment->csi->si->application->cluster, + &csi_assignment->name); + + + for (si_assignment = csi_assignment->csi->si->assigned_sis; + si_assignment != NULL; si_assignment = si_assignment->next) { + SaNameT su_name; + amf_su_dn_make (component->su,&su_name); + + if (name_match(&su_name, &si_assignment->name)) { + break; + } + } + + return si_assignment; +} + struct amf_csi_assignment *amf_csi_assignment_deserialize ( struct amf_csi *csi, char *buf, int size) { @@ -746,8 +806,11 @@ struct amf_csi_assignment *amf_csi_assignment_deserialize ( TRACE8 ("%s", obj->name.value); obj->csi = csi; obj->comp = amf_comp_find (csi->si->application->cluster, &obj->name); + assert (obj->comp != NULL); obj->next = csi->assigned_csis; csi->assigned_csis = obj; + + obj->si_assignment = si_assignment_find(obj); return obj; } } @@ -785,9 +848,7 @@ struct amf_csi_assignment *amf_csi_assignment_find ( char *csi_assignment_name; char *buf; - ENTER ("%s", name->value); - - /* malloc new buffer since we need to write to the buffer */ + /* malloc new buffer since we need to write to the buffer */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value, name->length + 1); @@ -808,7 +869,7 @@ struct amf_csi_assignment *amf_csi_assignment_find ( *(si_name - 1) = '\0'; *(app_name - 1) = '\0'; - /* jump to value */ + /* jump to value */ csi_assignment_name += 11; csi_name += 7; si_name += 6; @@ -867,11 +928,11 @@ void *amf_csi_attribute_serialize ( /* count value and write to buf */ for (i = 0; csi_attribute->value && - csi_attribute->value[i] != NULL; i++); + csi_attribute->value[i] != NULL; i++); buf = amf_serialize_SaUint32T (buf, &size, &offset, i); for (i = 0; csi_attribute->value && - csi_attribute->value[i] != NULL; i++) { + csi_attribute->value[i] != NULL; i++) { buf = amf_serialize_SaStringT ( buf, &size, &offset, csi_attribute->value[i]); } diff --git a/exec/amfsu.c b/exec/amfsu.c index 1595cfe2..89ee88b7 100644 --- a/exec/amfsu.c +++ b/exec/amfsu.c @@ -128,7 +128,7 @@ #include "print.h" #include "main.h" -static int presence_state_all_comps_in_su_are_set (struct amf_su *su, +int amf_su_presence_state_all_comps_in_su_are_set (struct amf_su *su, SaAmfPresenceStateT state) { int all_set = 1; @@ -219,8 +219,8 @@ static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi, csi_assignment->next = csi->assigned_csis; csi->assigned_csis = csi_assignment; amf_comp_dn_make (comp, &csi_assignment->name); - csi_assignment->csi = csi; csi_assignment->comp = comp; + csi_assignment->csi = csi; csi_assignment->saAmfCSICompHAState = 0; /* undefined confirmed HA state */ csi_assignment->requested_ha_state = ha_state; csi_assignment->si_assignment = si_assignment; @@ -235,8 +235,8 @@ static void su_restart (struct amf_su *su) amf_su_dn_make (su, &dn); log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tSU restart", dn.value); - + "action:\n\t\tSU restart", dn.value); + su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING; su->restart_control_state = SU_RC_RESTART_SU_INSTANTIATING; su->escalation_level_history_state = @@ -256,7 +256,7 @@ static void comp_restart (struct amf_comp *comp) ENTER ("'%s'", comp->name.value); amf_comp_dn_make (comp, &dn); log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tcomponent restart", dn.value); + "action:\n\t\tcomponent restart", dn.value); comp->su->restart_control_state = SU_RC_RESTART_COMP_DEACTIVATING; comp->su->restart_control_state = SU_RC_RESTART_COMP_RESTARTING; @@ -311,7 +311,7 @@ void amf_su_assign_si (struct amf_su *su, struct amf_si *si, for (csi = si->csi_head; csi != NULL; csi = csi->next) { if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value, - cs_type->length)) { + cs_type->length)) { comp_assign_csi (comp, csi, si_assignment, ha_state); no_of_assignments++; } @@ -406,7 +406,7 @@ static void su_comp_presence_state_changed ( * TODO: send to node */ case SU_RC_ESCALATION_LEVEL_0: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( comp->su, SA_AMF_PRESENCE_INSTANTIATED)) { su_presence_state_set ( @@ -418,7 +418,7 @@ static void su_comp_presence_state_changed ( reassign_sis (comp->su); break; case SU_RC_RESTART_SU_INSTANTIATING: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( comp->su, SA_AMF_PRESENCE_INSTANTIATED)) { su->restart_control_state = SU_RC_RESTART_SU_SETTING; @@ -433,7 +433,7 @@ static void su_comp_presence_state_changed ( } break; case SA_AMF_PRESENCE_UNINSTANTIATED: - if (presence_state_all_comps_in_su_are_set ( + if (amf_su_presence_state_all_comps_in_su_are_set ( su, SA_AMF_PRESENCE_UNINSTANTIATED)) { su_presence_state_set (comp->su, @@ -458,25 +458,25 @@ static void su_comp_op_state_changed ( switch (state) { case SA_AMF_OPERATIONAL_ENABLED: - { - struct amf_comp *comp_compare; - int all_set = 1; - for (comp_compare = comp->su->comp_head; - comp_compare != NULL; comp_compare = comp_compare->next) { - if (comp_compare->saAmfCompOperState != + { + struct amf_comp *comp_compare; + int all_set = 1; + for (comp_compare = comp->su->comp_head; + comp_compare != NULL; comp_compare = comp_compare->next) { + if (comp_compare->saAmfCompOperState != SA_AMF_OPERATIONAL_ENABLED) { - all_set = 0; - break; + all_set = 0; + break; + } } + if (all_set) { + su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_ENABLED); + } else { + su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED); + } + break; } - if (all_set) { - su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_ENABLED); - } else { - su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED); - } - break; - } case SA_AMF_OPERATIONAL_DISABLED: break; default: @@ -534,17 +534,9 @@ void amf_su_comp_error_suspected ( { ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value); - /* - * Defer all new events. Workaround to be able to use gdb. - */ - if (su->sg->avail_state != SG_AC_Idle) { - ENTER ("Comp '%s', SU '%s'", comp->name.value, su->name.value); - fprintf (stderr, "Warning Debug: event deferred!\n"); - return; - } - switch (su->restart_control_state) { case SU_RC_ESCALATION_LEVEL_0: + if (comp->saAmfCompRestartCount >= su->sg->saAmfSGCompRestartMax) { su->restart_control_state = SU_RC_ESCALATION_LEVEL_1; amf_su_comp_error_suspected (su, comp, recommended_recovery); @@ -572,29 +564,23 @@ void amf_su_comp_error_suspected ( /* * TODO: delegate to node */ - struct amf_si_assignment *si_assignment = - amf_su_get_next_si_assignment (su, NULL); - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { - SaNameT dn; + SaNameT dn; - su_operational_state_set (su, SA_AMF_OPERATIONAL_DISABLED); - amf_comp_operational_state_set ( - comp, SA_AMF_OPERATIONAL_DISABLED); - amf_comp_dn_make (comp, &dn); - log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action:\n\t\tSU failover", dn.value); - amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node); - return; - } else { - su_restart (comp->su); - } + su_operational_state_set (su, SA_AMF_OPERATIONAL_DISABLED); + amf_comp_operational_state_set ( + comp, SA_AMF_OPERATIONAL_DISABLED); + amf_comp_dn_make (comp, &dn); + log_printf (LOG_NOTICE, "Error detected for '%s', recovery " + "action:\n\t\tSU failover", dn.value); + amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node); + return; } else { su_restart (comp->su); } break; default: - assert (0); + break; } } @@ -620,9 +606,13 @@ void amf_su_terminate (struct amf_su *su) char *amf_su_dn_make (struct amf_su *su, SaNameT *name) { - int i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, + int i; + + assert (su != NULL); + + i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, "safSu=%s,safSg=%s,safApp=%s", - su->name.value, su->sg->name.value, su->sg->application->name.value); + su->name.value, su->sg->name.value, su->sg->application->name.value); assert (i <= SA_MAX_NAME_LENGTH); name->length = i; return (char *)name->value; @@ -676,7 +666,7 @@ struct amf_si_assignment *amf_su_get_next_si_assignment ( void amf_su_foreach_si_assignment ( struct amf_su *su, void (*foreach_fn)(struct amf_su *su, - struct amf_si_assignment *si_assignment)) + struct amf_si_assignment *si_assignment)) { struct amf_si_assignment *si_assignment; @@ -715,7 +705,7 @@ int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su) { int cnt = 0; struct amf_si_assignment *si_assignment; - + si_assignment = amf_su_get_next_si_assignment (su, NULL); while (si_assignment != NULL) { if (su->sg->avail_state == SG_AC_AssigningOnRequest && @@ -736,7 +726,7 @@ SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su) { if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) && ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) || - (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { + (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { return SA_AMF_READINESS_IN_SERVICE; } else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) { @@ -773,7 +763,7 @@ struct amf_su *amf_su_new (struct amf_sg *sg, char *name) } if (tail == NULL) { - sg->su_head = su; + sg->su_head = su; } else { tail->next = su; } @@ -883,6 +873,8 @@ struct amf_su *amf_su_find (struct amf_cluster *cluster, SaNameT *name) char *ptrptr; char *buf; + assert (cluster != NULL && name != NULL); + /* malloc new buffer since strtok_r writes to its first argument */ buf = amf_malloc (name->length + 1); memcpy (buf, name->value, name->length + 1); diff --git a/exec/amfutil.c b/exec/amfutil.c index 655caf0f..82a9f8d5 100644 --- a/exec/amfutil.c +++ b/exec/amfutil.c @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include "../include/saAis.h" #include "../include/saAmf.h" @@ -113,21 +115,21 @@ static const char *assignment_state_text[] = { static int init_category (struct amf_comp *comp, char *loc) { - if (strcmp (loc, "sa_aware") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_SA_AWARE; - } else if (strcmp (loc, "proxy") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_PROXY; - } else if (strcmp (loc, "proxied") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_PROXIED; - } else if (strcmp (loc, "local") == 0) { - comp->saAmfCompCategory = SA_AMF_COMP_LOCAL; - } else { - return -1; - } - - return 0; -} - + if (strcmp (loc, "sa_aware") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_SA_AWARE; + } else if (strcmp (loc, "proxy") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_PROXY; + } else if (strcmp (loc, "proxied") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_PROXIED; + } else if (strcmp (loc, "local") == 0) { + comp->saAmfCompCategory = SA_AMF_COMP_LOCAL; + } else { + return -1; + } + + return 0; +} + static int init_capability (struct amf_comp *comp, char *loc) { if (strcmp (loc, "x_active_and_y_standby") == 0) { @@ -147,10 +149,10 @@ static int init_capability (struct amf_comp *comp, char *loc) } else { return -1; } - + return 0; } - + static int init_recovery_on_error (struct amf_comp *comp, char *loc) { if (strcmp (loc, "component_restart") == 0) { @@ -258,8 +260,7 @@ struct amf_cluster *amf_config_read (char **error_string) int csi_dependencies_cnt = 0; char *error_reason = NULL; char *value; - - filename = getenv("OPENAIS_AMF_CONFIG_FILE"); + filename = getenv ("OPENAIS_AMF_CONFIG_FILE"); if (!filename) { filename = "/etc/ais/amf.conf"; } @@ -267,7 +268,7 @@ struct amf_cluster *amf_config_read (char **error_string) fp = fopen (filename, "r"); if (fp == 0) { sprintf (buf, "Can't read %s file reason = (%s).\n", - filename, strerror (errno)); + filename, strerror (errno)); *error_string = buf; return NULL; } @@ -395,6 +396,10 @@ struct amf_cluster *amf_config_read (char **error_string) } else if ((loc = strstr_rs (line, "safSg=")) != 0) { sg = amf_sg_new (app, trim_str (loc)); sg_cnt++; + sg->recovery_scope.comp = NULL; + sg->recovery_scope.recovery_type = 0; + sg->recovery_scope.sis = NULL; + sg->recovery_scope.sus = NULL; current_parse = AMF_SG; su_cnt = 0; } else if ((loc = strstr_rs (line, "safSi=")) != 0) { @@ -406,7 +411,7 @@ struct amf_cluster *amf_config_read (char **error_string) if (sg_cnt == 1) { for (si = app->si_head; si != NULL; si = si->next) { memcpy (&si->saAmfSIProtectedbySG, &sg->name, - sizeof (SaNameT)); + sizeof (SaNameT)); } } else { for (si = app->si_head; si != NULL; si = si->next) { @@ -689,7 +694,7 @@ struct amf_cluster *amf_config_read (char **error_string) } else { comp_cs_type_cnt++; comp->saAmfCompCsTypes = realloc (comp->saAmfCompCsTypes, - (comp_cs_type_cnt + 1) * sizeof(SaNameT)); + (comp_cs_type_cnt + 1) * sizeof(SaNameT)); comp->saAmfCompCsTypes[comp_cs_type_cnt] = NULL; comp->saAmfCompCsTypes[comp_cs_type_cnt - 1] = amf_malloc (sizeof(SaNameT)); setSaNameT (comp->saAmfCompCsTypes[comp_cs_type_cnt - 1], line); @@ -702,7 +707,7 @@ struct amf_cluster *amf_config_read (char **error_string) } else if ((loc = strchr (line, '=')) != 0) { comp_env_var_cnt++; comp->saAmfCompCmdEnv = realloc (comp->saAmfCompCmdEnv, - (comp_env_var_cnt + 1) * sizeof(SaStringT)); + (comp_env_var_cnt + 1) * sizeof(SaStringT)); comp->saAmfCompCmdEnv[comp_env_var_cnt] = NULL; env_var = comp->saAmfCompCmdEnv[comp_env_var_cnt - 1] = amf_malloc (strlen (line) + 1); strcpy (env_var, line); @@ -744,9 +749,9 @@ struct amf_cluster *amf_config_read (char **error_string) csi->si = si; setSaNameT (&csi->name, trim_str (loc)); current_parse = AMF_CSI; - } else if ((loc = strstr_rs (line, "saAmfSIProtectedbySG{")) != 0) { + } else if ((loc = strstr_rs (line, "saAmfSIProtectedbySG=")) != 0) { setSaNameT (&si->saAmfSIProtectedbySG, loc); - } else if ((loc = strstr_rs (line, "saAmfSIRank{")) != 0) { + } else if ((loc = strstr_rs (line, "saAmfSIRank=")) != 0) { si->saAmfSIRank = atoi (loc); } else if ((loc = strstr_rs (line, "saAmfSINumCSIs=")) != 0) { si->saAmfSINumCSIs = atoi (loc); @@ -860,7 +865,7 @@ struct amf_cluster *amf_config_read (char **error_string) parse_error: sprintf (buf, "parse error at %s: %d: %s\n", - filename, line_number, error_reason); + filename, line_number, error_reason); *error_string = buf; fclose (fp); return NULL; @@ -879,7 +884,7 @@ static void print_si_assignment (struct amf_su *su, { log_printf (LOG_INFO, " safSi=%s\n", si_assignment->si->name.value); log_printf (LOG_INFO, " HA state: %s\n", - ha_state_text[si_assignment->saAmfSISUHAState]); + ha_state_text[si_assignment->saAmfSISUHAState]); } void amf_runtime_attributes_print (struct amf_cluster *cluster) @@ -896,49 +901,49 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) log_printf (LOG_INFO, "==================================================="); log_printf (LOG_INFO, "safCluster=%s", getSaNameT(&cluster->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[cluster->saAmfClusterAdminState]); + admin_state_text[cluster->saAmfClusterAdminState]); for (node = cluster->node_head; node != NULL; node = node->next) { log_printf (LOG_INFO, " safNode=%s\n", getSaNameT (&node->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[node->saAmfNodeAdminState]); + admin_state_text[node->saAmfNodeAdminState]); log_printf (LOG_INFO, " oper state: %s\n", - oper_state_text[node->saAmfNodeOperState]); + oper_state_text[node->saAmfNodeOperState]); } for (app = cluster->application_head; app != NULL; app = app->next) { log_printf (LOG_INFO, " safApp=%s\n", getSaNameT(&app->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[app->saAmfApplicationAdminState]); + admin_state_text[app->saAmfApplicationAdminState]); log_printf (LOG_INFO, " num_sg: %d\n", app->saAmfApplicationCurrNumSG); - for (sg = app->sg_head; sg != NULL; sg = sg->next) { - log_printf (LOG_INFO, " safSG=%s\n", getSaNameT(&sg->name)); + for (sg = app->sg_head; sg != NULL; sg = sg->next) { + log_printf (LOG_INFO, " safSg=%s\n", getSaNameT(&sg->name)); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[sg->saAmfSGAdminState]); + admin_state_text[sg->saAmfSGAdminState]); log_printf (LOG_INFO, " assigned SUs %d\n", - sg->saAmfSGNumCurrAssignedSUs); + sg->saAmfSGNumCurrAssignedSUs); log_printf (LOG_INFO, " non inst. spare SUs %d\n", - sg->saAmfSGNumCurrNonInstantiatedSpareSUs); + sg->saAmfSGNumCurrNonInstantiatedSpareSUs); log_printf (LOG_INFO, " inst. spare SUs %d\n", - sg->saAmfSGNumCurrInstantiatedSpareSUs); + sg->saAmfSGNumCurrInstantiatedSpareSUs); for (su = sg->su_head; su != NULL; su = su->next) { log_printf (LOG_INFO, " safSU=%s\n", getSaNameT(&su->name)); log_printf (LOG_INFO, " oper state: %s\n", - oper_state_text[su->saAmfSUOperState]); + oper_state_text[su->saAmfSUOperState]); log_printf (LOG_INFO, " admin state: %s\n", - admin_state_text[su->saAmfSUAdminState]); + admin_state_text[su->saAmfSUAdminState]); log_printf (LOG_INFO, " readiness state: %s\n", readiness_state_text[amf_su_get_saAmfSUReadinessState (su)]); log_printf (LOG_INFO, " presence state: %s\n", - presence_state_text[su->saAmfSUPresenceState]); + presence_state_text[su->saAmfSUPresenceState]); log_printf (LOG_INFO, " hosted by node %s\n", - su->saAmfSUHostedByNode.value); + su->saAmfSUHostedByNode.value); log_printf (LOG_INFO, " num active SIs %d\n", - amf_su_get_saAmfSUNumCurrActiveSIs (su)); + amf_su_get_saAmfSUNumCurrActiveSIs (su)); log_printf (LOG_INFO, " num standby SIs %d\n", - amf_su_get_saAmfSUNumCurrStandbySIs (su)); + amf_su_get_saAmfSUNumCurrStandbySIs (su)); log_printf (LOG_INFO, " restart count %d\n", - su->saAmfSURestartCount); + su->saAmfSURestartCount); log_printf (LOG_INFO, " restart control state %d\n", - su->restart_control_state); + su->restart_control_state); log_printf (LOG_INFO, " SU failover cnt %d\n", su->su_failover_cnt); log_printf (LOG_INFO, " assigned SIs:"); amf_su_foreach_si_assignment (su, print_si_assignment); @@ -956,7 +961,7 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) log_printf (LOG_INFO, " num standby CSIs %d\n", amf_comp_get_saAmfCompNumCurrStandbyCsi (comp)); log_printf (LOG_INFO, " restart count %d\n", - comp->saAmfCompRestartCount); + comp->saAmfCompRestartCount); log_printf (LOG_INFO, " assigned CSIs:"); amf_comp_foreach_csi_assignment ( comp, print_csi_assignment); @@ -969,11 +974,11 @@ void amf_runtime_attributes_print (struct amf_cluster *cluster) admin_state_text[si->saAmfSIAdminState]); log_printf (LOG_INFO, " assignm. state: %s\n", assignment_state_text[ - amf_si_get_saAmfSIAssignmentState (si)]); + amf_si_get_saAmfSIAssignmentState (si)]); log_printf (LOG_INFO, " active assignments: %d\n", - amf_si_get_saAmfSINumCurrActiveAssignments (si)); + amf_si_get_saAmfSINumCurrActiveAssignments (si)); log_printf (LOG_INFO, " standby assignments: %d\n", - amf_si_get_saAmfSINumCurrStandbyAssignments (si)); + amf_si_get_saAmfSINumCurrStandbyAssignments (si)); for (csi = si->csi_head; csi != NULL; csi = csi->next) { log_printf (LOG_INFO, " safCsi=%s\n", getSaNameT(&csi->name)); } @@ -991,22 +996,22 @@ int amf_enabled (struct objdb_iface_ver0 *objdb) objdb->object_find_reset (OBJECT_PARENT_HANDLE); if (objdb->object_find ( - OBJECT_PARENT_HANDLE, - "amf", - strlen ("amf"), - &object_service_handle) == 0) { + OBJECT_PARENT_HANDLE, + "amf", + strlen ("amf"), + &object_service_handle) == 0) { value = NULL; - if ( !objdb->object_key_get (object_service_handle, - "mode", - strlen ("mode"), - (void *)&value, - NULL) && value) { + if (!objdb->object_key_get (object_service_handle, + "mode", + strlen ("mode"), + (void *)&value, + NULL) && value) { if (strcmp (value, "enabled") == 0) { enabled = 1; } else - if (strcmp (value, "disabled") == 0) { + if (strcmp (value, "disabled") == 0) { enabled = 0; } } @@ -1146,7 +1151,7 @@ char *amf_deserialize_SaStringT (char *buf, SaStringT *str) memcpy (tmp_str, tmp, len); tmp_str[len] = '\0'; *str = tmp_str; - } else { + } else { *str = NULL; } @@ -1185,3 +1190,86 @@ void *_amf_malloc (size_t size, char *file, unsigned int line) return tmp; } + +int sa_amf_grep_one_sub_match(const char *string, char *pattern, + SaNameT *matches_arr) +{ + int status; + regex_t re; + size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + + int i; + + ENTER("'%s %s'",string, pattern); + + if (regcomp(&re, pattern, REG_EXTENDED) != 0) { + status = 0; + goto out; + } + status = regexec(&re, string, nmatch, pmatch, 0); + if (status != 0) { + regfree(&re); + status = 0; + goto out; + } else { + + + for (i = 0; i < nmatch; i++) { + int sub_string_len; + sub_string_len = pmatch[i].rm_eo - pmatch[i].rm_so; + if (i==1) { + + memcpy(matches_arr[i].value, string + pmatch[i].rm_so, + sub_string_len); + matches_arr[i].value[sub_string_len] = '\0'; + } + } + status = 1; + regfree(&re); + } + out: + return status; + +} + + +int sa_amf_grep(const char *string, char *pattern, size_t nmatch, + SaNameT *matches_arr) +{ + int status; + regex_t re; + regmatch_t pmatch[nmatch]; + + int i; + + ENTER("'%s %s'",string, pattern); + if (regcomp(&re, pattern, REG_EXTENDED) != 0) { + status = 0; + goto out; + } + status = regexec(&re, string, nmatch, pmatch, 0); + if (status != 0) { + regfree(&re); + status = 0; + goto out; + } else { + + + for (i = 0; i < nmatch; i++) { + int sub_string_len; + sub_string_len = pmatch[i].rm_eo - pmatch[i].rm_so; + memcpy(matches_arr[i].value, string + pmatch[i].rm_so, + sub_string_len); + matches_arr[i].value[sub_string_len] = '\0'; + matches_arr[i].length = sub_string_len; + } + status = 1; + regfree(&re); + } +out: + return status; + +} + +