From 77668738f6f1fa402e7bd47abf760a4e2d7468c3 Mon Sep 17 00:00:00 2001 From: Lon Hohberger Date: Thu, 18 Jan 2007 07:39:42 +0000 Subject: [PATCH] - improves the inclosed dokumentation in amfsu,amfsg and amnode. - improves error handling caused by the INSTANTIATE or CLEANUP command while recovering with component_restart or su_restart git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1347 fd59a12c-fef9-0310-b244-a6a79926bd2f --- exec/amf.c | 3 +- exec/amf.h | 8 +- exec/amfcomp.c | 16 +- exec/amfnode.c | 13 +- exec/amfsg.c | 1071 ++++++++++++++++++++++++-------------- exec/amfsu.c | 1346 ++++++++++++++++++++++++++++++------------------ 6 files changed, 1557 insertions(+), 900 deletions(-) diff --git a/exec/amf.c b/exec/amf.c index 871237cd..af2a63ca 100644 --- a/exec/amf.c +++ b/exec/amf.c @@ -1575,12 +1575,13 @@ static void message_handler_req_exec_amf_clc_cleanup_completed ( { struct req_exec_amf_clc_cleanup_completed *req_exec = message; amf_comp_t *comp; - ENTER (""); if (scsm.state != NORMAL_OPERATION) { return; } comp = amf_comp_find (amf_cluster, &req_exec->compName); + ENTER ("%s",comp->name.value); + if (comp == NULL) { log_printf (LOG_ERR, "Error: '%s' not found", req_exec->compName.value); return; diff --git a/exec/amf.h b/exec/amf.h index b526bd6c..585d8a25 100644 --- a/exec/amf.h +++ b/exec/amf.h @@ -145,7 +145,7 @@ typedef enum { SG_AC_RemovingAssignment, SG_AC_AssigningActiveworkload, SG_AC_AssigningAutoAdjust, - SG_AC_AssigningStandBy, + SG_AC_AssigningWorkload, SG_AC_WaitingAfterOperationFailed, SG_AC_RemovingStandbyAssignments } sg_avail_control_state_t; @@ -238,6 +238,12 @@ typedef struct amf_fifo { uint8_t data[]; } amf_fifo_t; + +typedef enum { + AMF_PRESENCE_TERMINATION_FAILED_REBOOT = + SA_AMF_PRESENCE_TERMINATION_FAILED + 1 +} AmfPresenceStateT; + typedef struct amf_cluster { /* Configuration Attributes */ SaNameT name; diff --git a/exec/amfcomp.c b/exec/amfcomp.c index f9036ac9..39955b99 100644 --- a/exec/amfcomp.c +++ b/exec/amfcomp.c @@ -644,7 +644,6 @@ static void mcast_cleanup_completion_event (void *context) (struct clc_command_run_data *)context; struct req_exec_amf_clc_cleanup_completed req; struct iovec iovec; - req.header.size = sizeof (struct req_exec_amf_clc_cleanup_completed); req.header.id = SERVICE_ID_MAKE (AMF_SERVICE, MESSAGE_REQ_EXEC_AMF_CLC_CLEANUP_COMPLETED); @@ -1443,9 +1442,12 @@ void amf_comp_cleanup_failed_completed (amf_comp_t *comp) */ void amf_comp_cleanup_completed (struct amf_comp *comp) { - TRACE2("Exec CLC cleanup completed for '%s'", comp->name.value); + TRACE2("Exec CLC cleanup completed for '%s' %d", comp->name.value, + comp->saAmfCompPresenceState); + stop_component_cleanup_timer (comp); + /* Set all CSI's confirmed HA state to unknown */ amf_comp_foreach_csi_assignment (comp, clear_ha_state); @@ -1453,6 +1455,9 @@ void amf_comp_cleanup_completed (struct amf_comp *comp) if (comp->saAmfCompPresenceState == SA_AMF_PRESENCE_RESTARTING) { amf_comp_instantiate (comp); + } else if (comp->saAmfCompPresenceState == + SA_AMF_PRESENCE_TERMINATION_FAILED) { + comp_presence_state_set (comp, SA_AMF_PRESENCE_TERMINATION_FAILED); } else { comp_presence_state_set (comp, SA_AMF_PRESENCE_UNINSTANTIATED); } @@ -1631,7 +1636,6 @@ void amf_comp_instantiate_tmo_event (struct amf_comp *comp) amf_comp_operational_state_set (comp, SA_AMF_OPERATIONAL_DISABLED); comp_presence_state_set (comp, SA_AMF_PRESENCE_INSTANTIATION_FAILED); - break; case SA_AMF_PRESENCE_INSTANTIATED: assert (comp->instantiate_timeout_handle == 0); @@ -1923,7 +1927,9 @@ void amf_comp_terminate (struct amf_comp *comp) if (amf_comp_is_error_suspected(comp)) { clc_interfaces[comp->comptype]->cleanup (comp); } else { - clc_interfaces[comp->comptype]->terminate (comp); + /*TODO temination implementation */ + /*clc_interfaces[comp->comptype]->terminate (comp);*/ + clc_interfaces[comp->comptype]->cleanup (comp); } } } @@ -1968,7 +1974,7 @@ SaAisErrorT amf_comp_hastate_get ( return SA_AIS_OK; } - return SA_AIS_ERR_INVALID_PARAM; + return SA_AIS_ERR_NOT_EXIST; } /** diff --git a/exec/amfnode.c b/exec/amfnode.c index 27186476..0d237761 100644 --- a/exec/amfnode.c +++ b/exec/amfnode.c @@ -499,11 +499,7 @@ void amf_node_comp_failover_req (amf_node_t *node, amf_comp_t *comp) if (comp->su->saAmfSUFailover) { /* SU failover */ amf_sg_failover_su_req (comp->su->sg,comp->su, node); - - } else { - /* TODO: component failover */ - assert (0); - } + } break; case NODE_ACSM_IDLE_ESCALLATION_LEVEL_3: if (comp->su->su_failover_cnt < node->saAmfNodeSuFailoverMax) { @@ -511,13 +507,10 @@ void amf_node_comp_failover_req (amf_node_t *node, amf_comp_t *comp) /* SU failover */ amf_sg_failover_su_req (comp->su->sg,comp->su, node); - } else { - /* TODO: component failover */ - assert (0); - } + } } else { node->history_state = NODE_ACSM_IDLE_ESCALLATION_LEVEL_0; - amf_node_failover (node); + amf_node_failover (node); } break; default: diff --git a/exec/amfsg.c b/exec/amfsg.c index 2109b949..4048113f 100644 --- a/exec/amfsg.c +++ b/exec/amfsg.c @@ -83,7 +83,7 @@ * initiated. * <3> All active assignments can not be transferred at the same time to the * different components because the rules for dependencies between SI and - * SI application wide and CSI and CSI within one SI, has to be respected. + * SI cluster wide and CSI and CSI within one SI, has to be respected. * * SG is fully responsible for step <1> but not fully responsible for handling * step <2> and <3>. However, SG uses an attribute called 'dependency level' @@ -103,10 +103,11 @@ * - administrative state machine (ADSM) (NOT IN THIS RELEASE) * - availability control state machine (ACSM) * - * The availability control state machine contains two states and one of them + * The availability control state machine contains three states and one of them * is composite. Being a composite state means that it contains substates. * The states are: * - IDLE (non composite state) + * - INSTANTIATING_SERVICE_UNITS * - MANAGING_SG (composite state) * MANAGING_SG is entered at several different events which has in common * the need to set up or change the assignment of SIs to SUs. Only one such @@ -117,11 +118,11 @@ * - start (requests SG to order SU to instantiate all SUs in SG and waits * for SU to indicate presence state change reports from the SUs and * finally responds 'started' to the requester) - * - assign (requests SG to assign SIs to SUs according to pre-configured - * rules (if not already done) and transfer the HA state of - * the SIs on the requested SI dependency level. Then SG waits for - * confirmation that the HA state has been succesfully set and - * finally responds 'assigned' to the reqeuster) + * - assign_si (requests SG to assign SIs to SUs according to pre-configured + * rules (if not already done) and transfer the HA state of + * the SIs on the requested SI dependency level. Then SG waits for + * confirmation that the HA state has been succesfully set and + * finally responds 'assigned' to the reqeuster) * - auto_adjust (this event indicates that the auto-adjust probation timer has * expired and that SG should evaluate current assignments of * SIs to SUs and if needed remove current assignments and @@ -138,6 +139,227 @@ * in 3.12.1.3 and respond to the requester when recovery is * completed) * +* 1. SG Availability Control State Machine + * ========================================== + * + * 1.1 State Transition Table + * + * State: Event: Action: New state: + * ============================================================================ + * IDLE start A48,A28 INSTANTIATING_SUs + * IDLE assign_si A48,A31 ASSIGNING_ON_REQ + * IDLE failover_su A48,[C22]A10,A11 DEACTIVATING_DEP + * IDLE failover_su A48,[!C22]A12 TERMINATING_SUSP + * IDLE failover_node A48,[!C22]A12 TERMINATING_SUSP + * IDLE failover_node A48,[C22]A10,A11 DEACTIVATING_DEP + * IDLE failover_node A48,[C100]A34 IDLE + * INSTANTIATING_SUs start A48,A28 INSTANTIATING_SUs + * INSTANTIATING_SUs su_state_chg [C101]A26,A53 IDLE + * INSTANTIATING_SUs su_state_chg [C102]A26,A53 IDLE + * INSTANTIATING_SUs assign_si A31 ASSIGNING_ON_REQ + * INSTANTIATING_SUs failover_su A52 INSTANTIATING_SUs + * INSTANTIATING_SUs failover_node A52 INSTANTIATING_SUs + * ASSIGNING_ON_REQ ha_state_assumed [C15]A54 IDLE + * ASSIGNING_ON_REQ failover_su A52 ASSIGNING_ON_REQ + * ASSIGNING_ON_REQ failover_node A52 ASSIGNING_ON_REQ + * DEACTIVATING_DEP si_deactivated [C20] REMOVING_S-BY_ASS + * DEACTIVATING_DEP si_deactivated [!C20]A12 TERMINATING_SUSP + * TERMINATING_SUSP su_state_chg [C103]A24,A20 ASS_S-BY_TO_SPARE + * TERMINATING_SUSP su_state_chg [C104]A24,A50 REMOVING_S-BY_ASS + * TERMINATING_SUSP su_state_chg [C105]A16,A17 ACTIVATING_S-BY + * TERMINATING_SUSP su_state_chg [C106]A20 ASS_S-BY_TO_SPARE + * TERMINATING_SUSP su_state_chg [C108]A23 REPAIRING_SU + * TERMINATING_SUSP su_state_chg [C109] IDLE + * TERMINATING_SUSP failover_su A52 TERMINATING_SUSP + * TERMINATING_SUSP failover_node A52 TERMINATING_SUSP + * REMOVING_S-BY_ASS assignment_removed A51 REMOVING_S-BY_ASS + * REMOVING_S-BY_ASS assignment_removed [C27]&[C24] ACTIVATING_S-BY + * REMOVING_S-BY_ASS assignment_removed [C110]A20 ASS_S-BY_TO_SPARE + * REMOVING_S-BY_ASS assignment_removed [C111]A23 REPAIRING_SU + * REMOVING_S-BY_ASS assignment_removed [C112] IDLE + * REMOVING_S-BY_ASS failover_su A52 REMOVING_S-BY_ASS + * REMOVING_S-BY_ASS failover_node A52 REMOVING_S-BY_ASS + * ACTIVATING_S-BY su_activated [C2]&[C11]A20 ASS_S-BY_TO_SPARE + * ACTIVATING_S-BY su_activated [C113]A23 REPAIRING_SU + * ACTIVATING_S-BY su_activated [C114] IDLE + * ACTIVATING_S-BY failover_su A52 ACTIVATING_S-BY + * ACTIVATING_S-BY failover_node A52 ACTIVATING_S-BY + * ASS_S-BY_TO_SPARE ha_state_assumed [C115]A23 REPAIRING_SU + * ASS_S-BY_TO_SPARE ha_state_assumed [C116] IDLE + * ASS_S-BY_TO_SPARE failover_su A52 ASS_S-BY_TO_SPARE + * ASS_S-BY_TO_SPARE failover_node A52 ASS_S-BY_TO_SPARE + * REPAIRING_SU su_state_chg [C28]A36,A37,A31 ASSIGNING_WL + * REPAIRING_SU su_state_chg [C28][C31] IDLE + * REPAIRING_SU failover_su A52 REPAIRING_SU + * REPAIRING_SU failover_node A52 REPAIRING_SU + * ASSIGNING_WL ha_state_assumed [C15] IDLE + * ASSIGNING_WL failover_su A52 ASSIGNING_WL + * ASSIGNING_WL failover_node A52 ASSIGNING_WL + * + * 1.2 State Description + * ===================== + * IDLE - SG is synchronized and idle. When IDLE state is set, + * the oldest deferred event (if any) is recalled. + * INSTANTIATING_SUs - INSTANTIATING_SERVICE_UNITS + * SG has ordered all contained SUs to instantiate and + * waits for the SUs to report a change of their + * presence state. SG is also prepared to accept an + * order to assign workload in this state. + * ASSIGNING_ON_REQ - ASSIGNING_ON_REQUEST + * SG has on request assigned workload to all service units + * on the requested dependency level and waits for SIs to + * indicate that the requested HA-state have been set to the + * appropriate components. + * TERMINATING_SUSP - TERMINATING_SUSPECTED + * SG has cleaned up all components suspected to be + * erroneous and waits for the concerned SUs to report a + * change of their presence states. + * REMOVING_S-BY_ASS - REMOVING_STANDBY_ASSIGNMENTS + * This state is only applicable to the n-plus-m redundancy + * model. In this state, SG has initiated the removal of + * those assignments from standby SUs that do not match the + * assignments of error suspected SUs. The reason for this + * removal is a preparation for not violating the rule which + * says an SU can not have both active and standby assign- + * ments simultaneously in the n-plus-m redundancy model. + * ACTIVATING_S-BY - ACTIVATING_STANDBY + * SG has located all standby SI-assignments in the recovery + * scope and ordered the corresponding SI to set the active + * HA-state and waits for SI to indicate that the requested + * HA-state has been set to the appropriate components. + * ASS_S-BY_TO_SPARE - ASSIGNING_STANDBY_TO_SPARE + * Current SG is configured with a spare SU. In this state, + * SG has requested all SIs to assign the standby HA-state + * to the spare SU and waits for the SIs to indicate that + * the standby HA-state have been set. + * REPAIRING_SU - REPAIRING_SU + * In this state SG has initiated instantiation of all SUs + * in current recovery scope until the configured preference + * of number of instantiated SUs is fulfiled. SG then waits + * for the concerned SUs to report a change of presence + * state. + * ASSIGNING_WL - ASSIGNING_WORKLOAD + * In this state SG has initiated the assignment of workload + * to all or a subset of its contained SUs and waits for the + * concerned SIs to indicated that the requested HA-state + * has been assumed. + * + * 1.3 Actions + * =========== + * A1 - + * A2 - + * A3 - + * A4 - + * A5 - + * A6 - + * A7 - + * A8 - + * A9 - + * A10 - [foreach SI in the scope]&[foreach SI-assignment with + * confirmed-HA-state == ACTIVE]/set requested-ha-state = QUIESCED + * A11 - [foreach SI in the scope]/deactivate SI + * A12 - [foreach suspected SU]/terminate all components + * A13 - + * A14 - + * A15 - + * A16 - [foreach SI in the scope]&[foreach SI-assignment with + * confirmed-HA-state == STANDBY]/set requested-ha-state = ACTIVE + * A17 - [foreach SI in the scope]/activate SI + * A18 - + * A19 - + * A20 - + * A21 - + * A22 - + * A23 - + * A24 - + * A25 - + * A26 - + * A27 - + * A28 - + * A29 - + * A30 - + * A31 - + * A32 - + * A33 - + * A34 - + * A35 - + * A36 - + * A37 - + * A38 - + * A39 - + * A40 - + * A41 - + * A42 - + * A43 - + * A44 - + * A45 - + * A46 - + * A47 - + * A48 - + * A49 - + + * + * 1.4 Composite Guards + * ==================== + * The meaning with these guards is just to save space in the state transition + * table above. + * C100 - [C7]&[!C22]&[C20] + * C101 - [C12]&[C28] + * C102 - [C13]&[C28] + * C103 - [C6]&[C21]&[C11] + * C104 - [C6]&[C25]&[C26] + * C105 - [C6]&(!([C25]|[C26]))&[C24] + * C106 - [C6]&(!([C25]|[C26]))&[!C24]&[C11] + * C107 - + * C108 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[!C30] + * C109 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[C30] + * C110 - [C27]&[!C24]&[C11] + * C111 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[!C30] + * C112 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[C30] + * C113 - [C2]&[!C11]&[C9]&[C10]&[!C30] + * C114 - [C2]&[!C11]&[C9]&[C10]&[C30] + * C115 - [C9]&[C10]&[!C30] + * C116 - [C9]&[C10]&[C30] + * + * 1.4 Guards + * ========== + * C1 - + * C2 - all SI in the recovery scope + * C3 - + * C4 - + * C5 - + * C6 - all supected SUs or components have presence state == UNINSTANTIATED + * or presence state == INSTANTIATION_FAILED + * C7 - original event == failover node + * C8 - + * C9 - original event == failover su + * C10 - SaAmfSGAutoRepair == true + * C11 - spare SUs exist + * C12 - original event == start(all SUs) + * C13 - original event == start(node) + * C14 - + * C15 - all SI-assignments on current dependency-level have requested-ha-state + == confirmed-ha-state or operation failed + * C16 - + * C17 - + * C18 - + * C19 - + * C20 - all suspected SUs have presence state == UNINSTANTIATED or + presence state == xx_FAILED + * C21 - no SI in the scope has an SI-assignment with HA-state == STANDBY + * C22 - the concerned entity has ACTIVE HA-state + * C23 - + * C24 - at least one SI-assignment related to an SI in the scope has HA-state + == STANDBY + * C25 - redundancy model == N plus M + * C26 - at least one SU has STANDBY assignments for more SIs than those SIs + that are within the recovery scope + * C27 - no SI-assignment related to SI protected by current SG has requested + HA-state == REMOVED + * C28 - no SU has presence state == INSTANTIATING + * C29 - + * C30 - more SUs not needed or the node hosting the SU to repair is disabled. + * C31 - no new additional assignments needed or possible */ #include @@ -154,9 +376,11 @@ static void delete_si_assignments_in_scope (struct amf_sg *sg); static void acsm_enter_repairing_su (struct amf_sg *sg); static void standby_su_activated_cbfn ( struct amf_si_assignment *si_assignment, int result); - static void dependent_si_deactivated_cbfn ( struct amf_si_assignment *si_assignment, int result); +static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg); +static void assign_si_assumed_cbfn ( + struct amf_si_assignment *si_assignment, int result); static void acsm_enter_removing_standby_assignments (amf_sg_t *sg); static void acsm_enter_assigning_standby_to_spare (amf_sg_t *sg); @@ -179,6 +403,10 @@ typedef struct sg_event { amf_node_t *node; } sg_event_t; +/****************************************************************************** + * Internal (static) utility functions + *****************************************************************************/ + static int is_cluster_start(amf_node_t *node_to_start) { return node_to_start == NULL; @@ -400,45 +628,6 @@ static inline int su_presense_state_is_not (amf_su_t *su, state2 && su->saAmfSUPresenceState != state3) ? 1 : 0; } -/** - * Callback function used by SI when there is no dependent SI to - * deactivate. - * @param sg - */ -static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg) -{ - struct amf_su **sus = sg->recovery_scope.sus; - - ENTER("'%s'", sg->name.value); - - /* - * Select next state depending on if some - * SU in the scope is needs to be terminated. - */ - - while (*sus != NULL) { - amf_su_t *su = *sus; - - ENTER("SU %s pr_state='%d'",su->name.value, - su->saAmfSUPresenceState); - - if (su_presense_state_is_ored (su, - SA_AMF_PRESENCE_UNINSTANTIATED, - SA_AMF_PRESENCE_TERMINATION_FAILED, - SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { - sus++; - continue; - } - break; - } - - if (*sus != NULL) { - acsm_enter_terminating_suspected (sg); - } else { - delete_si_assignments_in_scope(sg); - acsm_enter_removing_standby_assignments (sg); - } -} static void timer_function_dependent_si_deactivated2 (void *data) { @@ -680,66 +869,6 @@ static int is_all_si_assigned (amf_sg_t *sg) return (confirmed_assignments == si_assignment_cnt); } -static void assign_si_assumed_cbfn ( - struct amf_si_assignment *si_assignment, int result) -{ - struct amf_sg *sg = si_assignment->su->sg; - int si_assignment_cnt = 0; - int confirmed_assignments = 0; - - ENTER ("'%s', %d", si_assignment->si->name.value, result); - - - switch (sg->avail_state) { - case SG_AC_AssigningOnRequest: - if (is_all_si_assigned (sg)) { - acsm_enter_idle (sg); - amf_application_sg_assigned (sg->application, sg); - } else { - dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments); - } - break; - case SG_AC_AssigningStandBy: - { - if (is_all_si_assigned(sg)) { - acsm_enter_idle (sg); - } - break; - } - case SG_AC_AssigningStandbyToSpare: - { - if(is_all_si_assigned (sg)) { - /* - * All si_assignments has asumed - * Prescense state SA_AMF_HA_STANDBY - */ - switch (sg->recovery_scope.event_type) { - case SG_FAILOVER_NODE_EV: - acsm_enter_idle (sg); - break; - case SG_FAILOVER_SU_EV: - if (sg->saAmfSGAutoRepair == SA_TRUE) { - acsm_enter_repairing_su (sg); - } - break; - default: - assert (0); - break; - } - } else { - si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY; - } - } - break; - default: - dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt, - confirmed_assignments); - amf_runtime_attributes_print (amf_cluster); - assert (0); - break; - } -} - /** * Inquire if SI is assigned to SU * @param si @@ -1252,94 +1381,6 @@ static void acsm_enter_removing_standby_assignments (amf_sg_t *sg) } } -/** - * Callback function used by SI when an SI has been deactivated. - * @param si_assignment - * @param result - */ -static void dependent_si_deactivated_cbfn ( - struct amf_si_assignment *si_assignment, int result) -{ - struct amf_sg *sg = si_assignment->su->sg; - struct amf_su **sus = sg->recovery_scope.sus; - struct amf_su *su; - - ENTER ("'%s', %d", si_assignment->si->name.value, result); - - /* - * If all SI assignments for all SUs in the SG are not pending, - * goto next state (TerminatingSuspected). - */ - for (su = sg->su_head ; su != NULL; su = su->next) { - struct amf_si_assignment *si_assignment; - si_assignment = amf_su_get_next_si_assignment(su, NULL); - - while (si_assignment != NULL) { - if (si_assignment->saAmfSISUHAState != - si_assignment->requested_ha_state) { - goto still_wating; - } - si_assignment = amf_su_get_next_si_assignment(su, - si_assignment); - } - } - -still_wating: - - if (su == NULL) { - sus = si_assignment->su->sg->recovery_scope.sus; - - /* - * Select next state depending on if some - * SU in the scope is needs to be terminated. - */ - - while (*sus != NULL) { - if (su_presense_state_is_not (*sus, - SA_AMF_PRESENCE_UNINSTANTIATED, - SA_AMF_PRESENCE_TERMINATION_FAILED, - SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { - break; - } - sus++; - } - if (*sus != NULL) { - acsm_enter_terminating_suspected (sg); - } else { - acsm_enter_removing_standby_assignments (sg); - } - } - LEAVE(""); -} - -static void standby_su_activated_cbfn ( - struct amf_si_assignment *si_assignment, int result) -{ - struct amf_su **sus = si_assignment->su->sg->recovery_scope.sus; - struct amf_si **sis = si_assignment->su->sg->recovery_scope.sis; - - ENTER ("'%s', %d", si_assignment->si->name.value, result); - - /* - * If all SI assignments for all SIs in the scope are activated, goto next - * state. - */ - - while (*sis != NULL) { - if ((*sis)->assigned_sis != NULL && - (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { - break; - } - sis++; - } - - if (*sis == NULL) { - - acsm_enter_assigning_standby_to_spare ((*sus)->sg); - } -} - - static inline int div_round (int a, int b) { int res; @@ -1475,7 +1516,7 @@ static int amf_si_get_saAmfSINumReqStandbyAssignments(struct amf_si *si) return number_of_req_active_assignments; } -static int sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) +static int sg_assign_active_nplusm (struct amf_sg *sg, int su_active_assign) { struct amf_su *su; struct amf_si *si; @@ -1537,7 +1578,7 @@ static int sg_assign_nm_active (struct amf_sg *sg, int su_active_assign) return total_assigned; } -static int sg_assign_nm_standby (struct amf_sg *sg, int su_standby_assign) +static int sg_assign_standby_nplusm (struct amf_sg *sg, int su_standby_assign) { struct amf_su *su; struct amf_si *si; @@ -1659,9 +1700,16 @@ static int su_standby_out_of_service_count_get (amf_sg_t *sg) } /** + * This function calculates the number of active and standby assignments that + * shall be done according to what is configured and the number of in-service + * SUs available. This function leaves possible existing assignments as they are + * but possibly adds new assignments. This function also initiates the transfer + * of the calculated assignments to the SUs that shall execute them. + * * TODO: dependency_level not used, hard coded * @param sg * @param dependency_level + * @return - the sum of assignments initiated */ static int assign_si (struct amf_sg *sg, int dependency_level) { @@ -1770,8 +1818,8 @@ static int assign_si (struct amf_sg *sg, int dependency_level) inservice_count, su_active_assign, su_standby_assign, su_spare_assign); if (inservice_count > 0) { - assigned = sg_assign_nm_active (sg, su_active_assign); - assigned += sg_assign_nm_standby (sg, su_standby_assign); + assigned = sg_assign_active_nplusm (sg, su_active_assign); + assigned += sg_assign_standby_nplusm (sg, su_standby_assign); sg->saAmfSGNumCurrAssignedSUs = inservice_count; /** @@ -1856,7 +1904,41 @@ static void remove_sis_for_term_failed_su_from_scope (amf_sg_t *sg, #endif /** - * sg_su_state_changed_in_instantiated + * This function returns 1 if the redundancy model is N plus M and at least one + * component of the specified SU has an active HA-state, else the function + * returns 0. + * @param sg + * @param su + * @return int + */ +static int is_comp_in_active_ha_state_nplusm ( + amf_sg_t *sg, amf_su_t *su) +{ + amf_comp_t *component; + amf_csi_assignment_t *csi_assignment; + int comp_is_in_active_ha_state = 0; + + if(sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) { + for (component = su->comp_head; component != NULL; + component = component->next) { + csi_assignment = amf_comp_get_next_csi_assignment(component, NULL); + while (csi_assignment != NULL) { + if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_ACTIVE) { + comp_is_in_active_ha_state = 1; + goto out; + } + csi_assignment = amf_comp_get_next_csi_assignment(component, + csi_assignment); + } + } + } +out: + return comp_is_in_active_ha_state; +} + +/** + * This function handles a change of presence state reported by an SU contained + * in specified SG. The new presence state is INSTANTIATED. * @param sg * @param su */ @@ -1878,7 +1960,7 @@ static void sg_su_state_changed_to_instantiated (struct amf_sg *sg, struct amf_s SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -1896,7 +1978,8 @@ static void sg_su_state_changed_to_instantiated (struct amf_sg *sg, struct amf_s } /** - * amf_sg_su_state_changed_in_uninstantiated + * This function handles a change of presence state reported by an SU contained + * in specified SG. The new presence state is UNINSTANTIATED. * @param sg * @param su */ @@ -1932,7 +2015,7 @@ static void amf_sg_su_state_changed_to_uninstantiated (amf_sg_t *sg, SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -1946,33 +2029,9 @@ static void amf_sg_su_state_changed_to_uninstantiated (amf_sg_t *sg, } } -static int npm_and_comp_in_active_ha_state ( - amf_sg_t *sg, amf_su_t *su) -{ - amf_comp_t *component; - amf_csi_assignment_t *csi_assignment; - int comp_is_in_active_ha_state = 0; - - if(sg->saAmfSGRedundancyModel == SA_AMF_NPM_REDUNDANCY_MODEL) { - for (component = su->comp_head; component != NULL; - component = component->next) { - csi_assignment = amf_comp_get_next_csi_assignment(component, NULL); - while (csi_assignment != NULL) { - if (csi_assignment->saAmfCSICompHAState == SA_AMF_HA_ACTIVE) { - comp_is_in_active_ha_state = 1; - goto out; - } - csi_assignment = amf_comp_get_next_csi_assignment(component, - csi_assignment); - } - } - } -out: - return comp_is_in_active_ha_state; -} - /** - * amf_sg_su_state_changed_in_termination_failed + * This function handles a change of presence state reported by an SU contained + * in specified SG. The new presence state is TERMINATION_FAILED. * @param sg * @param su */ @@ -1981,12 +2040,12 @@ static void amf_sg_su_state_changed_to_termination_failed (amf_sg_t *sg, { ENTER("%s %s",sg->name.value, su->name.value); if (no_su_has_presence_state(sg, sg->node_to_start, - SA_AMF_PRESENCE_INSTANTIATING)) { - if (npm_and_comp_in_active_ha_state (sg, su)) { + SA_AMF_PRESENCE_TERMINATING)) { + if (is_comp_in_active_ha_state_nplusm (sg, su)) { acsm_enter_idle (sg); goto out; } - + if (all_su_in_scope_has_either_two_presence_state (sg, SA_AMF_PRESENCE_UNINSTANTIATED, SA_AMF_PRESENCE_TERMINATION_FAILED)) { @@ -2005,8 +2064,10 @@ static void amf_sg_su_state_changed_to_termination_failed (amf_sg_t *sg, out: return; } + /** - * amf_sg_su_state_changed_in_instantiation_failed + * This function handles a change of presence state reported by an SU contained + * in specified SG. The new presence state is INSTANTIATION_FAILED. * @param sg * @param su */ @@ -2029,7 +2090,7 @@ static void amf_sg_su_state_changed_to_instantiation_failed (amf_sg_t *sg, SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -2043,85 +2104,17 @@ static void amf_sg_su_state_changed_to_instantiation_failed (amf_sg_t *sg, } } -int amf_sg_assign_si_req (struct amf_sg *sg, int dependency_level) -{ - int posible_to_assign_si; - sg->recovery_scope.event_type = SG_ASSIGN_SI_EV; - sg->avail_state = SG_AC_AssigningOnRequest; - - if ((posible_to_assign_si = assign_si (sg, dependency_level)) == 0) { - acsm_enter_idle (sg); - } - return posible_to_assign_si; -} - - -void amf_sg_failover_node_req (struct amf_sg *sg, struct amf_node *node) -{ - ENTER("'%s, %s'",node->name.value, sg->name.value); - sg_event_t sg_event; - - switch (sg->avail_state) { - case SG_AC_Idle: - set_scope_for_failover_node(sg, node); - if (has_any_su_in_scope_active_workload (sg)) { - acsm_enter_deactivating_dependent_workload (sg); - } else { - amf_su_t **sus = sg->recovery_scope.sus; - - /* - * Select next state depending on if some - * SU in the scope needs to be terminated. - */ - while (*sus != NULL) { - - amf_su_t *su = *sus; - ENTER("SU %s pr_state='%d'",su->name.value, - su->saAmfSUPresenceState); - - if (su_presense_state_is_ored (su, - SA_AMF_PRESENCE_UNINSTANTIATED, - SA_AMF_PRESENCE_TERMINATION_FAILED, - SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { - sus++; - continue; - } - break; - } - - if (*sus != NULL) { - acsm_enter_terminating_suspected (sg); - } else { - delete_si_assignments_in_scope (sg); - acsm_enter_idle (sg); - } - - } - break; - case SG_AC_DeactivatingDependantWorkload: - case SG_AC_TerminatingSuspected: - case SG_AC_ActivatingStandby: - case SG_AC_AssigningStandbyToSpare: - case SG_AC_ReparingComponent: - case SG_AC_ReparingSu: - case SG_AC_AssigningOnRequest: - case SG_AC_InstantiatingServiceUnits: - case SG_AC_RemovingAssignment: - case SG_AC_AssigningActiveworkload: - case SG_AC_AssigningAutoAdjust: - case SG_AC_AssigningStandBy: - case SG_AC_WaitingAfterOperationFailed: - case SG_AC_RemovingStandbyAssignments: - sg_set_event (SG_FAILOVER_NODE_EV, sg, 0, 0, node, &sg_event); - sg_defer_event (SG_FAILOVER_NODE_EV, &sg_event); - break; - default: - assert (0); - break; - - } -} +/****************************************************************************** + * Event methods + *****************************************************************************/ +/** + * This function starts all SUs in the SG or all SUs on a specified node. + * @param sg + * @param node - Node on which all SUs shall be started or + * NULL indicating that all SUs on all nodes shall be started. + * @return - No of SUs that has been attempted to start + */ int amf_sg_start (struct amf_sg *sg, struct amf_node *node) { @@ -2182,7 +2175,7 @@ int amf_sg_start (struct amf_sg *sg, struct amf_node *node) case SG_AC_RemovingAssignment: case SG_AC_AssigningActiveworkload: case SG_AC_AssigningAutoAdjust: - case SG_AC_AssigningStandBy: + case SG_AC_AssigningWorkload: case SG_AC_WaitingAfterOperationFailed: case SG_AC_RemovingStandbyAssignments: default: @@ -2192,6 +2185,153 @@ int amf_sg_start (struct amf_sg *sg, struct amf_node *node) return instantiated_sus; } +/** + * This function initiates assignment of the subset of the workload which + * matches the specified workload dependency level, to all SUs contained in the + * SG according to the requirements specified in the configuration. + * @param sg - + * @param dependency_level - Dependency level to assign + * @return - No of SUs that has been attempted to start + */ +int amf_sg_assign_si_req (struct amf_sg *sg, int dependency_level) +{ +/* TODO: Introduce state control in this function + */ + int posible_to_assign_si; + sg->recovery_scope.event_type = SG_ASSIGN_SI_EV; + sg->avail_state = SG_AC_AssigningOnRequest; + + if ((posible_to_assign_si = assign_si (sg, dependency_level)) == 0) { + acsm_enter_idle (sg); + } + return posible_to_assign_si; +} + +/** + * This function is called because an error has been detected and the analysis + * (done elsewhere) indicated that this error shall be recovered by a Node + * failover. This function initiates the recovery action 'Node failover'. + * @param sg + * @param su - SU to failover + * @param node - + */ +void amf_sg_failover_node_req (struct amf_sg *sg, struct amf_node *node) +{ + ENTER("'%s, %s'",node->name.value, sg->name.value); + sg_event_t sg_event; + + switch (sg->avail_state) { + case SG_AC_Idle: + set_scope_for_failover_node(sg, node); + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + amf_su_t **sus = sg->recovery_scope.sus; + + /* + * Select next state depending on if some + * SU in the scope needs to be terminated. + */ + while (*sus != NULL) { + + amf_su_t *su = *sus; + ENTER("SU %s pr_state='%d'",su->name.value, + su->saAmfSUPresenceState); + + if (su_presense_state_is_ored (su, + SA_AMF_PRESENCE_UNINSTANTIATED, + SA_AMF_PRESENCE_TERMINATION_FAILED, + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope (sg); + acsm_enter_idle (sg); + } + + } + break; + case SG_AC_DeactivatingDependantWorkload: + case SG_AC_TerminatingSuspected: + case SG_AC_ActivatingStandby: + case SG_AC_AssigningStandbyToSpare: + case SG_AC_ReparingComponent: + case SG_AC_ReparingSu: + case SG_AC_AssigningOnRequest: + case SG_AC_InstantiatingServiceUnits: + case SG_AC_RemovingAssignment: + case SG_AC_AssigningActiveworkload: + case SG_AC_AssigningAutoAdjust: + case SG_AC_AssigningWorkload: + case SG_AC_WaitingAfterOperationFailed: + case SG_AC_RemovingStandbyAssignments: + sg_set_event (SG_FAILOVER_NODE_EV, sg, 0, 0, node, &sg_event); + sg_defer_event (SG_FAILOVER_NODE_EV, &sg_event); + break; + default: + assert (0); + break; + + } +} + +/** + * This function is called because an error has been detected and the analysis + * (done elsewhere) indicated that this error shall be recovered by an SU + * failover. This function initiates the recovery action 'SU failover'. + * @param sg + * @param su - SU to failover + * @param node - + */ +void amf_sg_failover_su_req (struct amf_sg *sg, struct amf_su *su, + struct amf_node *node) +{ + ENTER ("%s", su->name.value); + sg_event_t sg_event; + + switch (sg->avail_state) { + case SG_AC_Idle: + su->su_failover_cnt += 1; + set_scope_for_failover_su (sg, su); + if (has_any_su_in_scope_active_workload (sg)) { + acsm_enter_deactivating_dependent_workload (sg); + } else { + acsm_enter_terminating_suspected (sg); + } + break; + case SG_AC_DeactivatingDependantWorkload: + case SG_AC_TerminatingSuspected: + case SG_AC_ActivatingStandby: + case SG_AC_AssigningStandbyToSpare: + case SG_AC_ReparingComponent: + case SG_AC_ReparingSu: + case SG_AC_AssigningOnRequest: + case SG_AC_InstantiatingServiceUnits: + case SG_AC_RemovingAssignment: + case SG_AC_AssigningActiveworkload: + case SG_AC_AssigningAutoAdjust: + case SG_AC_AssigningWorkload: + case SG_AC_WaitingAfterOperationFailed: + case SG_AC_RemovingStandbyAssignments: + sg_set_event (SG_FAILOVER_SU_EV, sg, su, 0, 0, &sg_event); + sg_defer_event (SG_FAILOVER_SU_EV, &sg_event); + break; + default: + assert (0); + break; + } +} + +/****************************************************************************** + * Event response methods + *****************************************************************************/ + #ifdef COMPILE_OUT void amf_sg_su_state_changed_2 (struct amf_sg *sg, struct amf_su *su, SaAmfStateT type, int state) @@ -2212,7 +2352,7 @@ void amf_sg_su_state_changed_2 (struct amf_sg *sg, SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -2247,7 +2387,7 @@ void amf_sg_su_state_changed_2 (struct amf_sg *sg, SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -2293,7 +2433,7 @@ void amf_sg_su_state_changed_2 (struct amf_sg *sg, SA_AMF_PRESENCE_INSTANTIATED, SA_AMF_PRESENCE_INSTANTIATION_FAILED, SA_AMF_PRESENCE_UNINSTANTIATED)) { - su->sg->avail_state = SG_AC_AssigningStandBy; + su->sg->avail_state = SG_AC_AssigningWorkload; if (assign_si (sg, 0) == 0) { acsm_enter_idle (sg); } @@ -2312,82 +2452,257 @@ void amf_sg_su_state_changed_2 (struct amf_sg *sg, } #endif +/** + * SU indicates to SG that one of its state machines has changed state. + * @param sg - SG which contains the SU that has changed state + * @param su - SU which has changed state + * @param type - Indicates which state machine that has changed state + * @param state - The new state that has been assumed. + * + */ void amf_sg_su_state_changed (struct amf_sg *sg, struct amf_su *su, SaAmfStateT type, int state) { ENTER ("'%s' SU '%s' state %s", sg->name.value, su->name.value, amf_presence_state(state)); - - if (type == SA_AMF_PRESENCE_STATE) { - switch (state) { - case SA_AMF_PRESENCE_INSTANTIATED: - sg_su_state_changed_to_instantiated(sg, su); - break; - case SA_AMF_PRESENCE_UNINSTANTIATED: - amf_sg_su_state_changed_to_uninstantiated(sg, su); - break; - case SA_AMF_PRESENCE_TERMINATION_FAILED: - amf_sg_su_state_changed_to_termination_failed(sg, su); - break; - case SA_AMF_PRESENCE_INSTANTIATING: - ; /* nop */ - break; - case SA_AMF_PRESENCE_INSTANTIATION_FAILED: - amf_sg_su_state_changed_to_instantiation_failed(sg, su); - break; - default : - dprintf("sg->avail_state = %d, su instantiation state = %d", - sg->avail_state, state); - assert (0); - break; + if (sg->avail_state != SG_AC_Idle) { + if (type == SA_AMF_PRESENCE_STATE) { + switch (state) { + case SA_AMF_PRESENCE_INSTANTIATED: + sg_su_state_changed_to_instantiated(sg, su); + break; + case SA_AMF_PRESENCE_UNINSTANTIATED: + amf_sg_su_state_changed_to_uninstantiated(sg, su); + break; + case SA_AMF_PRESENCE_TERMINATION_FAILED: + amf_sg_su_state_changed_to_termination_failed(sg, su); + break; + case SA_AMF_PRESENCE_INSTANTIATING: + ; /* nop */ + break; + case SA_AMF_PRESENCE_INSTANTIATION_FAILED: + amf_sg_su_state_changed_to_instantiation_failed(sg, su); + break; + default : + dprintf("sg->avail_state = %d, su instantiation state = %d", + sg->avail_state, state); + assert (0); + break; + } } } + } +/** + * Callback function used by SI when there is no dependent SI to + * deactivate. + * @param sg + */ +static void dependent_si_deactivated_cbfn2 (struct amf_sg *sg) +{ + struct amf_su **sus = sg->recovery_scope.sus; + + ENTER("'%s'", sg->name.value); + + /* + * Select next state depending on if some + * SU in the scope needs to be terminated. + */ + + while (*sus != NULL) { + amf_su_t *su = *sus; + + ENTER("SU %s pr_state='%d'",su->name.value, + su->saAmfSUPresenceState); + + if (su_presense_state_is_ored (su, + SA_AMF_PRESENCE_UNINSTANTIATED, + SA_AMF_PRESENCE_TERMINATION_FAILED, + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + sus++; + continue; + } + break; + } + + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + delete_si_assignments_in_scope(sg); + acsm_enter_removing_standby_assignments (sg); + } +} + +/** + * Callback function used by SI when an SI has been deactivated, i.e. + * transitioned from active HA-state to any other state. + * @param si_assignment + * @param result - Indicates the result of the operation. + */ +static void dependent_si_deactivated_cbfn ( + struct amf_si_assignment *si_assignment, int result) +{ + struct amf_sg *sg = si_assignment->su->sg; + struct amf_su **sus = sg->recovery_scope.sus; + struct amf_su *su; + + ENTER ("'%s', %d", si_assignment->si->name.value, result); + + /* + * If all SI assignments for all SUs in the SG are not pending, + * goto next state (TerminatingSuspected). + */ + for (su = sg->su_head ; su != NULL; su = su->next) { + struct amf_si_assignment *si_assignment; + si_assignment = amf_su_get_next_si_assignment(su, NULL); + + while (si_assignment != NULL) { + if (si_assignment->saAmfSISUHAState != + si_assignment->requested_ha_state) { + goto still_wating; + } + si_assignment = amf_su_get_next_si_assignment(su, + si_assignment); + } + } + +still_wating: + + if (su == NULL) { + sus = si_assignment->su->sg->recovery_scope.sus; + + /* + * Select next state depending on if some + * SU in the scope is needs to be terminated. + */ + + while (*sus != NULL) { + if (su_presense_state_is_not (*sus, + SA_AMF_PRESENCE_UNINSTANTIATED, + SA_AMF_PRESENCE_TERMINATION_FAILED, + SA_AMF_PRESENCE_INSTANTIATION_FAILED)) { + break; + } + sus++; + } + if (*sus != NULL) { + acsm_enter_terminating_suspected (sg); + } else { + acsm_enter_removing_standby_assignments (sg); + } + } + LEAVE(""); +} + +/** + * Callback function used by SI to indicate an SI has assumed a new HA-state or + * that the attempt to do so failed. + * @param si_assignment + * @param result - Indicates the result of the operation. + */ +static void assign_si_assumed_cbfn ( + struct amf_si_assignment *si_assignment, int result) +{ + struct amf_sg *sg = si_assignment->su->sg; + int si_assignment_cnt = 0; + int confirmed_assignments = 0; + + ENTER ("'%s', %d", si_assignment->si->name.value, result); + + + switch (sg->avail_state) { + case SG_AC_AssigningOnRequest: + if (is_all_si_assigned (sg)) { + acsm_enter_idle (sg); + amf_application_sg_assigned (sg->application, sg); + } else { + dprintf ("%d, %d", si_assignment_cnt, confirmed_assignments); + } + break; + case SG_AC_AssigningWorkload: + { + if (is_all_si_assigned(sg)) { + acsm_enter_idle (sg); + } + break; + } + case SG_AC_AssigningStandbyToSpare: + { + if(is_all_si_assigned (sg)) { + /* + * All si_assignments has asumed + * Prescense state SA_AMF_HA_STANDBY + */ + switch (sg->recovery_scope.event_type) { + case SG_FAILOVER_NODE_EV: + acsm_enter_idle (sg); + break; + case SG_FAILOVER_SU_EV: + if (sg->saAmfSGAutoRepair == SA_TRUE) { + acsm_enter_repairing_su (sg); + } + break; + default: + assert (0); + break; + } + } else { + si_assignment->saAmfSISUHAState = SA_AMF_HA_STANDBY; + } + } + break; + default: + dprintf ("%d, %d, %d", sg->avail_state, si_assignment_cnt, + confirmed_assignments); + amf_runtime_attributes_print (amf_cluster); + assert (0); + break; + } +} + +/** + * Callback function used by SI when an SI has been activated, i.e. transitioned + * from any HA-state to an active HA-state. + * @param si_assignment + * @param result - Indicates the result of the operation. + */ +static void standby_su_activated_cbfn ( + struct amf_si_assignment *si_assignment, int result) +{ + struct amf_su **sus = si_assignment->su->sg->recovery_scope.sus; + struct amf_si **sis = si_assignment->su->sg->recovery_scope.sis; + + ENTER ("'%s', %d", si_assignment->si->name.value, result); + + /* + * If all SI assignments for all SIs in the scope are activated, goto next + * state. + */ + + while (*sis != NULL) { + if ((*sis)->assigned_sis != NULL && + (*sis)->assigned_sis->saAmfSISUHAState != SA_AMF_HA_ACTIVE) { + break; + } + sis++; + } + + if (*sis == NULL) { + + acsm_enter_assigning_standby_to_spare ((*sus)->sg); + } +} + +/****************************************************************************** + * General methods + *****************************************************************************/ + void amf_sg_init (void) { log_init ("AMF"); } -void amf_sg_failover_su_req (struct amf_sg *sg, struct amf_su *su, - struct amf_node *node) -{ - ENTER ("%s", su->name.value); - sg_event_t sg_event; - - switch (sg->avail_state) { - case SG_AC_Idle: - su->su_failover_cnt += 1; - set_scope_for_failover_su (sg, su); - if (has_any_su_in_scope_active_workload (sg)) { - acsm_enter_deactivating_dependent_workload (sg); - } else { - acsm_enter_terminating_suspected (sg); - } - break; - case SG_AC_DeactivatingDependantWorkload: - case SG_AC_TerminatingSuspected: - case SG_AC_ActivatingStandby: - case SG_AC_AssigningStandbyToSpare: - case SG_AC_ReparingComponent: - case SG_AC_ReparingSu: - case SG_AC_AssigningOnRequest: - case SG_AC_InstantiatingServiceUnits: - case SG_AC_RemovingAssignment: - case SG_AC_AssigningActiveworkload: - case SG_AC_AssigningAutoAdjust: - case SG_AC_AssigningStandBy: - case SG_AC_WaitingAfterOperationFailed: - case SG_AC_RemovingStandbyAssignments: - sg_set_event (SG_FAILOVER_SU_EV, sg, su, 0, 0, &sg_event); - sg_defer_event (SG_FAILOVER_SU_EV, &sg_event); - break; - default: - assert (0); - break; - } -} - /** * Constructor for SG objects. Adds SG to the list owned by * the specified application. Always returns a valid SG diff --git a/exec/amfsu.c b/exec/amfsu.c index ec67a559..3bb929c7 100644 --- a/exec/amfsu.c +++ b/exec/amfsu.c @@ -89,32 +89,195 @@ * reported to IMM when it is changed. * * The restart control state machine (RCSM) is used to implement level 1 of - * the error escallation polycy described in chapter 3.12.2 of the spec. It - * also implements component restart and service unit restart as described in + * the error escallation policy described in chapter 3.12.2 of the spec. It also + * implements component restart and service unit restart as described in * paragraph 3.12.1.2 and 3.12.1.3. * RCSM contains three composite states. * Being a composite state means that the state contains substates. * RCSM composite states are: - * - ESCALLATION_LEVEL (LEVEL_0, LEVEL_1 and LEVEL_2) + * - IDLE (LEVEL_0, LEVEL_1 and LEVEL_2) * - RESTARTING_COMPONENT (DEACTIVATING, RESTARTING, SETTING and ACTIVATING) * - RESTARTING_SERVICE_UNIT (DEACTIVATING, TERMINATING, INSTANTIATING, * and ACTIVATING) * - * ESCALLATION_LEVEL is a kind of idle state where no actions are performed - * and used only to remember the escallation level. Substate LEVEL_0 indicates - * no escallation. LEVEL_1 indicates that a component restart has been - * executed recently and the escallation timer is still running. At this level - * component restart requests will transition to RESTARTING_COMPONENT but - * if there are too many restart requests before the probation timer expires - * then a transition will be made to LEVEL_2 and the restart request will - * be forwarded to the node instance hosting this component. - * State RESTARTING_SERVICE_UNIT will only be assumed if the node explicitly - * requests the SU to execute a restart of itself (after having evaluated its - * part of the error escallation policy). + * IDLE is a kind of state where no actions are performed and used only to + * remember the escallation level. Substate LEVEL_0 indicates no escallation. + * LEVEL_1 indicates that a component restart has been executed recently and the + * escallation timer is still running. At this level component restart requests + * will transition to RESTARTING_COMPONENT but if there are too many restart + * requests before the probation timer expires then a transition will be made to + * LEVEL_2 and the restart request will be forwarded to the node instance + * hosting this component. State RESTARTING_SERVICE_UNIT will only be assumed if + * the node explicitly requests the SU to execute a restart of itself (after + * having evaluated its part of the error escallation policy). * - */ - - /* +* 1. Service Unit Restart Control State Machine +* ============================================ + * + * 1.1 State Transition Table + * =========================== + * + * State: Event: Action: New state: + * =========================================================================== + * IDLE_ESCALATION_x comp_restart A9 RS_COMP_RESTARTING + * IDLE_ESCALATION_x su_restart A20 RS_SU_INSTANTIATING + * IDLE_ESCALATION_0 error_suspected A1 IDLE_ESCALATION_1 + * IDLE_ESCALATION_1 error_suspected [!C3] A1 IDLE_ESCALATION_1 + * IDLE_ESCALATION_1 error_suspected [C3] A2 IDLE_ESCALATION_2 + * IDLE_ESCALATION_2 error_suspected A2 IDLE_ESCALATION_2 + * RS_COMP_RESTARTING comp_instantiated A11 RS_COMP_SETTING + * RS_COMP_RESTARTING comp_inst_failed A19 IDLE_ESCALATION_x + * RS_COMP_RESTARTING comp_term_failed A19 IDLE_ESCALATION_x + * RS_COMP_RESTARTING error_suspected A18 RS_COMP_RESTARTING + * RS_COMP_SETTING ha_state_assumed [C7] A19 IDLE_ESCALATION_x + * RS_COMP_SETTING error_suspected A18 RS_COMP_SETTING + * RS_SU_INSTANTIATING comp_instantiated [C12] A11 RS_SU_SETTING + * RS_SU_INSTANTIATING comp_inst_failed [C12] A19 IDLE_ESCALATION_X + * RS_SU_INSTANTIATING comp_term_failed [C12] A19 IDLE_ESCALATION_X + * RS_SU_INSTANTIATING error_suspected A18 RS_SU_INSTANTIATING + * RS_SU_SETTING ha_state_assumed [C10] A19 IDLE_ESCALATION_X + * RS_SU_SETTING error_suspected A18 RS_SU_SETTING + * + * 1.2 State Description + * ===================== + * IDLE_ESCALATION_x - This is just an abbreviated notation for + * IDLE_ESCALATION_0, IDLE_ESCALATION_1 or IDLE_ESCALATION_2 + * + * IDLE_ESCALATION_0 - SU_RC_IDLE_ESCALATION_LEVEL_0 + * Service unit is idle and the restart probation timer is + * off. + * + * IDLE_ESCALATION_1 - SU_RC_IDLE_ESCALATION_LEVEL_1 + * Service unit is idle and the restart probation timer is + * on. This indicates there has recently been an error + * detected on at least one of its components which has been + * recovered by a component restart but we are still in the + * probation period which follows every restart. + * + * IDLE_ESCALATION_2 - SU_RC_IDLE_ESCALATION_LEVEL_2 + * Service unit is idle and handling on potential new error + * indications on any of its components has been delegated + * to the node object where the service unit is hosted. + * + * RS_COMP_DEACTIVATING - SU_RC_RESTART_COMP_DEACTIVATING + * Service unit is busy handling restart of one of its + * components. In this sub-state, the service unit is + * waiting for acknowledgements that all components which + * had csi-assignments that were dependent of csi- + * assignments associated to the restarting component + * have been de-activated. This is a neccesary step to + * take before the component to restart is terminated, + * to avoid that the csi or si dependency rules are + * violated. + * + * RS_COMP_RESTARTING - SU_RC_RESTART_COMP_RESTARTING + * Service unit is busy handling restart of one of its + * components. In this sub-state, the service unit has + * ordered one of its components to restart and waits for + * the component to indicate that the restart is done. + * + * RS_COMP_SETTING - SU_RC_RESTART_COMP_SETTING + * Service unit is busy handling restart of one of its + * components. In this sub-state, the service unit has ordered + * the component that just have been restarted to re-assume + * the HA-states it had before, provided none of the states + * were ACTIVE. It waits for an acknowledgement that the + * setting of the HA-states are done. + * + * RS_COMP_ACTIVATING - SU_RC_RESTART_COMP_ACTIVATING + * Service unit is busy handling restart of one of its + * components. In this sub-state, the service unit has + * ordered the component that just have been restarted to + * re-assume the active HA-states it had before and also + * to activate the csi-assignments that possibly were + * de-activated because of this restart. The service unit + * waits in this state for an acknowledgement of the + * activation. + * + * RS_SU_DEACTIVATING - SU_RC_RESTART_SU_DEACTIVATING + * Service unit is busy handling restart of all of its + * components. In this sub-state, the service unit is + * waiting for acknowledgements that all components which + * had csi-assignments that were dependent of si- + * assignments associated to this service unit + * have been de-activated. This is a neccesary step to + * take before all components of the service unit are + * terminated, to avoid that the csi or si dependency rules + * are violated. + * + * RS_SU_TERMINATING - SU_RC_RESTART_SU_TERMINATING + * Service unit is busy handling restart of all of its + * components. In this sub-state, the service unit has + * ordered all its components to terminate and is waiting + * for an acknowledgement that all components are done with + * the termination. + * + * RS_SU_INSTANTIATING - SU_RC_RESTART_SU_INSTANTIATING + * Service unit is busy handling restart of all of its + * components. In this sub-state, the service unit has + * ordered all components to instantiate and is waiting + * for an acknowledgement that all components are done with + * the instantiation. + * + * RS_SU_SETTING - SU_RC_RESTART_SU_SETTING + * Service unit is busy handling restart of all of its + * components. In this sub-state, the service unit has ordered + * all components that just have been restarted to re-assume + * the HA-states they had before, provided none of the states + * were ACTIVE. The service unit waits for an acknowledgement + * that the setting of the HA-states are done. + * + * RS_SU_ACTIVATING - SU_RC_RESTART_SU_ACTIVATING + * Service unit is busy handling restart of all of its + * components. In this sub-state, the service unit has + * ordered all components that just have been restarted to + * re-assume the active HA-states they had before and also + * to activate the csi-assignments that possibly were + * de-activated because of this restart. The service unit + * waits in this state for an acknowledgement of the + * activation. + * + * 1.3 Actions + * =========== + * A1 - generate event comp_restart + * A2 - forward component restart request to the node which hosts current su + * A3 - + * A4 - + * A5 - + * A6 - + * A7 - + * A8 - + * A9 - order component to restart + * A10 - + * A11 - initiate setting of the same HA-state as was set before the restart + * A12 - + * A13 - + * A14 - + * A15 - + * A16 - + * A17 - + * A18 - defer the event + * A19 - recall deferred event + * A20 - restart all components contained in current su + * + * 1.4 Guards + * ========== + * C1 - + * C2 - + * C3 - the component has been restarted SaAmfSGCompRestartMax number of times + * C4 - + * C5 - for each si-assignment related to the restarting component where + * requested-ha-state != confirmed-ha-state + * C6 - + * C7 - all si-assignments related to the restarting component have + * requested-ha-state == confirmed-ha-state or the operation has failed + * C8 - + * C9 - + * C10 - all si-assignments related to current service unit have + * requested-ha-state == confirmed-ha-state or the operation has failed + * C11 - + * C12 - for each si-assignment related to current su where + * requested-ha-state != confirmed-ha-state * */ @@ -131,10 +294,14 @@ static int terminate_all_components_in_level (struct amf_su *su, SaUint32T current_instantiation_level); static int are_all_comps_in_level_uninst_or_term_failed (struct amf_su *su); +static int are_all_comps_in_level_uninst (struct amf_su *su); static int are_all_comps_in_level_instantiated (struct amf_su *su); static int instantiate_all_components_in_level (struct amf_su *su, SaUint32T current_instantiation_level); static SaUint32T su_lowest_comp_instantiation_level_set (struct amf_su *su); +static void si_ha_state_assumed_cbfn ( + struct amf_si_assignment *si_assignment, int result); +static int is_any_component_instantiating (amf_su_t *su); typedef struct su_event { amf_su_event_type_t event_type; @@ -143,13 +310,11 @@ typedef struct su_event { SaAmfRecommendedRecoveryT recommended_recovery; } su_event_t; -/** - * - * @param su - * @param comp - * @param su_event - * @param event_type - */ +/****************************************************************************** + * Internal (static) utility functions + *****************************************************************************/ + + static void su_event_set(struct amf_su *su, struct amf_comp *comp, SaAmfRecommendedRecoveryT recommended_recovery, su_event_t *su_event, amf_su_event_type_t event_type) @@ -166,6 +331,7 @@ static void su_defer_event (amf_su_t *su, amf_comp_t *comp, { su_event_t event; su_event_set(su, comp, recommended_recovery,&event, su_event_type); + ENTER("event_type = %d", event.event_type); amf_fifo_put (event.event_type, &event.su->deferred_events, sizeof (su_event_t), &event); @@ -221,6 +387,15 @@ static void clear_ha_state ( si_assignment->saAmfSISUHAState = 0; } +/** + * This function sets presence state to the specified value. It also has the + * following intentional side effects: + * - sets HA-state to unknown when presence state is set to UNINSTANTIATED + * - reports the change of presence state to the sg in which su is contained + * when the new state is 'stable' + * @param su + * @param presence_state - new value of presence state + */ static void su_presence_state_set (struct amf_su *su, SaAmfPresenceStateT presence_state) { @@ -243,6 +418,14 @@ static void su_presence_state_set (struct amf_su *su, } } +/** + * This function sets operational state to the specified value. It also has the + * following side effects: + * - sets the readiness state for su + * - sets the readiness state for all components contained in the su + * @param su + * @param oper_state - new value of operational state + */ void amf_su_operational_state_set (struct amf_su *su, SaAmfOperationalStateT oper_state) { @@ -266,6 +449,16 @@ void amf_su_operational_state_set (struct amf_su *su, } } +/** + * This function creates a new csi-assignment object and initializes it. The + * function also links the new csi-assignment object to the list of assignments + * held by the specified csi object, sets a pointer to the specified component + * and a pointer to the specified si-assignment. + * @param comp + * @param csi + * @param si_assignment + * @param ha_state - new value of ha-state + */ static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi, struct amf_si_assignment *si_assignment, SaAmfHAStateT ha_state) { @@ -302,55 +495,17 @@ static void comp_restart (struct amf_comp *comp) amf_comp_restart (comp); } -static void si_ha_state_assumed_cbfn ( - struct amf_si_assignment *si_assignment, int result) -{ - struct amf_si_assignment *tmp_si_assignment; - struct amf_comp *comp; - struct amf_csi_assignment *csi_assignment; - int all_confirmed = 1; - ENTER (""); - tmp_si_assignment = amf_su_get_next_si_assignment(si_assignment->su, NULL); - - while (tmp_si_assignment != NULL) { - for (comp = tmp_si_assignment->su->comp_head; comp != NULL; - comp = comp->next) { - - csi_assignment = amf_comp_get_next_csi_assignment(comp, NULL); - while (csi_assignment != NULL) { - - if (csi_assignment->requested_ha_state != - csi_assignment->saAmfCSICompHAState) { - all_confirmed = 0; - } - csi_assignment = amf_comp_get_next_csi_assignment( - comp, csi_assignment); - } - } - tmp_si_assignment = amf_su_get_next_si_assignment( - si_assignment->su, tmp_si_assignment); - } - - if (all_confirmed) { - switch (si_assignment->su->restart_control_state) { - case SU_RC_RESTART_COMP_SETTING: - log_printf (LOG_NOTICE, "Component restart recovery finished"); - break; - case SU_RC_RESTART_SU_SETTING: - log_printf (LOG_NOTICE, "SU restart recovery finished"); - break; - default: - assert (0); - break; - } - si_assignment->su->restart_control_state = - si_assignment->su->escalation_level_history_state; - su_recall_deferred_events (si_assignment->su); - - } -} - -static void reassign_sis(struct amf_su *su) +/** + * Set the same HA-state as the before the restart to the SI-assignments + * associated with current SU. As a side effect, this HA-state will also be set + * to all components which are associated with the csi-assignments associated to + * the specified su via its csi and si objects. + * @param su + * @param current_instantiation_level + * + * @return - 1 if there were no components on the specified instantiation level + */ +static void reassume_ha_state(struct amf_su *su) { struct amf_si_assignment *si_assignment; @@ -365,7 +520,6 @@ static void reassign_sis(struct amf_su *su) } } - static int is_any_component_instantiating (amf_su_t *su) { amf_comp_t *component; @@ -398,20 +552,6 @@ static int is_any_component_terminating (amf_su_t *su) return any_component_terminating; } -static int is_any_component_restarting (amf_su_t *su) -{ - amf_comp_t *component; - int any_component_terminating = 0; - for (component = su->comp_head; component != NULL; - component = component->next) { - if (component->saAmfCompPresenceState == - SA_AMF_PRESENCE_RESTARTING) { - any_component_terminating = 1; - break; - } - } - return any_component_terminating; -} static int is_any_comp_instantiation_failed (amf_su_t *su) { @@ -429,6 +569,18 @@ static int is_any_comp_instantiation_failed (amf_su_t *su) return comp_instantiation_failed; } +/** + * Finds the component within the specified su that has the highest value of it + * presence state. With current definition of values the highest value can also + * be regarded as the 'worst' in the sence of capability to be assigned + * workload. In the 'best' presence state (INSTANTIATED) the component is + * immediately available to take workload while in the 'worst' state + * (TERMINATION_FAILED) it can not take any workload before it has been manually + * repaired. + * @param su + * + * @return - worst presence state + */ static SaAmfPresenceStateT get_worst_comps_presence_state_in_su (amf_su_t *su) { amf_comp_t *component; @@ -443,12 +595,27 @@ static SaAmfPresenceStateT get_worst_comps_presence_state_in_su (amf_su_t *su) return worst_presence_state; } +/** + * + * @param su + */ +void su_history_state_set(struct amf_su *su, SaAmfPresenceStateT state) +{ + su->restart_control_state = su->escalation_level_history_state; + su->saAmfSUPresenceState = state; +} +/** + * A component notifies its parent su that its presence state has changed. + * @param su + * @param comp - component which has changed its presence state + * @param state - new value of presence state + */ static void su_comp_presence_state_changed (struct amf_su *su, struct amf_comp *comp, int state) { ENTER ("'%s', '%s' %d %d", su->name.value, comp->name.value, state, su->restart_control_state); - + amf_node_t *node = amf_node_find (&comp->su->saAmfSUHostedByNode); switch (state) { case SA_AMF_PRESENCE_INSTANTIATED: switch (su->restart_control_state) { @@ -475,21 +642,23 @@ static void su_comp_presence_state_changed (struct amf_su *su, break; case SU_RC_RESTART_COMP_RESTARTING: su->restart_control_state = SU_RC_RESTART_COMP_SETTING; - reassign_sis (comp->su); + reassume_ha_state (comp->su); break; case SU_RC_RESTART_SU_INSTANTIATING: - if (!is_any_component_restarting(su)) { + if (!is_any_component_instantiating(su)) { if (amf_su_are_all_comps_in_su ( comp->su, SA_AMF_PRESENCE_INSTANTIATED)) { su->restart_control_state = SU_RC_RESTART_SU_SETTING; su_presence_state_set (comp->su, SA_AMF_PRESENCE_INSTANTIATED); - reassign_sis (comp->su); + reassume_ha_state (comp->su); } else { if (is_any_comp_instantiation_failed (su)) { su_presence_state_set (comp->su, SA_AMF_PRESENCE_INSTANTIATION_FAILED); } else { + TRACE1("%s,%s",comp->su->name.value, + comp->name.value); assert (0); } } @@ -502,15 +671,44 @@ static void su_comp_presence_state_changed (struct amf_su *su, } break; case SA_AMF_PRESENCE_UNINSTANTIATED: - if (!is_any_component_terminating (su)) { - if (are_all_comps_in_level_uninst_or_term_failed (su)) { - if (terminate_all_components_in_level (su, - --su->current_comp_instantiation_level)) { - su_presence_state_set (su, - get_worst_comps_presence_state_in_su (su)); + switch (su->restart_control_state) { + case SU_RC_IDLE_ESCALATION_LEVEL_0: + case SU_RC_IDLE_ESCALATION_LEVEL_1: + case SU_RC_IDLE_ESCALATION_LEVEL_2: + + if (!is_any_component_terminating (su)) { + if (are_all_comps_in_level_uninst_or_term_failed (su)) { + if (terminate_all_components_in_level (su, + --su->current_comp_instantiation_level)) { + su_presence_state_set (su, + get_worst_comps_presence_state_in_su (su)); + } + } + } + break; + case SU_RC_RESTART_SU_INSTANTIATING: + break; + case SU_RC_RESTART_COMP_RESTARTING: + break; + case SU_RC_RESTART_SU_TERMINATING: + if (!is_any_component_terminating (su)) { + if (are_all_comps_in_level_uninst (su)) { + if (terminate_all_components_in_level (su, + --su->current_comp_instantiation_level)) { + su->restart_control_state = + SU_RC_RESTART_SU_INSTANTIATING; + instantiate_all_components_in_level (su, + su_lowest_comp_instantiation_level_set (su)); + } + } else { + su_history_state_set (su, + SA_AMF_PRESENCE_TERMINATION_FAILED); + } } - } - } + break; + default: + break; + } break; case SA_AMF_PRESENCE_INSTANTIATING: su_presence_state_set (comp->su,SA_AMF_PRESENCE_INSTANTIATING); @@ -529,20 +727,30 @@ static void su_comp_presence_state_changed (struct amf_su *su, SA_AMF_PRESENCE_INSTANTIATION_FAILED); } break; - case SU_RC_RESTART_COMP_RESTARTING: - su->restart_control_state = - su->escalation_level_history_state; - - su_presence_state_set (comp->su, - SA_AMF_PRESENCE_INSTANTIATION_FAILED); - break; + case SU_RC_RESTART_COMP_RESTARTING: + if (!is_any_component_instantiating (su)) { + if (node->saAmfNodeRebootOnInstantiationFailure) { + su_history_state_set (su, + AMF_PRESENCE_TERMINATION_FAILED_REBOOT); + amf_node_failover(node); + }else{ + su_history_state_set (su, + SA_AMF_PRESENCE_INSTANTIATION_FAILED); + amf_node_comp_failover_req(node, comp); + } + } + break; case SU_RC_RESTART_SU_INSTANTIATING: if (!is_any_component_instantiating (su)) { - su->restart_control_state = - su->escalation_level_history_state; - - su_presence_state_set (comp->su, - SA_AMF_PRESENCE_INSTANTIATION_FAILED); + if (node->saAmfNodeRebootOnInstantiationFailure) { + su_history_state_set (su, + AMF_PRESENCE_TERMINATION_FAILED_REBOOT); + amf_node_failover(node); + }else{ + su_history_state_set (su, + SA_AMF_PRESENCE_INSTANTIATION_FAILED); + amf_sg_failover_su_req(comp->su->sg, comp->su, node); + } } break; default: @@ -559,43 +767,30 @@ static void su_comp_presence_state_changed (struct amf_su *su, case SU_RC_IDLE_ESCALATION_LEVEL_0: case SU_RC_IDLE_ESCALATION_LEVEL_1: case SU_RC_IDLE_ESCALATION_LEVEL_2: - if (!is_any_component_terminating (su)) { - if (are_all_comps_in_level_uninst_or_term_failed (su)) { - if (terminate_all_components_in_level (su, - --su->current_comp_instantiation_level)) { - su_presence_state_set (su, - get_worst_comps_presence_state_in_su (su)); - } - } - } break; case SU_RC_RESTART_COMP_RESTARTING: - su->restart_control_state = - su->escalation_level_history_state; - - su_presence_state_set (comp->su, - SA_AMF_PRESENCE_TERMINATION_FAILED); - - break; case SU_RC_RESTART_SU_INSTANTIATING: - /* - * TODO Reconsider SU restart control concerning - * TERMINATING and INSANITATION - */ - case SU_RC_RESTART_SU_TERMINATING: - if (!is_any_component_terminating (su)) { - su->restart_control_state = - su->escalation_level_history_state; - - su_presence_state_set (comp->su, + if (!node->saAmfNodeRebootOnInstantiationFailure) { + su_history_state_set (su, SA_AMF_PRESENCE_TERMINATION_FAILED); + } else { + /* TODO Implement and request Node Failed Fast */ + ; + } + break; + case SU_RC_RESTART_SU_TERMINATING: + if (!node->saAmfNodeRebootOnInstantiationFailure) { + su_history_state_set (su, + SA_AMF_PRESENCE_TERMINATION_FAILED); + } else { + /* TODO Implement and request Node Failed Fast */ + ; } break; default: assert (0); break; } - break; default: assert (0); @@ -603,6 +798,12 @@ static void su_comp_presence_state_changed (struct amf_su *su, } } +/** + * A component notifies its parent su that its operational state has changed. + * @param su + * @param comp - component which has changed its operational state + * @param state - new value of operational state + */ static void su_comp_op_state_changed ( struct amf_su *su, struct amf_comp *comp, int state) { @@ -642,9 +843,11 @@ static void su_comp_op_state_changed ( } /** - * + * Instantiates all components on specified instantiation level. * @param su - * @param comp + * @param current_instantiation_level + * + * @return - 1 if there were no components on the specified instantiation level */ static int instantiate_all_components_in_level (struct amf_su *su, SaUint32T current_instantiation_level) @@ -700,169 +903,6 @@ static int are_all_comps_in_level_uninst_or_term_failed( return all; } - -int amf_su_are_all_comps_in_su (struct amf_su *su, - SaAmfPresenceStateT state) -{ - int all_comps_in_su_are_set = 1; - amf_comp_t *component; - for (component = su->comp_head; component != NULL; - component = component->next) { - - if (component->saAmfCompPresenceState != state) { - all_comps_in_su_are_set = 0; - } - } - return all_comps_in_su_are_set; -} - -void amf_su_restart (struct amf_su *su) -{ - struct amf_comp *comp; - SaNameT dn; - - ENTER ("'%s'", su->name.value); - - amf_su_dn_make (su, &dn); - log_printf (LOG_NOTICE, "Error detected for '%s', recovery " - "action: SU restart", dn.value); - - su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING; - su->restart_control_state = SU_RC_RESTART_SU_INSTANTIATING; - su->escalation_level_history_state = SU_RC_IDLE_ESCALATION_LEVEL_2; - - su->saAmfSURestartCount += 1; - - for (comp = su->comp_head; comp != NULL; comp = comp->next) { - amf_comp_restart (comp); - } -} - -int amf_su_instantiate (struct amf_su *su) -{ - ENTER ("'%s %d'", su->name.value, su->saAmfSUPresenceState); - - int performs_instantiating = 1; - - switch (su->saAmfSUPresenceState) { - case SA_AMF_PRESENCE_UNINSTANTIATED: - instantiate_all_components_in_level(su, - su_lowest_comp_instantiation_level_set (su)); - break; - case SA_AMF_PRESENCE_RESTARTING: - case SA_AMF_PRESENCE_INSTANTIATING: - break; - case SA_AMF_PRESENCE_INSTANTIATED: - case SA_AMF_PRESENCE_TERMINATING: - case SA_AMF_PRESENCE_INSTANTIATION_FAILED: - case SA_AMF_PRESENCE_TERMINATION_FAILED: - performs_instantiating = 0; - break; - default: - assert (0); - break; - - } - return performs_instantiating; -} - -amf_si_assignment_t *amf_su_assign_si (struct amf_su *su, struct amf_si *si, - SaAmfHAStateT ha_state) -{ - struct amf_si_assignment *si_assignment; - - dprintf ("Creating SI '%s' to SU '%s' with hastate %s\n", - getSaNameT (&si->name), getSaNameT (&su->name), - amf_ha_state (ha_state)); - - si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); - amf_su_dn_make (su, &si_assignment->name); - si_assignment->saAmfSISUHAState = 0; /* undefined confirmed HA state */ - si_assignment->requested_ha_state = ha_state; - si_assignment->next = si->assigned_sis; - si->assigned_sis = si_assignment; - si_assignment->si = si; - si_assignment->su = su; - - { - struct amf_csi *csi; - struct amf_comp *comp; - SaNameT *cs_type; - int i; - - /* - ** for each component in SU, find a CSI in the SI with the same type - */ - for (comp = su->comp_head; comp != NULL; comp = comp->next) { - int no_of_cs_types = 0; - for (i = 0; comp->saAmfCompCsTypes[i]; i++) { - cs_type = comp->saAmfCompCsTypes[i]; - no_of_cs_types++; - int no_of_assignments = 0; - - for (csi = si->csi_head; csi != NULL; csi = csi->next) { - if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value, - cs_type->length)) { - comp_assign_csi (comp, csi, si_assignment, ha_state); - no_of_assignments++; - } - } - if (no_of_assignments == 0) { - log_printf ( - LOG_WARNING, "\t No CSIs of type %s configured?!!\n", - getSaNameT (cs_type)); - } - } - if (no_of_cs_types == 0) { - log_printf (LOG_LEVEL_ERROR, - "\t No CS types configured for comp %s ?!!\n", - getSaNameT (&comp->name)); - } - } - } - return si_assignment; -} - - -/** - * Used by a component to report a state change event - * @param su - * @param comp - * @param type type of state - * @param state new state - */ -void amf_su_comp_state_changed ( - struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state) -{ - switch (type) { - case SA_AMF_PRESENCE_STATE: - su_comp_presence_state_changed (su, comp, state); - break; - case SA_AMF_OP_STATE: - su_comp_op_state_changed (su, comp, state); - break; - default: - assert (0); - break; - } -} - -/** - * Determine if the SU is hosted on the local node. - * @param su - * - * @return int - */ -int amf_su_is_local (struct amf_su *su) -{ - if (name_match (&this_amf_node->name, &su->saAmfSUHostedByNode)) { - return 1; - } else { - return 0; - } -} - - static void su_rc_enter_idle_escalation_level_1 (amf_comp_t *component, SaAmfRecommendedRecoveryT recommended_recovery) { @@ -875,7 +915,6 @@ static void su_rc_enter_idle_escalation_level_1 (amf_comp_t *component, comp_restart (component); } } - static void su_rc_enter_idle_escalation_level_2 (amf_comp_t *component, SaAmfRecommendedRecoveryT recommended_recovery) { @@ -884,7 +923,132 @@ static void su_rc_enter_idle_escalation_level_2 (amf_comp_t *component, amf_node_t *node = amf_node_find (&component->su->saAmfSUHostedByNode); amf_node_comp_restart_req (node, component); } +static int get_instantiation_max_level (amf_su_t *su) +{ + amf_comp_t *comp; + int instantiation_level = 0; + for (comp = su->comp_head; comp != NULL; comp = comp->next) { + if (comp->saAmfCompInstantiationLevel > instantiation_level) { + instantiation_level = comp->saAmfCompInstantiationLevel; + } + } + return instantiation_level; +} +/** + * Initiates the termination of all components which have the specified + * instantiation level. + * @param su + * @param current_instantiation_level + * + * @return int -1 if no component has the specified instantiation level + */ +static int terminate_all_components_in_level (struct amf_su *su, + SaUint32T current_instantiation_level) +{ + amf_comp_t *comp; + int all_components_in_level = 1; + for (comp = su->comp_head; comp != NULL; comp = comp->next) { + /* + * Terminate all components in instantiation level in SU + * abruptly. + */ + if (comp->saAmfCompInstantiationLevel == current_instantiation_level) { + amf_comp_error_suspected_set (comp); + amf_comp_terminate (comp); + all_components_in_level = 0; + } + } + return all_components_in_level; +} +/** + * su_current_instantiation_level_init + * @param su + * + * @return SaUint32T - the value of the instantiation level which has been set + */ +static SaUint32T su_lowest_comp_instantiation_level_set (struct amf_su *su) +{ + amf_comp_t *component = su->comp_head; + int comp_instantiation_level = component->saAmfCompInstantiationLevel; + for (; component != NULL; component = component->next) { + TRACE1("component->saAmfCompInstantiationLevel=%d", + component->saAmfCompInstantiationLevel); + if (component->saAmfCompInstantiationLevel < + comp_instantiation_level) { + comp_instantiation_level = + component->saAmfCompInstantiationLevel; + } + } + su->current_comp_instantiation_level = comp_instantiation_level; + return comp_instantiation_level; +} + +static int are_all_comps_in_level_uninst ( + struct amf_su *su) +{ + SaUint32T level = su->current_comp_instantiation_level; + amf_comp_t *comp; + int all = 1; + + for (comp = su->comp_head; comp != NULL; comp = comp->next) { + if (level == comp->saAmfCompInstantiationLevel) { + if (comp->saAmfCompPresenceState != SA_AMF_PRESENCE_UNINSTANTIATED) { + all = 0; + break; + } + } + } + return all; +} + + +/** + * An order to SU to instantiate its components. + * @param su + * + * @return int - 1 if its state allows it to request its contained components to + * instantiate or its state indicates that its components are in + * the process of instantiation. + */ +int amf_su_instantiate (struct amf_su *su) +{ + int is_instantiating = 1; + + ENTER ("'%s %d'", su->name.value, su->saAmfSUPresenceState); + switch (su->saAmfSUPresenceState) { + case SA_AMF_PRESENCE_UNINSTANTIATED: + instantiate_all_components_in_level(su, + su_lowest_comp_instantiation_level_set (su)); + break; + case SA_AMF_PRESENCE_RESTARTING: + case SA_AMF_PRESENCE_INSTANTIATING: + break; + case SA_AMF_PRESENCE_INSTANTIATED: + case SA_AMF_PRESENCE_TERMINATING: + case SA_AMF_PRESENCE_INSTANTIATION_FAILED: + case SA_AMF_PRESENCE_TERMINATION_FAILED: + is_instantiating = 0; + break; + default: + assert (0); + break; + + } + return is_instantiating; +} + +/** + * An order to SU to terminate its components. + * @param su + */ +void amf_su_terminate (struct amf_su *su) +{ + ENTER ("'%s'", su->name.value); + su->current_comp_instantiation_level = get_instantiation_max_level (su); + + terminate_all_components_in_level (su, su->current_comp_instantiation_level); +} /** * Called by a component to report a suspected error on a component @@ -954,211 +1118,118 @@ void amf_su_comp_error_suspected ( } } +/** + * An order to SU to unconditionally restart itself. + * @param su + */ +void amf_su_restart (struct amf_su *su) +{ + struct amf_comp *comp; + SaNameT dn; + + ENTER ("'%s'", su->name.value); + + amf_su_dn_make (su, &dn); + log_printf (LOG_NOTICE, "Error detected for '%s', recovery " + "action: SU restart", dn.value); + + /* + * TODO: Find out what the three lines below means ! + */ + su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING; + su->restart_control_state = SU_RC_RESTART_SU_TERMINATING; + su->escalation_level_history_state = SU_RC_IDLE_ESCALATION_LEVEL_2; + su->saAmfSURestartCount += 1; + + for (comp = su->comp_head; comp != NULL; comp = comp->next) { + amf_comp_terminate (comp); + } +} + +/****************************************************************************** + * Event response methods + *****************************************************************************/ + +/** + * Used by a component to report a state change event + * @param su + * @param comp + * @param type type of state + * @param state new state + */ +void amf_su_comp_state_changed ( + struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state) +{ + switch (type) { + case SA_AMF_PRESENCE_STATE: + su_comp_presence_state_changed (su, comp, state); + break; + case SA_AMF_OP_STATE: + su_comp_op_state_changed (su, comp, state); + break; + default: + assert (0); + break; + } +} + +static void si_ha_state_assumed_cbfn ( + struct amf_si_assignment *si_assignment, int result) +{ + struct amf_si_assignment *tmp_si_assignment; + struct amf_comp *comp; + struct amf_csi_assignment *csi_assignment; + int all_confirmed = 1; + ENTER (""); + tmp_si_assignment = amf_su_get_next_si_assignment(si_assignment->su, NULL); + + while (tmp_si_assignment != NULL) { + for (comp = tmp_si_assignment->su->comp_head; comp != NULL; + comp = comp->next) { + + csi_assignment = amf_comp_get_next_csi_assignment(comp, NULL); + while (csi_assignment != NULL) { + + if (csi_assignment->requested_ha_state != + csi_assignment->saAmfCSICompHAState) { + all_confirmed = 0; + } + csi_assignment = amf_comp_get_next_csi_assignment( + comp, csi_assignment); + } + } + tmp_si_assignment = amf_su_get_next_si_assignment( + si_assignment->su, tmp_si_assignment); + } + + if (all_confirmed) { + switch (si_assignment->su->restart_control_state) { + case SU_RC_RESTART_COMP_SETTING: + log_printf (LOG_NOTICE, "Component restart recovery finished"); + break; + case SU_RC_RESTART_SU_SETTING: + log_printf (LOG_NOTICE, "SU restart recovery finished"); + break; + default: + assert (0); + break; + } + si_assignment->su->restart_control_state = + si_assignment->su->escalation_level_history_state; + su_recall_deferred_events (si_assignment->su); + + } +} + +/****************************************************************************** + * General methods + *****************************************************************************/ + void amf_su_init (void) { log_init ("AMF"); } -static int get_instantiation_max_level (amf_su_t *su) -{ - amf_comp_t *comp; - int instantiation_level = 0; - for (comp = su->comp_head; comp != NULL; comp = comp->next) { - if (comp->saAmfCompInstantiationLevel > instantiation_level) { - instantiation_level = comp->saAmfCompInstantiationLevel; - } - } - return instantiation_level; -} - - - -/** - * - * @param su - * @param comp - */ -static int terminate_all_components_in_level (struct amf_su *su, - SaUint32T current_instantiation_level) -{ - amf_comp_t *comp; - int all_components_in_level = 1; - for (comp = su->comp_head; comp != NULL; comp = comp->next) { - /* - * Terminate all components in instantiation level in SU - * abruptly. - */ - if (comp->saAmfCompInstantiationLevel == current_instantiation_level) { - amf_comp_error_suspected_set (comp); - amf_comp_terminate (comp); - all_components_in_level = 0; - } - } - return all_components_in_level; -} - - - -/** - * su_current_instantiation_level_init - * @param su - */ -static SaUint32T su_lowest_comp_instantiation_level_set (struct amf_su *su) -{ - amf_comp_t *component = su->comp_head; - int comp_instantiation_level = component->saAmfCompInstantiationLevel; - for (; component != NULL; component = component->next) { - TRACE1("component->saAmfCompInstantiationLevel=%d", - component->saAmfCompInstantiationLevel); - - if (component->saAmfCompInstantiationLevel < - comp_instantiation_level) { - comp_instantiation_level = - component->saAmfCompInstantiationLevel; - } - } - su->current_comp_instantiation_level = comp_instantiation_level; - return comp_instantiation_level; -} - - -void amf_su_terminate (struct amf_su *su) -{ - ENTER ("'%s'", su->name.value); - su->current_comp_instantiation_level = get_instantiation_max_level (su); - - terminate_all_components_in_level (su, su->current_comp_instantiation_level); -} - -char *amf_su_dn_make (struct amf_su *su, SaNameT *name) -{ - int i; - - assert (su != NULL); - - i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, - "safSu=%s,safSg=%s,safApp=%s", - su->name.value, su->sg->name.value, su->sg->application->name.value); - assert (i <= SA_MAX_NAME_LENGTH); - name->length = i; - return (char *)name->value; -} - -struct amf_si_assignment *amf_su_get_next_si_assignment ( - struct amf_su *su, const struct amf_si_assignment *si_assignment) -{ - struct amf_si *si; - struct amf_si_assignment *tmp_si_assignment; - SaNameT dn; - - amf_su_dn_make (su, &dn); - - if (si_assignment == NULL) { - assert (su->sg); - assert (su->sg->application); - assert (su->sg->application->si_head); - si = su->sg->application->si_head; - tmp_si_assignment = si->assigned_sis; - } else { - tmp_si_assignment = si_assignment->next; - if (tmp_si_assignment == NULL) { - si = si_assignment->si->next; - if (si == NULL) { - return NULL; - } else { - tmp_si_assignment = si->assigned_sis; - } - } else { - si = tmp_si_assignment->si; - } - } - - for (; si != NULL; si = si->next) { - if (tmp_si_assignment == NULL && si != NULL) { - tmp_si_assignment = si->assigned_sis; - } - for (; tmp_si_assignment != NULL; - tmp_si_assignment = tmp_si_assignment->next) { - - if (name_match (&tmp_si_assignment->name, &dn)) { - return tmp_si_assignment; - } - } - } - - return NULL; -} - -void amf_su_foreach_si_assignment ( - struct amf_su *su, - void (*foreach_fn)(struct amf_su *su, - struct amf_si_assignment *si_assignment)) -{ - struct amf_si_assignment *si_assignment; - - assert (foreach_fn != NULL); - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - foreach_fn (su, si_assignment); - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); - } -} - -int amf_su_get_saAmfSUNumCurrActiveSIs(struct amf_su *su) -{ - int cnt = 0; - struct amf_si_assignment *si_assignment; - - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - if (su->sg->avail_state == SG_AC_AssigningOnRequest && - si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) { - cnt++; - } else { - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { - cnt++; - } - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); - } - - return cnt; -} - -int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su) -{ - int cnt = 0; - struct amf_si_assignment *si_assignment; - - si_assignment = amf_su_get_next_si_assignment (su, NULL); - while (si_assignment != NULL) { - if (su->sg->avail_state == SG_AC_AssigningOnRequest && - si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) { - cnt++; - } else { - if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { - cnt++; - } - } - si_assignment = amf_su_get_next_si_assignment (su, si_assignment); - } - - return cnt; -} - -SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su) -{ - if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) && - ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) || - (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { - return SA_AMF_READINESS_IN_SERVICE; - } else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) { - return SA_AMF_READINESS_STOPPING; - } else { - return SA_AMF_READINESS_OUT_OF_SERVICE; - } -} - /** * Constructor for SU objects. Adds SU last in the ordered * list owned by the specified SG. Always returns a @@ -1334,3 +1405,268 @@ end: return su; } +/** + * This function makes a distinguished name for specified su object. + * @param su + * @param name -[out] pointer to where the distinguished name shall be stored + * + * @return SaNameT* - distinguished name + */ +char *amf_su_dn_make (struct amf_su *su, SaNameT *name) +{ + int i; + + assert (su != NULL); + + i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH, + "safSu=%s,safSg=%s,safApp=%s", + su->name.value, su->sg->name.value, su->sg->application->name.value); + assert (i <= SA_MAX_NAME_LENGTH); + name->length = i; + return (char *)name->value; +} + +/** + * An order to SU to create an si-assignment object with a specified HA-state + * between it self and a specified si. The created si-assignment is initialized + * and linked to list of assignments held by the specified si. + * This function also orders creation of all csi-assignments required + * considering the cs-types specified for the components and csi objects + * respectively. + * @param su + * @param si + * @param ha_state + * + * @return amf_si_assignment_t* + */ +amf_si_assignment_t *amf_su_assign_si (struct amf_su *su, struct amf_si *si, + SaAmfHAStateT ha_state) +{ + struct amf_si_assignment *si_assignment; + + dprintf ("Creating SI '%s' to SU '%s' with hastate %s\n", + getSaNameT (&si->name), getSaNameT (&su->name), + amf_ha_state (ha_state)); + + si_assignment = amf_malloc (sizeof (struct amf_si_assignment)); + amf_su_dn_make (su, &si_assignment->name); + si_assignment->saAmfSISUHAState = 0; /* undefined confirmed HA state */ + si_assignment->requested_ha_state = ha_state; + si_assignment->next = si->assigned_sis; + si->assigned_sis = si_assignment; + si_assignment->si = si; + si_assignment->su = su; + + { + struct amf_csi *csi; + struct amf_comp *comp; + SaNameT *cs_type; + int i; + + /* + ** for each component in SU, find a CSI in the SI with the same type + */ + for (comp = su->comp_head; comp != NULL; comp = comp->next) { + int no_of_cs_types = 0; + for (i = 0; comp->saAmfCompCsTypes[i]; i++) { + cs_type = comp->saAmfCompCsTypes[i]; + no_of_cs_types++; + int no_of_assignments = 0; + + for (csi = si->csi_head; csi != NULL; csi = csi->next) { + if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value, + cs_type->length)) { + comp_assign_csi (comp, csi, si_assignment, ha_state); + no_of_assignments++; + } + } + if (no_of_assignments == 0) { + log_printf ( + LOG_WARNING, "\t No CSIs of type %s configured?!!\n", + getSaNameT (cs_type)); + } + } + if (no_of_cs_types == 0) { + log_printf (LOG_LEVEL_ERROR, + "\t No CS types configured for comp %s ?!!\n", + getSaNameT (&comp->name)); + } + } + } + return si_assignment; +} + +struct amf_si_assignment *amf_su_get_next_si_assignment ( + struct amf_su *su, const struct amf_si_assignment *si_assignment) +{ + struct amf_si *si; + struct amf_si_assignment *tmp_si_assignment; + SaNameT dn; + + amf_su_dn_make (su, &dn); + + if (si_assignment == NULL) { + assert (su->sg); + assert (su->sg->application); + assert (su->sg->application->si_head); + si = su->sg->application->si_head; + tmp_si_assignment = si->assigned_sis; + } else { + tmp_si_assignment = si_assignment->next; + if (tmp_si_assignment == NULL) { + si = si_assignment->si->next; + if (si == NULL) { + return NULL; + } else { + tmp_si_assignment = si->assigned_sis; + } + } else { + si = tmp_si_assignment->si; + } + } + + for (; si != NULL; si = si->next) { + if (tmp_si_assignment == NULL && si != NULL) { + tmp_si_assignment = si->assigned_sis; + } + for (; tmp_si_assignment != NULL; + tmp_si_assignment = tmp_si_assignment->next) { + + if (name_match (&tmp_si_assignment->name, &dn)) { + return tmp_si_assignment; + } + } + } + + return NULL; +} + +void amf_su_foreach_si_assignment ( + struct amf_su *su, + void (*foreach_fn)(struct amf_su *su, + struct amf_si_assignment *si_assignment)) +{ + struct amf_si_assignment *si_assignment; + + assert (foreach_fn != NULL); + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + foreach_fn (su, si_assignment); + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } +} + +/** + * This function calculates the number of si-assignments with active HA-state + * which currently are associated with the specified su. + * TODO: Split into two functions and remove dependency to sg's avail_state + * @param su + * + * @return int + */ +int amf_su_get_saAmfSUNumCurrActiveSIs(struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (su->sg->avail_state == SG_AC_AssigningOnRequest && + si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) { + cnt++; + } else { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) { + cnt++; + } + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + + return cnt; +} + +/** + * This function calculates the number of si-assignments with standby HA-state + * which currently are associated with the specified su. + * TODO: Split into two functions and remove dependency to sg's avail_state + * @param su + * + * @return int + */ +int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su) +{ + int cnt = 0; + struct amf_si_assignment *si_assignment; + + si_assignment = amf_su_get_next_si_assignment (su, NULL); + while (si_assignment != NULL) { + if (su->sg->avail_state == SG_AC_AssigningOnRequest && + si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) { + cnt++; + } else { + if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) { + cnt++; + } + } + si_assignment = amf_su_get_next_si_assignment (su, si_assignment); + } + + return cnt; +} + +/** + * This function calculates the readiness state for specified su + * @param su + * + * @return SaAmfReadinessStateT + */ +SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su) +{ + if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) && + ((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) || + (su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) { + return SA_AMF_READINESS_IN_SERVICE; + } else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) { + return SA_AMF_READINESS_STOPPING; + } else { + return SA_AMF_READINESS_OUT_OF_SERVICE; + } +} + +/** + * Determine if the SU is hosted on the local node. + * @param su + * + * @return int + */ +int amf_su_is_local (struct amf_su *su) +{ + if (name_match (&this_amf_node->name, &su->saAmfSUHostedByNode)) { + return 1; + } else { + return 0; + } +} + +/** + * Determine if all components have the specified HA-state. + * @param su + * @param state -specified HA-state + * + * @return int - return 0 if not all components have the specified HA-state + */ +int amf_su_are_all_comps_in_su (struct amf_su *su, + SaAmfPresenceStateT state) +{ + int all_comps_in_su_are_set = 1; + amf_comp_t *component; + for (component = su->comp_head; component != NULL; + component = component->next) { + + if (component->saAmfCompPresenceState != state) { + all_comps_in_su_are_set = 0; + } + } + return all_comps_in_su_are_set; +} +