mirror of
				https://git.proxmox.com/git/mirror_corosync
				synced 2025-11-04 08:24:07 +00:00 
			
		
		
		
	git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1541 fd59a12c-fef9-0310-b244-a6a79926bd2f
		
			
				
	
	
		
			1803 lines
		
	
	
		
			60 KiB
		
	
	
	
		
			C
		
	
	
	
	
	
			
		
		
	
	
			1803 lines
		
	
	
		
			60 KiB
		
	
	
	
		
			C
		
	
	
	
	
	
/** @file exec/amfsu.c
 | 
						|
 * 
 | 
						|
 * Copyright (c) 2002-2006 MontaVista Software, Inc.
 | 
						|
 * Author: Steven Dake (sdake@redhat.com)
 | 
						|
 *
 | 
						|
 * Copyright (c) 2006 Ericsson AB.
 | 
						|
 * Author: Hans Feldt, Anders Eriksson, Lars Holm
 | 
						|
 * - Introduced AMF B.02 information model
 | 
						|
 * - Use DN in API and multicast messages
 | 
						|
 * - (Re-)Introduction of event based multicast messages
 | 
						|
 * - Refactoring of code into several AMF files
 | 
						|
 * - Component/SU restart, SU failover
 | 
						|
 * - Constructors/destructors
 | 
						|
 * - Serializers/deserializers
 | 
						|
 *
 | 
						|
 * All rights reserved.
 | 
						|
 *
 | 
						|
 *
 | 
						|
 * This software licensed under BSD license, the text of which follows:
 | 
						|
 * 
 | 
						|
 * Redistribution and use in source and binary forms, with or without
 | 
						|
 * modification, are permitted provided that the following conditions are met:
 | 
						|
 *
 | 
						|
 * - Redistributions of source code must retain the above copyright notice,
 | 
						|
 *   this list of conditions and the following disclaimer.
 | 
						|
 * - Redistributions in binary form must reproduce the above copyright notice,
 | 
						|
 *   this list of conditions and the following disclaimer in the documentation
 | 
						|
 *   and/or other materials provided with the distribution.
 | 
						|
 * - Neither the name of the MontaVista Software, Inc. nor the names of its
 | 
						|
 *   contributors may be used to endorse or promote products derived from this
 | 
						|
 *   software without specific prior written permission.
 | 
						|
 *
 | 
						|
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 | 
						|
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 | 
						|
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 | 
						|
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 | 
						|
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 | 
						|
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 | 
						|
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 | 
						|
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 | 
						|
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 | 
						|
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 | 
						|
 * THE POSSIBILITY OF SUCH DAMAGE.
 | 
						|
 * 
 | 
						|
 * AMF Service Unit Class Implementation
 | 
						|
 * 
 | 
						|
 * This file contains functions for handling AMF-service units(SUs). It can be 
 | 
						|
 * viewed as the implementation of the AMF Service Unit class (called SU)
 | 
						|
 * as described in SAI-Overview-B.02.01. The SA Forum specification 
 | 
						|
 * SAI-AIS-AMF-B.02.01 has been used as specification of the behaviour
 | 
						|
 * and is referred to as 'the spec' below.
 | 
						|
 * 
 | 
						|
 * The functions in this file are responsible for:
 | 
						|
 *	- instantiating and terminating service units on request
 | 
						|
 *	  (considering the dependencies between components described in paragraph
 | 
						|
 *	   3.9.2)
 | 
						|
 *	- creating and deleting CSI-assignment objects between its components and
 | 
						|
 *	  CSI-objects upon request
 | 
						|
 *	- receiving error reports from its components and forwarding them to
 | 
						|
 *	  appropriate handler (SU or SG or node or cluster)
 | 
						|
 *	- implementing restart of itself and its components (paragraph 3.12.1.2)
 | 
						|
 *	- implementing error escallation level 1 (paragraph 3.12.2.2 in the spec)
 | 
						|
 *	- handling all run time attributes of the AMF SU; all cached
 | 
						|
 *	  attributes are stored as variables and sent to the IMM service
 | 
						|
 *	  upon the changes described in the specification.
 | 
						|
 *
 | 
						|
 * SU contains the following state machines:
 | 
						|
 *	- presence state machine (PRSM)
 | 
						|
 *	- administrative state machine (ADSM) (NOT IN THIS RELEASE)
 | 
						|
 *	- operational state machine (OPSM)
 | 
						|
 *	- readiness state machine (RESM)
 | 
						|
 *	- ha state per service instance (SI)
 | 
						|
 *	- restart control state machine (RCSM)
 | 
						|
 *
 | 
						|
 * The presence state machine orders intantiation of its components on request.
 | 
						|
 * It fully respects the dependency rules between components at instantiation
 | 
						|
 * such that it orders instantiation simultaneously only of components on the
 | 
						|
 * same instantiation level. The presence state machine is implemented with
 | 
						|
 * the states described in the spec and the state transitions are trigged by
 | 
						|
 * reported state transitions from its contained components according to
 | 
						|
 * paragraph 3.3.1.1.
 | 
						|
 *
 | 
						|
 * The operational state machine is not responsible for any control function. 
 | 
						|
 * It assumes the DISABLED state if an incoming operational state change report
 | 
						|
 * from a component indicates the component has assumed the DISABLED state.
 | 
						|
 * Operational state changes are reported to IMM.
 | 
						|
 *
 | 
						|
 * The readiness state machine is not used for any control but is updated and
 | 
						|
 * reported to IMM when it is changed.
 | 
						|
 *
 | 
						|
 * The restart control state machine (RCSM) is used to implement level 1 of
 | 
						|
 * the error escallation policy described in chapter 3.12.2 of the spec. It also
 | 
						|
 * implements component restart and service unit restart as described in
 | 
						|
 * paragraph 3.12.1.2 and 3.12.1.3.
 | 
						|
 * RCSM contains three composite states.
 | 
						|
 * Being a composite state means that the state contains substates.
 | 
						|
 * RCSM composite states are:
 | 
						|
 *  - IDLE (LEVEL_0, LEVEL_1 and LEVEL_2)
 | 
						|
 *	- RESTARTING_COMPONENT (DEACTIVATING, RESTARTING, SETTING and ACTIVATING)
 | 
						|
 *	- RESTARTING_SERVICE_UNIT (DEACTIVATING, TERMINATING, INSTANTIATING,
 | 
						|
 *                             and ACTIVATING)
 | 
						|
 * 
 | 
						|
 * IDLE is a kind of state where no actions are performed and used only to
 | 
						|
 * remember the escallation level. Substate LEVEL_0 indicates no escallation.
 | 
						|
 * LEVEL_1 indicates that a component restart has been executed recently and the
 | 
						|
 * escallation timer is still running. At this level component restart requests
 | 
						|
 * will transition to RESTARTING_COMPONENT but if there are too many restart
 | 
						|
 * requests before the probation timer expires then a transition will be made to
 | 
						|
 * LEVEL_2 and the restart request will be forwarded to the node instance
 | 
						|
 * hosting this component. State RESTARTING_SERVICE_UNIT will only be assumed if
 | 
						|
 * the node explicitly requests the SU to execute a restart of itself (after
 | 
						|
 * having evaluated its part of the error escallation policy).
 | 
						|
 * 
 | 
						|
* 1. Service Unit Restart Control State Machine
 | 
						|
*  ============================================
 | 
						|
 * 
 | 
						|
 * 1.1  State Transition Table
 | 
						|
 * ===========================
 | 
						|
 * 
 | 
						|
 * State:                  Event:                 Action:  New state:
 | 
						|
 * ===========================================================================
 | 
						|
 * IDLE_ESCALATION_x       comp_restart           A9	   RS_COMP_RESTARTING
 | 
						|
 * IDLE_ESCALATION_x       su_restart             A20      RS_SU_TERMINATING
 | 
						|
 * IDLE_ESCALATION_0       error_suspected        A1,A3    IDLE_ESCALATION_1
 | 
						|
 * IDLE_ESCALATION_1       error_suspected [!C3]  A1,A3    IDLE_ESCALATION_1
 | 
						|
 * IDLE_ESCALATION_1       error_suspected [C3]   A2,A5    IDLE_ESCALATION_2
 | 
						|
 * IDLE_ESCALATION_2       error_suspected        A2       IDLE_ESCALATION_2
 | 
						|
 * RS_COMP_RESTARTING      comp_instantiated      A11      RS_COMP_SETTING
 | 
						|
 * RS_COMP_RESTARTING      comp_inst_failed       A14,A15  RS_COMP_T-ING_2
 | 
						|
 * RS_COMP_RESTARTING      comp_term_failed       A19      IDLE_ESCALATION_x
 | 
						|
 * RS_COMP_RESTARTING      error_suspected        A18      RS_COMP_RESTARTING
 | 
						|
 * RS_COMP_T-ING_2         comp_uninst..ed [C8]   A16,A15  RS_COMP_T-ING_2
 | 
						|
 * RS_COMP_T-ING_2         comp_uninst..ed [C100]          IDLE_ESCALATION_x
 | 
						|
 * RS_COMP_T-ING_2         comp_uninst..ed [C101] A25      IDLE_ESCALATION_x
 | 
						|
 * RS_COMP_T-ING_2         comp_uninst..ed [C102] A26      IDLE_ESCALATION_x
 | 
						|
 * RS_COMP_T-ING_2         comp_term_failed [C100]         RS_COMP_T-ING_2
 | 
						|
 * RS_COMP_T-ING_2         error_suspected        A18      RS_COMP_T-ING_2      
 | 
						|
 * RS_COMP_SETTING         ha_state_assumed [C7]  A19      IDLE_ESCALATION_x
 | 
						|
 * RS_COMP_SETTING         error_suspected        A18      RS_COMP_SETTING
 | 
						|
 * RS_SU_TERMINATING       comp_uninst..ed [C8]   A16,A15  RS_SU_TERMINATING
 | 
						|
 * RS_SU_TERMINATING       comp_uninst..ed [C103] A17,A23  RS_SU_INSTANTIATING
 | 
						|
 * RS_SU_TERMINATING       comp_uninst..ed [C104] A19      IDLE_ESCALATION_x
 | 
						|
 * RS_SU_TERMINATING       comp_term_failed [C104]A19      IDLE_ESCALATION_X
 | 
						|
 * RS_SU_TERMINATING       error_suspected        A18      RS_SU_TERMINATING
 | 
						|
 * RS_SU_INSTANTIATING     comp_instantiated [C14]A21,A22  RS_SU_INSTANTIATING
 | 
						|
 * RS_SU_INSTANTIATING     comp_instantiated [C105]A15     RS_SU_T-ING_2
 | 
						|
 * RS_SU_INSTANTIATING     comp_instantiated [C106]A11     RS_SU_SETTING
 | 
						|
 * RS_SU_INSTANTIATING     comp_inst_failed [C105]A15      RS_SU_T-ING_2
 | 
						|
 * RS_SU_INSTANTIATING     error_suspected        A18      RS_SU_INSTANTIATING
 | 
						|
 * RS_SU_T-ING_2           comp_uninst..ed [C8]   A16,A15  RS_SU_T-ING_2
 | 
						|
 * RS_SU_T-ING_2           comp_uninst..ed [C100]          IDLE_ESCALATION_x
 | 
						|
 * RS_SU_T-ING_2           comp_uninst..ed [C101] A25      IDLE_ESCALATION_x
 | 
						|
 * RS_SU_T-ING_2           comp_uninst..ed [C102] A26      IDLE_ESCALATION_x
 | 
						|
 * RS_SU_T-ING_2           comp_term_failed [C100]         RS_SU_T-ING_2
 | 
						|
 * RS_SU_T-ING_2           error_suspected        A18      RS_SU_T-ING_2
 | 
						|
 * RS_SU_SETTING           ha_state_assumed [C10] A19      IDLE_ESCALATION_X
 | 
						|
 * RS_SU_SETTING           error_suspected        A18      RS_SU_SETTING
 | 
						|
 * 
 | 
						|
 * 1.2 State Description
 | 
						|
 * =====================
 | 
						|
 * IDLE_ESCALATION_x - This is just an abbreviated notation for 
 | 
						|
 *                     IDLE_ESCALATION_0, IDLE_ESCALATION_1 or IDLE_ESCALATION_2
 | 
						|
 *                     When leaving any of the idle states, a history state
 | 
						|
 *                     is used to save the (exact) state value. When returning
 | 
						|
 *                     to idle, the value of the history state is used to set
 | 
						|
 *                     the correct idle state.
 | 
						|
 *
 | 
						|
 * IDLE_ESCALATION_0 - SU_RC_IDLE_ESCALATION_LEVEL_0
 | 
						|
 *                    Service unit is idle and the restart probation timer is
 | 
						|
 *					  off.
 | 
						|
 *
 | 
						|
 * IDLE_ESCALATION_1 - SU_RC_IDLE_ESCALATION_LEVEL_1
 | 
						|
 *                    Service unit is idle and the restart probation timer is
 | 
						|
 *                    on. This indicates there has recently been an error
 | 
						|
 *                    detected on at least one of its components which has been
 | 
						|
 *                    recovered by a component restart but we are still in the
 | 
						|
 *                    probation period which follows every restart.
 | 
						|
 *
 | 
						|
 * IDLE_ESCALATION_2 - SU_RC_IDLE_ESCALATION_LEVEL_2
 | 
						|
 *                    Service unit is idle and handling on potential new error
 | 
						|
 *                    indications on any of its components has been delegated
 | 
						|
 *                    to the node object where the service unit is hosted.
 | 
						|
 *
 | 
						|
 * RS_COMP_DEACTIVATING - SU_RC_RESTART_COMP_DEACTIVATING
 | 
						|
 *                       Service unit is busy handling restart of one of its
 | 
						|
 *                       components. In this sub-state, the service unit is
 | 
						|
 *                       waiting for acknowledgements that all components which
 | 
						|
 *                       had csi-assignments that were dependent of csi-
 | 
						|
 *                       assignments associated to the restarting component
 | 
						|
 *                       have been de-activated. This is a neccesary step to
 | 
						|
 *                       take before the component to restart is terminated,
 | 
						|
 *                       to avoid that the csi or si dependency rules are
 | 
						|
 *                       violated.
 | 
						|
 *
 | 
						|
 * RS_COMP_RESTARTING - SU_RC_RESTART_COMP_RESTARTING
 | 
						|
 *                     Service unit is busy handling restart of one of its
 | 
						|
 *                     components. In this sub-state, the service unit has
 | 
						|
 *                     ordered one of its components to restart and waits for
 | 
						|
 *                     the component to indicate that the restart is done.
 | 
						|
 * 
 | 
						|
 * RS_COMP_T-ING_2 - SU_RC_RESTART_COMP_TERMINATING_AFTER_INST_FAILED
 | 
						|
 *					Service unit is busy handling restart of one of its
 | 
						|
 *					components. In this sub-state, the restart of the component
 | 
						|
 *					has failed and the rest of the components in the service 
 | 
						|
 *					unit has to be terminated.
 | 
						|
 *
 | 
						|
 * RS_COMP_SETTING - SU_RC_RESTART_COMP_SETTING
 | 
						|
 *                  Service unit is busy handling restart of one of its
 | 
						|
 *                  components. In this sub-state, the service unit has ordered
 | 
						|
 *                  the component that just have been restarted to re-assume
 | 
						|
 *                  the HA-states it had before, provided none of the states
 | 
						|
 *                  were ACTIVE. It waits for an acknowledgement that the
 | 
						|
 *                  setting of the HA-states are done.
 | 
						|
 * 
 | 
						|
 * RS_COMP_ACTIVATING - SU_RC_RESTART_COMP_ACTIVATING
 | 
						|
 *                     Service unit is busy handling restart of one of its
 | 
						|
 *                     components. In this sub-state, the service unit has
 | 
						|
 *                     ordered the component that just have been restarted to
 | 
						|
 *                     re-assume the active HA-states it had before and also
 | 
						|
 *                     to activate the csi-assignments that possibly were
 | 
						|
 *                     de-activated because of this restart. The service unit
 | 
						|
 *                     waits in this state for an acknowledgement of the
 | 
						|
 *                     activation.
 | 
						|
 *
 | 
						|
 * RS_SU_DEACTIVATING - SU_RC_RESTART_SU_DEACTIVATING
 | 
						|
 *                     Service unit is busy handling restart of all of its
 | 
						|
 *                     components. In this sub-state, the service unit is
 | 
						|
 *                     waiting for acknowledgements that all components which
 | 
						|
 *                     had csi-assignments that were dependent of si-
 | 
						|
 *                     assignments associated to this service unit
 | 
						|
 *                     have been de-activated. This is a neccesary step to
 | 
						|
 *                     take before all components of the service unit are
 | 
						|
 *                     terminated, to avoid that the csi or si dependency rules
 | 
						|
 *                     are violated.
 | 
						|
 *
 | 
						|
 * RS_SU_TERMINATING - SU_RC_RESTART_SU_TERMINATING
 | 
						|
 *                    Service unit is busy handling restart of all of its
 | 
						|
 *                    components. In this sub-state, the service unit has
 | 
						|
 *                    ordered all its components to terminate and is waiting
 | 
						|
 *                    for an acknowledgement that all components are done with
 | 
						|
 *                    the termination.
 | 
						|
 * 
 | 
						|
 * RS_SU_INSTANTIATING - SU_RC_RESTART_SU_INSTANTIATING
 | 
						|
 *                    Service unit is busy handling restart of all of its
 | 
						|
 *                    components. In this sub-state, the service unit has
 | 
						|
 *                    ordered all components to instantiate and is waiting
 | 
						|
 *                    for an acknowledgement that all components are done with
 | 
						|
 *                    the instantiation.
 | 
						|
 *
 | 
						|
 * RS_SU_T-ING_2 - SU_RC_RESTART_SU_TERMINATING_AFTER_INST_FAILED
 | 
						|
 *					Service unit is busy handling restart of all of its
 | 
						|
 *					components. In this sub-state, the instantiation at least
 | 
						|
 *					one component has failed and the rest of the components in
 | 
						|
 *					the service unit has to be terminated.
 | 
						|
 *
 | 
						|
 * RS_SU_SETTING -  SU_RC_RESTART_SU_SETTING
 | 
						|
 *                  Service unit is busy handling restart of all of its
 | 
						|
 *                  components. In this sub-state, the service unit has ordered
 | 
						|
 *                  all components that just have been restarted to re-assume
 | 
						|
 *                  the HA-states they had before, provided none of the states
 | 
						|
 *                  were ACTIVE. The service unit waits for an acknowledgement
 | 
						|
 *                  that the setting of the HA-states are done.
 | 
						|
 *
 | 
						|
 * RS_SU_ACTIVATING - SU_RC_RESTART_SU_ACTIVATING
 | 
						|
 *                   Service unit is busy handling restart of all of its
 | 
						|
 *                   components. In this sub-state, the service unit has
 | 
						|
 *                   ordered all components that just have been restarted to
 | 
						|
 *                   re-assume the active HA-states they had before and also
 | 
						|
 *                   to activate the csi-assignments that possibly were
 | 
						|
 *                   de-activated because of this restart. The service unit
 | 
						|
 *                   waits in this state for an acknowledgement of the
 | 
						|
 *                   activation.
 | 
						|
 *
 | 
						|
 * 1.3 Actions
 | 
						|
 * ===========
 | 
						|
 * A1  - generate event comp_restart
 | 
						|
 * A2  - forward component restart request to the node which hosts current su
 | 
						|
 * A3  - start probation timer (SaAmfSGCompRestartProb)
 | 
						|
 * A4  - [foreach component in su]/ cnt += SaAmfSGCompRestartCount
 | 
						|
 * A5  - stop probation timer
 | 
						|
 * A6  - restart ??
 | 
						|
 * A7  - set restarting_comp = component
 | 
						|
 * A8  - [foreach csi-assignment assigned to component] SI deactivate csi
 | 
						|
 * A9  - order component to restart
 | 
						|
 * A10 - set restarting_comp == ALL
 | 
						|
 * A11 - initiate setting of the same HA-state as was set before the restart
 | 
						|
 * A12 - SI activate
 | 
						|
 * A13 - [foreach si-assignment assigned to su] SI deactivate
 | 
						|
 * A14 - set current instantiation level = highest level 
 | 
						|
 * A15 - [foreach component on current instantiation level]/terminate component
 | 
						|
 * A16 - current instantiation level is decremented
 | 
						|
 * A17 - request the presence state state machine to instantiate the su
 | 
						|
 * A18 - defer the event
 | 
						|
 * A19 - recall deferred event
 | 
						|
 * A20 - restart all components contained in current su
 | 
						|
 * A21 - current instantiation level is incremented
 | 
						|
 * A22 - [foreach component on current instantiation level]/instantiate
 | 
						|
 *       component
 | 
						|
 * A23 - set current instantiation level = lowest level
 | 
						|
 * A24 - order SG to do component failover
 | 
						|
 * A25 - order Node to do node failover
 | 
						|
 * A26 - order SG to do SU failover
 | 
						|
 *
 | 
						|
 * 1.4 Guards
 | 
						|
 * ==========
 | 
						|
 * C1  - disableRestart == False
 | 
						|
 * C2  - the component has been restarted less than SaAmfSGCompRestartMax times
 | 
						|
 * C3  - the component has been restarted SaAmfSGCompRestartMax number of times
 | 
						|
 * C4  - all si-assignments have confirmed-ha-state == QUIESCED or the
 | 
						|
 *       operation failed flag set.
 | 
						|
 * C5  - for each si-assignment related to the restarting component where
 | 
						|
 *       requested-ha-state != confirmed-ha-state and requested-ha-state !=
 | 
						|
 *       ACTIVE
 | 
						|
 * C6  - - for each si-assignment related to the restarting component where
 | 
						|
 *       requested-ha-state != confirmed-ha-state and requested-ha-state ==
 | 
						|
 *       ACTIVE
 | 
						|
 * C7  - all si-assignments related to the restarting component have
 | 
						|
 *       requested-ha-state == confirmed-ha-state or has the operation failed
 | 
						|
 *       flag set
 | 
						|
 * C8  - all components on current instantiation level == UNINSTANTIATED
 | 
						|
 * C9  - current instantiation level < lowest instantiation level
 | 
						|
 * C10 - all si-assignments related to current service unit have
 | 
						|
 *       requested-ha-state == confirmed-ha-state or the operation failed
 | 
						|
 *       flag set.
 | 
						|
 * C11 - for each si-assignment related to current su where
 | 
						|
 *       requested-ha-state != confirmed-ha-state and requested-ha-state ==
 | 
						|
 *       ACTIVE
 | 
						|
 * C12 - for each si-assignment related to current su where
 | 
						|
 *       requested-ha-state != confirmed-ha-state and requested-ha-state ==
 | 
						|
 *       STANDBY
 | 
						|
 * C13 - at least one component has presence state == TERMINATION_FAILED
 | 
						|
 * C14 - all components on current instantiation level == INSTANTIATED,
 | 
						|
 *       INSTANTIATION_FAILED or INSTANTIATION_FAILED_REBOOT_NODE
 | 
						|
 * C15 - current instantiation level is highest
 | 
						|
 * C16 - all components has presence state == INSTANTIATED
 | 
						|
 * C17 - at least one component has presence state == INSTANTIATION_FAILED and
 | 
						|
 *       it is not allowed to reboot the node because of this problem
 | 
						|
 * C18 - at least one component has presence state == INSTANTIATION_FAILED and
 | 
						|
 *       it is allowed to reboot the node to recover from this problem
 | 
						|
 * C19 - all components in the SU permit restart
 | 
						|
 *
 | 
						|
 * 1.4.2 Composed Guards
 | 
						|
 * =====================
 | 
						|
 *
 | 
						|
 * C100 - C9 & C13
 | 
						|
 * C101 - C9 & C18
 | 
						|
 * C102 - C9 & C17
 | 
						|
 * C103 - C8 & C9 & !C13
 | 
						|
 * C104 - C8 & C9 & C13
 | 
						|
 * C105 - C14 & (C17 | C18)
 | 
						|
 * C106 - C14 & C15 & C16
 | 
						|
 *
 | 
						|
 * 1.5 Events
 | 
						|
 * ==========
 | 
						|
 *
 | 
						|
 * E1  - component restart request
 | 
						|
 * E2  - restart
 | 
						|
 * E3  - probation timer expired
 | 
						|
 * E4  - escalation reverted
 | 
						|
 * E5  - operation failed
 | 
						|
 * E6  - deactivated
 | 
						|
 * E7  - comp_state(prsm, INSTANTIATED)
 | 
						|
 * E8  - comp_state(prsm, INSTANTIATION_FAILED)
 | 
						|
 * E9  - comp_state(prsm, TERMINATION_FAILED)
 | 
						|
 * E10 - si_state(ha-state)
 | 
						|
 * E11 - activated
 | 
						|
 * E12 - comp_state(prsm, UNINSTANTIATED)
 | 
						|
 * E13 - 
 | 
						|
 * E14 - 
 | 
						|
 *
 | 
						|
 */
 | 
						|
 | 
						|
#include <stdio.h>
 | 
						|
#include <stdlib.h>
 | 
						|
#include <assert.h>
 | 
						|
#include <string.h>
 | 
						|
#include <errno.h>
 | 
						|
 | 
						|
#include "amf.h"
 | 
						|
#include "util.h"
 | 
						|
#include "logsys.h"
 | 
						|
#include "main.h"
 | 
						|
 | 
						|
LOGSYS_DECLARE_SUBSYS ("AMF", LOG_INFO);
 | 
						|
 | 
						|
static int terminate_all_components_in_level (struct amf_su *su, 
 | 
						|
	SaUint32T current_instantiation_level);
 | 
						|
static int are_all_comps_in_level_uninst_or_term_failed (struct amf_su *su);
 | 
						|
static int are_all_comps_in_level_instantiated (struct amf_su *su);
 | 
						|
static int instantiate_all_components_in_level (struct amf_su *su, 
 | 
						|
	SaUint32T current_instantiation_level);
 | 
						|
static SaUint32T su_lowest_comp_instantiation_level_set (struct amf_su *su);
 | 
						|
static void si_ha_state_assumed_cbfn (
 | 
						|
	struct amf_si_assignment *si_assignment, int result);
 | 
						|
static int is_any_component_instantiating (amf_su_t *su);
 | 
						|
 | 
						|
typedef struct su_event {
 | 
						|
	amf_su_event_type_t event_type;
 | 
						|
	amf_su_t *su;
 | 
						|
	amf_comp_t *comp;
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery;
 | 
						|
} su_event_t;
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * Internal (static) utility functions
 | 
						|
 *****************************************************************************/
 | 
						|
 | 
						|
 | 
						|
static void su_event_set(struct amf_su *su, struct amf_comp *comp, 
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery,
 | 
						|
	su_event_t *su_event, amf_su_event_type_t event_type)
 | 
						|
{
 | 
						|
	su_event->event_type = event_type;
 | 
						|
	su_event->comp = comp;
 | 
						|
	su_event->su = su;
 | 
						|
	su_event->recommended_recovery = recommended_recovery;
 | 
						|
}
 | 
						|
 | 
						|
static void su_defer_event (amf_su_t *su, amf_comp_t *comp, 
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery, 
 | 
						|
	amf_su_event_type_t su_event_type)
 | 
						|
{
 | 
						|
	su_event_t event;
 | 
						|
	su_event_set(su, comp, recommended_recovery,&event, su_event_type);
 | 
						|
 | 
						|
	ENTER("event_type = %d", event.event_type);
 | 
						|
	amf_fifo_put (event.event_type, &event.su->deferred_events,
 | 
						|
		sizeof (su_event_t), &event);
 | 
						|
}
 | 
						|
 | 
						|
static void su_recall_deferred_events (amf_su_t *su)
 | 
						|
{
 | 
						|
	su_event_t su_event;
 | 
						|
 | 
						|
	ENTER ("%s", su->name.value);
 | 
						|
	if (amf_fifo_get (&su->deferred_events, &su_event)) {
 | 
						|
		switch (su_event.event_type) {
 | 
						|
			case SU_COMP_ERROR_SUSPECTED_EV:
 | 
						|
				amf_su_comp_error_suspected (su_event.su,su_event.comp,
 | 
						|
					su_event.recommended_recovery);
 | 
						|
				break;
 | 
						|
			default:
 | 
						|
				dprintf("event_type = %d", su_event.event_type);
 | 
						|
				break;
 | 
						|
		}
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
static int has_component_restarted_max_times (amf_comp_t *comp, amf_su_t *su)
 | 
						|
{
 | 
						|
 return comp->saAmfCompRestartCount >= su->sg->saAmfSGCompRestartMax;
 | 
						|
}
 | 
						|
 | 
						|
#ifdef COMPILE_OUT
 | 
						|
static int has_su_restarted_max_times (amf_su_t *su)
 | 
						|
{
 | 
						|
	return su->saAmfSURestartCount >= su->sg->saAmfSGSuRestartMax;
 | 
						|
}
 | 
						|
#endif
 | 
						|
 | 
						|
/**
 | 
						|
 * This function only logs since the readiness state is runtime
 | 
						|
 * calculated.
 | 
						|
 * @param su
 | 
						|
 * @param amf_readiness_state
 | 
						|
 */
 | 
						|
static void su_readiness_state_set (struct amf_su *su,
 | 
						|
	SaAmfReadinessStateT readiness_state)
 | 
						|
{
 | 
						|
	log_printf (LOG_NOTICE, "Setting SU '%s' readiness state: %s\n",
 | 
						|
		su->name.value, amf_readiness_state (readiness_state));
 | 
						|
}
 | 
						|
 | 
						|
static void clear_ha_state (
 | 
						|
	struct amf_su *su, struct amf_si_assignment *si_assignment)
 | 
						|
{
 | 
						|
	ENTER ("");
 | 
						|
	si_assignment->saAmfSISUHAState = 0;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function sets presence state to the specified value. It also has the
 | 
						|
 * following intentional side effects:
 | 
						|
 * - sets HA-state to unknown when presence state is set to UNINSTANTIATED
 | 
						|
 * - reports the change of presence state to the sg in which su is contained
 | 
						|
 *   when the new state is 'stable'
 | 
						|
 * @param su
 | 
						|
 * @param presence_state - new value of presence state
 | 
						|
 */
 | 
						|
static void su_presence_state_set (struct amf_su *su,
 | 
						|
	SaAmfPresenceStateT presence_state)
 | 
						|
{
 | 
						|
	/*                                                              
 | 
						|
	 * Set all SI's confirmed HA state to unknown if uninstantiated
 | 
						|
	*/
 | 
						|
	if (su->saAmfSUPresenceState == SA_AMF_PRESENCE_UNINSTANTIATED) {
 | 
						|
		amf_su_foreach_si_assignment (su, clear_ha_state);
 | 
						|
	}
 | 
						|
 | 
						|
 | 
						|
	su->saAmfSUPresenceState = presence_state;
 | 
						|
	log_printf (LOG_NOTICE, "Setting SU '%s' presence state: %s\n",
 | 
						|
		su->name.value, amf_presence_state (presence_state));
 | 
						|
 | 
						|
 | 
						|
	if (su->restart_control_state != SU_RC_RESTART_SU_SETTING &&
 | 
						|
		su->restart_control_state != SU_RC_RESTART_COMP_RESTARTING) {
 | 
						|
		amf_sg_su_state_changed (su->sg, su, SA_AMF_PRESENCE_STATE, 
 | 
						|
			presence_state);
 | 
						|
	}
 | 
						|
}
 | 
						|
static void enter_idle (struct amf_su *su)
 | 
						|
{
 | 
						|
  su->restart_control_state = su->escalation_level_history_state;
 | 
						|
}
 | 
						|
 | 
						|
static void enter_idle_with_recall (struct amf_su *su)
 | 
						|
{
 | 
						|
	su->restart_control_state = su->escalation_level_history_state;
 | 
						|
	su_recall_deferred_events (su);
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function sets operational state to the specified value. It also has the
 | 
						|
 * following side effects:
 | 
						|
 * - sets the readiness state for su
 | 
						|
 * - sets the readiness state for all components contained in the su
 | 
						|
 * @param su
 | 
						|
 * @param oper_state - new value of operational state
 | 
						|
 */
 | 
						|
void amf_su_operational_state_set (struct amf_su *su,
 | 
						|
	SaAmfOperationalStateT oper_state)
 | 
						|
{
 | 
						|
	struct amf_comp* comp;
 | 
						|
 | 
						|
	su->saAmfSUOperState = oper_state;
 | 
						|
	log_printf (LOG_NOTICE, "Setting SU '%s' operational state: %s\n",
 | 
						|
		su->name.value, amf_op_state (oper_state));
 | 
						|
 | 
						|
	if (oper_state == SA_AMF_OPERATIONAL_ENABLED) {
 | 
						|
		su_readiness_state_set (su, SA_AMF_READINESS_IN_SERVICE);
 | 
						|
 | 
						|
		for (comp = su->comp_head; comp; comp = comp->next) {
 | 
						|
			amf_comp_readiness_state_set (comp, SA_AMF_READINESS_IN_SERVICE);
 | 
						|
		}
 | 
						|
	} else if (oper_state == SA_AMF_OPERATIONAL_DISABLED) {
 | 
						|
		su_readiness_state_set (su, SA_AMF_READINESS_OUT_OF_SERVICE);
 | 
						|
		for (comp = su->comp_head; comp; comp = comp->next) {
 | 
						|
			amf_comp_readiness_state_set (comp, SA_AMF_READINESS_OUT_OF_SERVICE);
 | 
						|
		}
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function creates a new csi-assignment object and initializes it. The
 | 
						|
 * function also links the new csi-assignment object to the list of assignments
 | 
						|
 * held by the specified csi object, sets a pointer to the specified component
 | 
						|
 * and a pointer to the specified si-assignment.
 | 
						|
 * @param comp
 | 
						|
 * @param csi
 | 
						|
 * @param si_assignment
 | 
						|
 * @param ha_state - new value of ha-state
 | 
						|
 */
 | 
						|
static void comp_assign_csi (struct amf_comp *comp, struct amf_csi *csi,
 | 
						|
	struct amf_si_assignment *si_assignment, SaAmfHAStateT ha_state)
 | 
						|
{
 | 
						|
	struct amf_csi_assignment *csi_assignment;
 | 
						|
 | 
						|
	dprintf ("  Creating CSI '%s' to comp '%s' with hastate %s\n",
 | 
						|
		getSaNameT (&csi->name), getSaNameT (&comp->name),
 | 
						|
		amf_ha_state (ha_state));
 | 
						|
 | 
						|
	csi_assignment = amf_malloc (sizeof (struct amf_csi_assignment));
 | 
						|
	csi_assignment->next = csi->assigned_csis;
 | 
						|
	csi->assigned_csis = csi_assignment;
 | 
						|
	amf_comp_dn_make (comp, &csi_assignment->name);
 | 
						|
	csi_assignment->comp = comp;
 | 
						|
	csi_assignment->csi = csi;
 | 
						|
	csi_assignment->saAmfCSICompHAState = 0; /* undefined confirmed HA state */
 | 
						|
	csi_assignment->requested_ha_state = ha_state;
 | 
						|
	csi_assignment->si_assignment = si_assignment;
 | 
						|
}
 | 
						|
 | 
						|
 | 
						|
static void comp_restart (struct amf_comp *comp)
 | 
						|
{
 | 
						|
	SaNameT dn;
 | 
						|
 | 
						|
	ENTER ("'%s'", comp->name.value);
 | 
						|
	amf_comp_dn_make (comp, &dn);
 | 
						|
	log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
 | 
						|
		"action: Component restart", dn.value);
 | 
						|
 | 
						|
	comp->su->restart_control_state = SU_RC_RESTART_COMP_DEACTIVATING;
 | 
						|
	comp->su->restart_control_state = SU_RC_RESTART_COMP_RESTARTING;
 | 
						|
	comp->su->escalation_level_history_state = SU_RC_IDLE_ESCALATION_LEVEL_1;
 | 
						|
	amf_comp_restart (comp);
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Set the same HA-state as the before the restart to the SI-assignments
 | 
						|
 * associated with current SU. As a side effect, this HA-state will also be set
 | 
						|
 * to all components which are associated with the csi-assignments associated to
 | 
						|
 * the specified su via its csi and si objects.
 | 
						|
 * @param su
 | 
						|
 * @param current_instantiation_level
 | 
						|
 * 
 | 
						|
 * @return - 1 if there were no components on the specified instantiation level
 | 
						|
 */
 | 
						|
static void reassume_ha_state(struct amf_su *su)
 | 
						|
{
 | 
						|
	struct amf_si_assignment *si_assignment;
 | 
						|
 | 
						|
	ENTER ("");
 | 
						|
 | 
						|
	si_assignment = amf_su_get_next_si_assignment(su, NULL);
 | 
						|
 | 
						|
	while (si_assignment != NULL) {
 | 
						|
		si_assignment->saAmfSISUHAState = 0; /* unknown */
 | 
						|
		amf_si_ha_state_assume (si_assignment, si_ha_state_assumed_cbfn);
 | 
						|
		si_assignment = amf_su_get_next_si_assignment(su, si_assignment);
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
static int is_any_component_instantiating (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *component;
 | 
						|
	int any_component_instantiating = 0;
 | 
						|
	for (component = su->comp_head; component != NULL; 
 | 
						|
		  component = component->next) {
 | 
						|
		if (component->saAmfCompPresenceState == 
 | 
						|
			SA_AMF_PRESENCE_INSTANTIATING) {
 | 
						|
			any_component_instantiating = 1;
 | 
						|
			break;
 | 
						|
		}
 | 
						|
		
 | 
						|
	}
 | 
						|
	return any_component_instantiating;
 | 
						|
}
 | 
						|
 | 
						|
static int is_any_component_terminating (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *component;
 | 
						|
	int any_component_terminating = 0;
 | 
						|
	for (component = su->comp_head; component != NULL; 
 | 
						|
		  component = component->next) {
 | 
						|
		if (component->saAmfCompPresenceState == 
 | 
						|
			SA_AMF_PRESENCE_TERMINATING) {
 | 
						|
			any_component_terminating = 1;
 | 
						|
			break;
 | 
						|
		}
 | 
						|
		
 | 
						|
	}
 | 
						|
	return any_component_terminating;
 | 
						|
}
 | 
						|
 | 
						|
 | 
						|
static int is_any_comp_instantiation_failed (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *comp_;
 | 
						|
	int comp_instantiation_failed = 0;
 | 
						|
 | 
						|
	for (comp_ = su->comp_head; comp_ != NULL; comp_ = comp_->next) {
 | 
						|
 | 
						|
		if (comp_->saAmfCompPresenceState == 
 | 
						|
			SA_AMF_PRESENCE_INSTANTIATION_FAILED) {
 | 
						|
			comp_instantiation_failed = 1;
 | 
						|
			break;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return comp_instantiation_failed;
 | 
						|
}
 | 
						|
 | 
						|
static int is_any_comp_termination_failed (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *comp_;
 | 
						|
	int comp_instantiation_failed = 0;
 | 
						|
 | 
						|
	for (comp_ = su->comp_head; comp_ != NULL; comp_ = comp_->next) {
 | 
						|
 | 
						|
		if (comp_->saAmfCompPresenceState == 
 | 
						|
			SA_AMF_PRESENCE_TERMINATION_FAILED) {
 | 
						|
			comp_instantiation_failed = 1;
 | 
						|
			break;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return comp_instantiation_failed;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Finds the component within the specified su that has the highest value of it
 | 
						|
 * presence state. With current definition of values the highest value can also
 | 
						|
 * be regarded as the 'worst' in the sence of capability to be assigned
 | 
						|
 * workload. In the 'best' presence state (INSTANTIATED) the component is
 | 
						|
 * immediately available to take workload while in the 'worst' state
 | 
						|
 * (TERMINATION_FAILED) it can not take any workload before it has been manually
 | 
						|
 * repaired.
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return - worst presence state
 | 
						|
 */
 | 
						|
static SaAmfPresenceStateT 	get_worst_comps_presence_state_in_su (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *component;
 | 
						|
	SaAmfPresenceStateT worst_presence_state = 0;
 | 
						|
 | 
						|
	for (component = su->comp_head; component != NULL; 
 | 
						|
		  component = component->next) {
 | 
						|
		if (component->saAmfCompPresenceState > worst_presence_state) {
 | 
						|
			worst_presence_state = component->saAmfCompPresenceState;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return worst_presence_state;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * 
 | 
						|
 * @param su
 | 
						|
 */
 | 
						|
void su_history_state_set(struct amf_su *su, SaAmfPresenceStateT state)
 | 
						|
{
 | 
						|
	su->restart_control_state = su->escalation_level_history_state;
 | 
						|
	su->saAmfSUPresenceState =  state;
 | 
						|
}
 | 
						|
/**
 | 
						|
 * A component notifies its parent su that its presence state has changed.
 | 
						|
 * @param su
 | 
						|
 * @param comp - component which has changed its presence state
 | 
						|
 * @param state - new value of presence state
 | 
						|
 */
 | 
						|
static void su_comp_presence_state_changed (struct amf_su *su, 
 | 
						|
	struct amf_comp *comp, int state)
 | 
						|
{
 | 
						|
	ENTER ("'%s', '%s' %d %d", su->name.value, comp->name.value, state,
 | 
						|
		su->restart_control_state);
 | 
						|
	amf_node_t *node = amf_node_find (&comp->su->saAmfSUHostedByNode);
 | 
						|
	switch (state) {
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATED:
 | 
						|
			switch (su->restart_control_state) {
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_1:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_2:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_0:
 | 
						|
					if (!is_any_component_instantiating (su)) {
 | 
						|
						if (are_all_comps_in_level_instantiated (su)) {
 | 
						|
							if (instantiate_all_components_in_level (su, 
 | 
						|
								++comp->su->current_comp_instantiation_level)) {
 | 
						|
								/* All levels of instantiation is done */
 | 
						|
								su_presence_state_set (comp->su, 
 | 
						|
									SA_AMF_PRESENCE_INSTANTIATED);
 | 
						|
							}
 | 
						|
						} else {
 | 
						|
							if (is_any_comp_instantiation_failed (su)) {
 | 
						|
								su_presence_state_set (su, 
 | 
						|
									SA_AMF_PRESENCE_INSTANTIATION_FAILED);
 | 
						|
 | 
						|
							} else {
 | 
						|
								assert (0);
 | 
						|
							}
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_COMP_RESTARTING:
 | 
						|
					su->restart_control_state = SU_RC_RESTART_COMP_SETTING;
 | 
						|
					reassume_ha_state (comp->su);
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_SU_INSTANTIATING:
 | 
						|
					if (!is_any_component_instantiating(su)) {
 | 
						|
						if (are_all_comps_in_level_instantiated (su)) {
 | 
						|
							if (instantiate_all_components_in_level (su, 
 | 
						|
								++comp->su->current_comp_instantiation_level)) {
 | 
						|
								su->restart_control_state = SU_RC_RESTART_SU_SETTING;
 | 
						|
								su_presence_state_set (comp->su, 
 | 
						|
									SA_AMF_PRESENCE_INSTANTIATED);
 | 
						|
								reassume_ha_state (comp->su);
 | 
						|
							}
 | 
						|
						} else if (is_any_comp_instantiation_failed (su)) {
 | 
						|
							su->restart_control_state = 
 | 
						|
								SU_RC_TERMINATING_AFTER_INSTANTIATION_FAILED;
 | 
						|
							terminate_all_components_in_level (su,
 | 
						|
								su->current_comp_instantiation_level);
 | 
						|
						} else {
 | 
						|
							assert (0);
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				default:
 | 
						|
					dprintf ("state %d", su->restart_control_state);
 | 
						|
					assert (0);
 | 
						|
					break;
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_UNINSTANTIATED:
 | 
						|
			switch (su->restart_control_state) {
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_0:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_1:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_2:
 | 
						|
					if (!is_any_component_terminating (su)) {
 | 
						|
						if (are_all_comps_in_level_uninst_or_term_failed (su)) {
 | 
						|
							if (terminate_all_components_in_level (su,
 | 
						|
								--su->current_comp_instantiation_level)) {
 | 
						|
								su_presence_state_set (su,
 | 
						|
									get_worst_comps_presence_state_in_su (su));
 | 
						|
							} else {
 | 
						|
								if (is_any_comp_termination_failed (su)) {
 | 
						|
									su_presence_state_set (comp->su, 
 | 
						|
										SA_AMF_PRESENCE_TERMINATION_FAILED);
 | 
						|
								} else {
 | 
						|
									assert (0);
 | 
						|
								}
 | 
						|
							}
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_SU_INSTANTIATING:
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_COMP_RESTARTING:
 | 
						|
					break;
 | 
						|
				case SU_RC_TERMINATING_AFTER_INSTANTIATION_FAILED:
 | 
						|
					if (!is_any_component_terminating (su)) {
 | 
						|
						if (terminate_all_components_in_level (su,
 | 
						|
							--su->current_comp_instantiation_level)) {
 | 
						|
							if (!is_any_comp_termination_failed (su)) {
 | 
						|
								su_presence_state_set (su, 
 | 
						|
									SA_AMF_PRESENCE_INSTANTIATION_FAILED);
 | 
						|
								if (node->saAmfNodeRebootOnInstantiationFailure) {
 | 
						|
									amf_node_failover(node);
 | 
						|
								} else {
 | 
						|
									amf_node_comp_failover_req(node, comp);
 | 
						|
								}
 | 
						|
								enter_idle (su);
 | 
						|
							} else {
 | 
						|
								if (!node->saAmfNodeRebootOnTerminationFailure) {
 | 
						|
									su_presence_state_set (su,
 | 
						|
										get_worst_comps_presence_state_in_su (su));
 | 
						|
								} else {
 | 
						|
									/* TODO Implement and request Node Failed Fast */
 | 
						|
									;
 | 
						|
								}
 | 
						|
								enter_idle_with_recall (su);
 | 
						|
							}
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_SU_TERMINATING:
 | 
						|
					if (!is_any_component_terminating (su)) {
 | 
						|
						if (terminate_all_components_in_level (su,
 | 
						|
							--su->current_comp_instantiation_level)) {
 | 
						|
							if (!is_any_comp_termination_failed (su)) {
 | 
						|
								su->restart_control_state = 
 | 
						|
									SU_RC_RESTART_SU_INSTANTIATING;
 | 
						|
								instantiate_all_components_in_level (su, 
 | 
						|
									su_lowest_comp_instantiation_level_set (
 | 
						|
									su));
 | 
						|
							} else {
 | 
						|
								if (!node->saAmfNodeRebootOnTerminationFailure) {
 | 
						|
									su_presence_state_set (su,
 | 
						|
										get_worst_comps_presence_state_in_su (su));
 | 
						|
								} else {
 | 
						|
									/* TODO Implement and request Node Failed Fast */
 | 
						|
									;
 | 
						|
								}
 | 
						|
								enter_idle_with_recall (su);
 | 
						|
							}
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				default:
 | 
						|
					dprintf ("state %d", su->restart_control_state);
 | 
						|
					assert (0);
 | 
						|
					break;
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATING:
 | 
						|
			su_presence_state_set (comp->su,SA_AMF_PRESENCE_INSTANTIATING);
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_RESTARTING:
 | 
						|
			if (amf_su_are_all_comps_in_su (su, SA_AMF_PRESENCE_RESTARTING)) {
 | 
						|
				su_presence_state_set (comp->su, SA_AMF_PRESENCE_RESTARTING);
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_TERMINATING:
 | 
						|
			su_presence_state_set (comp->su, SA_AMF_PRESENCE_TERMINATING);
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATION_FAILED:
 | 
						|
			switch (su->restart_control_state) {
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_0:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_1:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_2:
 | 
						|
					if (!is_any_component_instantiating (su)) {
 | 
						|
						su_presence_state_set (su, 
 | 
						|
							SA_AMF_PRESENCE_INSTANTIATION_FAILED);
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_COMP_RESTARTING:
 | 
						|
					su->restart_control_state = 
 | 
						|
						SU_RC_TERMINATING_AFTER_INSTANTIATION_FAILED;
 | 
						|
					amf_su_terminate (su);
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_SU_INSTANTIATING:
 | 
						|
					if (!is_any_component_instantiating (su)) {
 | 
						|
						su->restart_control_state = 
 | 
						|
							SU_RC_TERMINATING_AFTER_INSTANTIATION_FAILED;
 | 
						|
						su_presence_state_set (su, 
 | 
						|
							SA_AMF_PRESENCE_INSTANTIATION_FAILED);
 | 
						|
						terminate_all_components_in_level (su,
 | 
						|
							su->current_comp_instantiation_level);
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				default:
 | 
						|
					dprintf ("state %d", su->restart_control_state);
 | 
						|
					assert (0);
 | 
						|
					break;
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_TERMINATION_FAILED:
 | 
						|
			switch (su->restart_control_state) {
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_0:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_1:
 | 
						|
				case SU_RC_IDLE_ESCALATION_LEVEL_2:
 | 
						|
					break;
 | 
						|
				case SU_RC_RESTART_COMP_RESTARTING:
 | 
						|
					if (!node->saAmfNodeRebootOnTerminationFailure) {
 | 
						|
						su_presence_state_set (su,
 | 
						|
							SA_AMF_PRESENCE_TERMINATION_FAILED);
 | 
						|
						enter_idle_with_recall (su);
 | 
						|
					} else {
 | 
						|
						/* TODO Implement and request Node Failed Fast */
 | 
						|
						;
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				case SU_RC_TERMINATING_AFTER_INSTANTIATION_FAILED:
 | 
						|
				case SU_RC_RESTART_SU_TERMINATING:
 | 
						|
					if (!is_any_component_terminating (su)) {
 | 
						|
						if (terminate_all_components_in_level (su,
 | 
						|
							--su->current_comp_instantiation_level)) {
 | 
						|
							if (!node->saAmfNodeRebootOnTerminationFailure) {
 | 
						|
								su_presence_state_set (su,
 | 
						|
									get_worst_comps_presence_state_in_su (su));
 | 
						|
								enter_idle_with_recall (su);
 | 
						|
							} else {
 | 
						|
								/* TODO Implement and request Node Failed Fast */
 | 
						|
								;
 | 
						|
							}
 | 
						|
						}
 | 
						|
					}
 | 
						|
					break;
 | 
						|
				default:
 | 
						|
					log_printf (LOG_LEVEL_NOTICE,"%s %d",su->name.value, 
 | 
						|
						su->restart_control_state);
 | 
						|
					dprintf ("state %d", su->restart_control_state);
 | 
						|
					assert (0);
 | 
						|
					break;
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		default:
 | 
						|
			assert (0);
 | 
						|
			break;
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * A component notifies its parent su that its operational state has changed.
 | 
						|
 * @param su
 | 
						|
 * @param comp - component which has changed its operational state
 | 
						|
 * @param state - new value of operational state
 | 
						|
 */
 | 
						|
static void su_comp_op_state_changed (
 | 
						|
	struct amf_su *su, struct amf_comp *comp, int state)
 | 
						|
{
 | 
						|
	ENTER ("'%s', '%s' %d", su->name.value, comp->name.value, state);
 | 
						|
 | 
						|
	switch (state) {
 | 
						|
		case SA_AMF_OPERATIONAL_ENABLED:
 | 
						|
			{
 | 
						|
				struct amf_comp *comp_compare;
 | 
						|
				int all_set = 1;
 | 
						|
				for (comp_compare = comp->su->comp_head;
 | 
						|
					comp_compare != NULL; comp_compare = comp_compare->next) {
 | 
						|
					if (comp_compare->saAmfCompOperState !=
 | 
						|
						SA_AMF_OPERATIONAL_ENABLED) {
 | 
						|
 | 
						|
						all_set = 0;
 | 
						|
						break;
 | 
						|
					}
 | 
						|
				}
 | 
						|
				if (all_set) {
 | 
						|
					amf_su_operational_state_set (comp->su, 
 | 
						|
						SA_AMF_OPERATIONAL_ENABLED);
 | 
						|
				} else {
 | 
						|
					amf_su_operational_state_set (comp->su, 
 | 
						|
						SA_AMF_OPERATIONAL_DISABLED);
 | 
						|
				}
 | 
						|
				break;
 | 
						|
			}
 | 
						|
		case SA_AMF_OPERATIONAL_DISABLED:
 | 
						|
			amf_su_operational_state_set (comp->su, SA_AMF_OPERATIONAL_DISABLED);
 | 
						|
			break;
 | 
						|
		default:
 | 
						|
			assert (0);
 | 
						|
			break;
 | 
						|
	}
 | 
						|
	return;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Instantiates all components on specified instantiation level.
 | 
						|
 * @param su
 | 
						|
 * @param current_instantiation_level
 | 
						|
 * 
 | 
						|
 * @return - 1 if there were no components on the specified instantiation level
 | 
						|
 */
 | 
						|
static int instantiate_all_components_in_level (struct amf_su *su, 
 | 
						|
	SaUint32T current_instantiation_level)
 | 
						|
{
 | 
						|
	amf_comp_t *comp;
 | 
						|
	SaUint32T all_components_instantiated = 1;
 | 
						|
 | 
						|
	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
		if (su->current_comp_instantiation_level == 
 | 
						|
			comp->saAmfCompInstantiationLevel) {
 | 
						|
			all_components_instantiated = 0;
 | 
						|
			amf_comp_instantiate (comp);
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return all_components_instantiated;
 | 
						|
}
 | 
						|
 | 
						|
static int are_all_comps_in_level_instantiated (struct amf_su *su)
 | 
						|
{
 | 
						|
	SaUint32T level = su->current_comp_instantiation_level;
 | 
						|
	amf_comp_t *comp;
 | 
						|
	int all = 1;
 | 
						|
 | 
						|
	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
		if (level == comp->saAmfCompInstantiationLevel) {
 | 
						|
			if (comp->saAmfCompPresenceState != SA_AMF_PRESENCE_INSTANTIATED) {
 | 
						|
				all = 0;
 | 
						|
				break;
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	return all;
 | 
						|
}
 | 
						|
 | 
						|
 | 
						|
static int are_all_comps_in_level_uninst_or_term_failed(
 | 
						|
	struct amf_su *su)
 | 
						|
{
 | 
						|
	SaUint32T level = su->current_comp_instantiation_level;
 | 
						|
	amf_comp_t *comp;
 | 
						|
	int all = 1;
 | 
						|
 | 
						|
	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
		if (level == comp->saAmfCompInstantiationLevel) {
 | 
						|
			if (comp->saAmfCompPresenceState != SA_AMF_PRESENCE_UNINSTANTIATED &&
 | 
						|
				comp->saAmfCompPresenceState != SA_AMF_PRESENCE_TERMINATION_FAILED) {
 | 
						|
				all = 0;
 | 
						|
				break;
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	return all;
 | 
						|
}
 | 
						|
static void su_rc_enter_idle_escalation_level_1 (amf_comp_t *component,
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery)
 | 
						|
{
 | 
						|
	ENTER("");
 | 
						|
	component->su->restart_control_state = SU_RC_IDLE_ESCALATION_LEVEL_1;
 | 
						|
	if (has_component_restarted_max_times (component, component->su)) {
 | 
						|
		component->su->restart_control_state = SU_RC_IDLE_ESCALATION_LEVEL_2;
 | 
						|
		amf_su_comp_error_suspected (component->su, component, recommended_recovery);
 | 
						|
	} else {
 | 
						|
		comp_restart (component);
 | 
						|
	}
 | 
						|
}
 | 
						|
static void su_rc_enter_idle_escalation_level_2 (amf_comp_t *component,
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery)
 | 
						|
{
 | 
						|
	ENTER("");
 | 
						|
	component->su->restart_control_state = SU_RC_IDLE_ESCALATION_LEVEL_2;
 | 
						|
	amf_node_t *node = amf_node_find (&component->su->saAmfSUHostedByNode);
 | 
						|
	amf_node_comp_restart_req (node, component); 
 | 
						|
}
 | 
						|
static int get_instantiation_max_level (amf_su_t *su)
 | 
						|
{
 | 
						|
	amf_comp_t *comp;
 | 
						|
	int instantiation_level = 0;
 | 
						|
	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
		if (comp->saAmfCompInstantiationLevel > instantiation_level) {
 | 
						|
		   instantiation_level =  comp->saAmfCompInstantiationLevel;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return instantiation_level;
 | 
						|
}
 | 
						|
/**
 | 
						|
 * Initiates the termination of all components which have the specified
 | 
						|
 * instantiation level.
 | 
						|
 * @param su
 | 
						|
 * @param current_instantiation_level
 | 
						|
 * 
 | 
						|
 * @return int -1 if no component has the specified instantiation level
 | 
						|
 */
 | 
						|
static int terminate_all_components_in_level (struct amf_su *su, 
 | 
						|
	SaUint32T current_instantiation_level)
 | 
						|
{
 | 
						|
	amf_comp_t *comp;
 | 
						|
	int all_components_in_level = 1;
 | 
						|
	TRACE8("terminate comp->saAmfCompInstantiationLevel=%u", 
 | 
						|
		current_instantiation_level);
 | 
						|
	for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
		/* 
 | 
						|
         * Terminate all components in instantiation level in SU
 | 
						|
         * abruptly.
 | 
						|
         */
 | 
						|
		if (comp->saAmfCompInstantiationLevel == current_instantiation_level) {
 | 
						|
			amf_comp_error_suspected_set (comp);
 | 
						|
			amf_comp_terminate (comp);
 | 
						|
			all_components_in_level = 0;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return all_components_in_level;
 | 
						|
}
 | 
						|
/**
 | 
						|
 * su_current_instantiation_level_init
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return SaUint32T - the value of the instantiation level which has been set
 | 
						|
 */
 | 
						|
static SaUint32T su_lowest_comp_instantiation_level_set (struct amf_su *su)
 | 
						|
{
 | 
						|
	amf_comp_t *component = su->comp_head;
 | 
						|
	int comp_instantiation_level = component->saAmfCompInstantiationLevel;
 | 
						|
	for (; component != NULL; component = component->next) {
 | 
						|
		TRACE1("component->saAmfCompInstantiationLevel=%d",
 | 
						|
			component->saAmfCompInstantiationLevel);
 | 
						|
 | 
						|
			if (component->saAmfCompInstantiationLevel < 
 | 
						|
				comp_instantiation_level) {
 | 
						|
				comp_instantiation_level = 
 | 
						|
					component->saAmfCompInstantiationLevel;
 | 
						|
			}
 | 
						|
	}
 | 
						|
	su->current_comp_instantiation_level = comp_instantiation_level;
 | 
						|
	return comp_instantiation_level;
 | 
						|
}
 | 
						|
 | 
						|
 | 
						|
/**
 | 
						|
 * An order to SU to instantiate its components.
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return int - 1 if its state allows it to request its contained components to
 | 
						|
 *               instantiate or its state indicates that its components are in
 | 
						|
 *               the process of instantiation.
 | 
						|
 */
 | 
						|
int amf_su_instantiate (struct amf_su *su)
 | 
						|
{
 | 
						|
	int is_instantiating = 1;
 | 
						|
 | 
						|
	ENTER ("'%s %d'", su->name.value, su->saAmfSUPresenceState);
 | 
						|
	switch (su->saAmfSUPresenceState) {
 | 
						|
		case SA_AMF_PRESENCE_UNINSTANTIATED:
 | 
						|
			instantiate_all_components_in_level(su, 
 | 
						|
				su_lowest_comp_instantiation_level_set (su));
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_RESTARTING:
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATING:
 | 
						|
			break;
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATED:
 | 
						|
		case SA_AMF_PRESENCE_TERMINATING:
 | 
						|
		case SA_AMF_PRESENCE_INSTANTIATION_FAILED:
 | 
						|
		case SA_AMF_PRESENCE_TERMINATION_FAILED:
 | 
						|
			is_instantiating = 0;
 | 
						|
			break;
 | 
						|
		default:
 | 
						|
			assert (0);
 | 
						|
			break;
 | 
						|
		
 | 
						|
	}
 | 
						|
	return is_instantiating;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * An order to SU to terminate its components.
 | 
						|
 * @param su
 | 
						|
 */
 | 
						|
void amf_su_terminate (struct amf_su *su)
 | 
						|
{
 | 
						|
	ENTER ("'%s'", su->name.value);
 | 
						|
	su->current_comp_instantiation_level = get_instantiation_max_level (su);
 | 
						|
 | 
						|
	terminate_all_components_in_level (su, su->current_comp_instantiation_level);
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Called by a component to report a suspected error on a component
 | 
						|
 * @param su
 | 
						|
 * @param comp
 | 
						|
 * @param recommended_recovery
 | 
						|
 */
 | 
						|
void amf_su_comp_error_suspected (
 | 
						|
	struct amf_su *su,
 | 
						|
	struct amf_comp *comp,
 | 
						|
	SaAmfRecommendedRecoveryT recommended_recovery)
 | 
						|
{
 | 
						|
	ENTER ("Comp '%s', SU '%s' %d", comp->name.value, su->name.value,
 | 
						|
		su->restart_control_state);
 | 
						|
 | 
						|
	switch (su->restart_control_state) {
 | 
						|
		case SU_RC_IDLE_ESCALATION_LEVEL_0:
 | 
						|
				su_rc_enter_idle_escalation_level_1 (comp,
 | 
						|
					recommended_recovery);
 | 
						|
			break;
 | 
						|
 | 
						|
		case SU_RC_IDLE_ESCALATION_LEVEL_1:
 | 
						|
			if (has_component_restarted_max_times (comp, su)) {
 | 
						|
				su_rc_enter_idle_escalation_level_2 (comp,
 | 
						|
					recommended_recovery);
 | 
						|
			} else {
 | 
						|
				comp_restart (comp);
 | 
						|
			}
 | 
						|
			break;
 | 
						|
		case SU_RC_IDLE_ESCALATION_LEVEL_2: {
 | 
						|
				amf_node_t *node = amf_node_find (&comp->su->saAmfSUHostedByNode);
 | 
						|
				amf_node_comp_restart_req (node, comp); 
 | 
						|
 | 
						|
#ifdef COMPILE_OUT
 | 
						|
				if (su->saAmfSURestartCount >= su->sg->saAmfSGSuRestartMax) {
 | 
						|
 | 
						|
					/*
 | 
						|
					 * TODO: delegate to node
 | 
						|
					*/
 | 
						|
					SaNameT dn;
 | 
						|
					amf_comp_operational_state_set (comp, 
 | 
						|
						SA_AMF_OPERATIONAL_DISABLED);
 | 
						|
					amf_su_operational_state_set (su, 
 | 
						|
						SA_AMF_OPERATIONAL_DISABLED);
 | 
						|
					
 | 
						|
					amf_comp_dn_make (comp, &dn);
 | 
						|
					log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
 | 
						|
						"action:\n\t\tSU failover", dn.value);
 | 
						|
					amf_sg_failover_su_req (comp->su->sg, comp->su, this_amf_node);
 | 
						|
					return;
 | 
						|
				} else {
 | 
						|
					su_restart (comp->su);
 | 
						|
				}
 | 
						|
#endif
 | 
						|
				break;
 | 
						|
			}
 | 
						|
		case SU_RC_RESTART_SU_SETTING:
 | 
						|
		case SU_RC_RESTART_COMP_RESTARTING:
 | 
						|
		case SU_RC_RESTART_COMP_SETTING:
 | 
						|
			/* TODO: Complete the implementation of SU defer event */
 | 
						|
			su_defer_event (su, comp, recommended_recovery,
 | 
						|
				SU_COMP_ERROR_SUSPECTED_EV); 
 | 
						|
			break;
 | 
						|
		default:
 | 
						|
			dprintf ("restart_control_state = %d",su->restart_control_state);
 | 
						|
			break;
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * An order to SU to unconditionally restart itself.
 | 
						|
 * @param su
 | 
						|
 */
 | 
						|
void amf_su_restart (struct amf_su *su)
 | 
						|
{
 | 
						|
	SaNameT dn;
 | 
						|
 | 
						|
	ENTER ("'%s'", su->name.value);
 | 
						|
 | 
						|
	amf_su_dn_make (su, &dn);
 | 
						|
	log_printf (LOG_NOTICE, "Error detected for '%s', recovery "
 | 
						|
		"action: SU restart", dn.value);
 | 
						|
 | 
						|
	su->restart_control_state = SU_RC_RESTART_SU_DEACTIVATING;
 | 
						|
	su->restart_control_state = SU_RC_RESTART_SU_TERMINATING;
 | 
						|
	su->escalation_level_history_state = SU_RC_IDLE_ESCALATION_LEVEL_2;
 | 
						|
	su->current_comp_instantiation_level = get_instantiation_max_level (su);
 | 
						|
	su->saAmfSURestartCount += 1;
 | 
						|
	terminate_all_components_in_level(su, su->current_comp_instantiation_level);
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * Event response methods
 | 
						|
 *****************************************************************************/
 | 
						|
 | 
						|
/**
 | 
						|
 * Used by a component to report a state change event
 | 
						|
 * @param su
 | 
						|
 * @param comp
 | 
						|
 * @param type type of state
 | 
						|
 * @param state new state
 | 
						|
 */
 | 
						|
void amf_su_comp_state_changed (
 | 
						|
	struct amf_su *su, struct amf_comp *comp, SaAmfStateT type, int state)
 | 
						|
{
 | 
						|
	switch (type) {
 | 
						|
		case SA_AMF_PRESENCE_STATE:
 | 
						|
			su_comp_presence_state_changed (su, comp, state);
 | 
						|
			break;
 | 
						|
		case SA_AMF_OP_STATE:
 | 
						|
			su_comp_op_state_changed (su, comp, state);
 | 
						|
			break;
 | 
						|
		default:
 | 
						|
			assert (0);
 | 
						|
			break;
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
static void si_ha_state_assumed_cbfn (
 | 
						|
	struct amf_si_assignment *si_assignment, int result)
 | 
						|
{
 | 
						|
	struct amf_si_assignment *tmp_si_assignment;
 | 
						|
	struct amf_comp *comp;
 | 
						|
	struct amf_csi_assignment *csi_assignment;
 | 
						|
	int all_confirmed = 1;
 | 
						|
	ENTER ("");
 | 
						|
	tmp_si_assignment = amf_su_get_next_si_assignment(si_assignment->su, NULL);
 | 
						|
 | 
						|
	while (tmp_si_assignment != NULL) {
 | 
						|
		for (comp = tmp_si_assignment->su->comp_head; comp != NULL;
 | 
						|
			  comp = comp->next) {
 | 
						|
 | 
						|
			csi_assignment = amf_comp_get_next_csi_assignment(comp, NULL);
 | 
						|
			while (csi_assignment != NULL) {
 | 
						|
 | 
						|
				if (csi_assignment->requested_ha_state != 
 | 
						|
					csi_assignment->saAmfCSICompHAState) {
 | 
						|
					all_confirmed = 0;
 | 
						|
				}
 | 
						|
				csi_assignment = amf_comp_get_next_csi_assignment(
 | 
						|
					comp, csi_assignment);
 | 
						|
			}
 | 
						|
		}
 | 
						|
		tmp_si_assignment = amf_su_get_next_si_assignment(
 | 
						|
			si_assignment->su, tmp_si_assignment);
 | 
						|
	}
 | 
						|
 | 
						|
	if (all_confirmed) {
 | 
						|
		switch (si_assignment->su->restart_control_state) {
 | 
						|
			case SU_RC_RESTART_COMP_SETTING:
 | 
						|
				log_printf (LOG_NOTICE, "Component restart recovery finished");
 | 
						|
				break;
 | 
						|
			case SU_RC_RESTART_SU_SETTING:
 | 
						|
				log_printf (LOG_NOTICE, "SU restart recovery finished");
 | 
						|
				break;
 | 
						|
			default:
 | 
						|
				assert (0);
 | 
						|
				break;
 | 
						|
		}
 | 
						|
		enter_idle_with_recall (si_assignment->su);
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/******************************************************************************
 | 
						|
 * General methods
 | 
						|
 *****************************************************************************/
 | 
						|
 | 
						|
/**
 | 
						|
 * Constructor for SU objects. Adds SU last in the ordered
 | 
						|
 * list owned by the specified SG. Always returns a
 | 
						|
 * valid SU object, out-of-memory problems are handled here.
 | 
						|
 * Default values are initialized.
 | 
						|
 * @param sg
 | 
						|
 * @param name
 | 
						|
 * 
 | 
						|
 * @return struct amf_su*
 | 
						|
 */
 | 
						|
struct amf_su *amf_su_new (struct amf_sg *sg, char *name)
 | 
						|
{
 | 
						|
	struct amf_su *tail = sg->su_head;
 | 
						|
	struct amf_su *su = amf_calloc (1, sizeof (struct amf_su));
 | 
						|
 | 
						|
	while (tail != NULL) {
 | 
						|
		if (tail->next == NULL) {
 | 
						|
			break;
 | 
						|
		}
 | 
						|
		tail = tail->next;
 | 
						|
	}
 | 
						|
 | 
						|
	if (tail == NULL) {
 | 
						|
		sg->su_head = su;
 | 
						|
	} else {
 | 
						|
		tail->next = su;
 | 
						|
	}
 | 
						|
	su->sg = sg;
 | 
						|
 | 
						|
	/* setup default values from spec. */
 | 
						|
	su->saAmfSURank = 0;
 | 
						|
	su->saAmfSUIsExternal = 0;
 | 
						|
	su->saAmfSUFailover = 1;
 | 
						|
	su->saAmfSUAdminState = SA_AMF_ADMIN_UNLOCKED;
 | 
						|
	su->saAmfSUOperState = SA_AMF_OPERATIONAL_DISABLED;
 | 
						|
	su->saAmfSUPresenceState = SA_AMF_PRESENCE_UNINSTANTIATED;
 | 
						|
	su->restart_control_state = SU_RC_IDLE_ESCALATION_LEVEL_0;
 | 
						|
	su->current_comp_instantiation_level = 0;
 | 
						|
	setSaNameT (&su->name, name);
 | 
						|
 | 
						|
	return su;
 | 
						|
}
 | 
						|
 | 
						|
void amf_su_delete (struct amf_su *su)
 | 
						|
{
 | 
						|
	struct amf_comp *comp;
 | 
						|
 | 
						|
	for (comp = su->comp_head; comp != NULL;) {
 | 
						|
		struct amf_comp *tmp = comp;
 | 
						|
		comp = comp->next;
 | 
						|
		amf_comp_delete (tmp);
 | 
						|
	}
 | 
						|
 | 
						|
	free (su);
 | 
						|
}
 | 
						|
 | 
						|
void *amf_su_serialize (struct amf_su *su, int *len)
 | 
						|
{
 | 
						|
	char *buf = NULL;
 | 
						|
	int offset = 0, size = 0;
 | 
						|
 | 
						|
	TRACE8 ("%s", su->name.value);
 | 
						|
 | 
						|
	buf = amf_serialize_SaNameT (buf, &size, &offset, &su->name);
 | 
						|
	buf = amf_serialize_SaUint32T (buf, &size, &offset, su->saAmfSURank);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUNumComponents);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUIsExternal);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUFailover);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUPreInstantiable);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUOperState);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUAdminState);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSUPresenceState);
 | 
						|
	buf = amf_serialize_SaNameT (buf, &size, &offset, &su->saAmfSUHostedByNode);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->saAmfSURestartCount);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->restart_control_state);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->escalation_level_history_state);
 | 
						|
	buf = amf_serialize_SaStringT (
 | 
						|
		buf, &size, &offset, su->clccli_path);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->su_failover_cnt);
 | 
						|
	buf = amf_serialize_SaUint32T (
 | 
						|
		buf, &size, &offset, su->current_comp_instantiation_level);
 | 
						|
 | 
						|
	*len = offset;
 | 
						|
 | 
						|
	return buf;
 | 
						|
}
 | 
						|
 | 
						|
struct amf_su *amf_su_deserialize (struct amf_sg *sg, char *buf)
 | 
						|
{
 | 
						|
	char *tmp = buf;
 | 
						|
	struct amf_su *su = amf_su_new (sg, "");
 | 
						|
 | 
						|
	tmp = amf_deserialize_SaNameT (tmp, &su->name);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSURank);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUNumComponents);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUIsExternal);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUFailover);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUPreInstantiable);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUOperState);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUAdminState);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSUPresenceState);
 | 
						|
	tmp = amf_deserialize_SaNameT (tmp, &su->saAmfSUHostedByNode);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->saAmfSURestartCount);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->restart_control_state);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->escalation_level_history_state);
 | 
						|
	tmp = amf_deserialize_SaStringT (tmp, &su->clccli_path);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->su_failover_cnt);
 | 
						|
	tmp = amf_deserialize_SaUint32T (tmp, &su->current_comp_instantiation_level);
 | 
						|
 | 
						|
	return su;
 | 
						|
}
 | 
						|
 | 
						|
struct amf_su *amf_su_find (struct amf_cluster *cluster, SaNameT *name)
 | 
						|
{
 | 
						|
	struct amf_application *app;
 | 
						|
	struct amf_sg *sg;
 | 
						|
	struct amf_su *su = NULL;
 | 
						|
	char *app_name;
 | 
						|
	char *sg_name;
 | 
						|
	char *su_name;
 | 
						|
	char *ptrptr;
 | 
						|
	char *buf;
 | 
						|
 | 
						|
	assert (cluster != NULL && name != NULL);
 | 
						|
 | 
						|
	/* malloc new buffer since strtok_r writes to its first argument */
 | 
						|
	buf = amf_malloc (name->length + 1);
 | 
						|
	memcpy (buf, name->value, name->length + 1);
 | 
						|
 | 
						|
	su_name = strtok_r(buf, ",", &ptrptr);
 | 
						|
	sg_name = strtok_r(NULL, ",", &ptrptr);
 | 
						|
	app_name = strtok_r(NULL, ",", &ptrptr);
 | 
						|
 | 
						|
	if (su_name == NULL || sg_name == NULL || app_name == NULL) {
 | 
						|
		goto end;
 | 
						|
	}
 | 
						|
 | 
						|
	su_name += 6;
 | 
						|
	sg_name += 6;
 | 
						|
	app_name += 7;
 | 
						|
 | 
						|
	app = amf_application_find (cluster, app_name);
 | 
						|
	if (app == NULL) {
 | 
						|
		goto end;
 | 
						|
	}
 | 
						|
 | 
						|
	for (sg = app->sg_head; sg != NULL; sg = sg->next) {
 | 
						|
		if (strncmp (sg_name, (char*)sg->name.value,
 | 
						|
			sg->name.length) == 0) {
 | 
						|
			for (su = sg->su_head; su != NULL; su = su->next) {
 | 
						|
				if (su->name.length == strlen(su_name) && 
 | 
						|
					strncmp (su_name, (char*)su->name.value,
 | 
						|
					su->name.length) == 0) {
 | 
						|
					goto end;
 | 
						|
				}
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
end:
 | 
						|
	free (buf);
 | 
						|
	return su;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function makes a distinguished name for specified su object.
 | 
						|
 * @param su
 | 
						|
 * @param name -[out] pointer to where the distinguished name shall be stored
 | 
						|
 * 
 | 
						|
 * @return SaNameT* - distinguished name
 | 
						|
 */
 | 
						|
char *amf_su_dn_make (struct amf_su *su, SaNameT *name)
 | 
						|
{
 | 
						|
	int i;
 | 
						|
 | 
						|
	assert (su != NULL);
 | 
						|
 | 
						|
	i = snprintf((char*) name->value, SA_MAX_NAME_LENGTH,
 | 
						|
		"safSu=%s,safSg=%s,safApp=%s",
 | 
						|
		su->name.value, su->sg->name.value, su->sg->application->name.value);
 | 
						|
	assert (i <= SA_MAX_NAME_LENGTH);
 | 
						|
	name->length = i;
 | 
						|
	return (char *)name->value;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * An order to SU to create an si-assignment object with a specified HA-state
 | 
						|
 * between it self and a specified si. The created si-assignment is initialized
 | 
						|
 * and linked to list of assignments held by the specified si.
 | 
						|
 * This function also orders creation of all csi-assignments required
 | 
						|
 * considering the cs-types specified for the components and csi objects
 | 
						|
 * respectively.
 | 
						|
 * @param su
 | 
						|
 * @param si
 | 
						|
 * @param ha_state
 | 
						|
 * 
 | 
						|
 * @return amf_si_assignment_t*
 | 
						|
 */
 | 
						|
amf_si_assignment_t *amf_su_assign_si (struct amf_su *su, struct amf_si *si,
 | 
						|
	SaAmfHAStateT ha_state)
 | 
						|
{
 | 
						|
	struct amf_si_assignment *si_assignment;
 | 
						|
 | 
						|
	dprintf ("Creating SI '%s' to SU '%s' with hastate %s\n",
 | 
						|
		getSaNameT (&si->name), getSaNameT (&su->name),
 | 
						|
		amf_ha_state (ha_state));
 | 
						|
 | 
						|
	si_assignment = amf_malloc (sizeof (struct amf_si_assignment));
 | 
						|
	amf_su_dn_make (su, &si_assignment->name);
 | 
						|
	si_assignment->saAmfSISUHAState = 0; /* undefined confirmed HA state */
 | 
						|
	si_assignment->requested_ha_state = ha_state;
 | 
						|
	si_assignment->next = si->assigned_sis;
 | 
						|
	si->assigned_sis = si_assignment;
 | 
						|
	si_assignment->si = si;
 | 
						|
	si_assignment->su = su;
 | 
						|
 | 
						|
	{
 | 
						|
		struct amf_csi *csi;
 | 
						|
		struct amf_comp *comp;
 | 
						|
		SaNameT *cs_type;
 | 
						|
		int i;
 | 
						|
 | 
						|
		/*
 | 
						|
		** for each component in SU, find a CSI in the SI with the same type
 | 
						|
		*/
 | 
						|
		for (comp = su->comp_head; comp != NULL; comp = comp->next) {
 | 
						|
			int no_of_cs_types = 0;
 | 
						|
			for (i = 0; comp->saAmfCompCsTypes[i]; i++) {
 | 
						|
				cs_type = comp->saAmfCompCsTypes[i];
 | 
						|
				no_of_cs_types++;
 | 
						|
				int no_of_assignments = 0;
 | 
						|
 | 
						|
				for (csi = si->csi_head; csi != NULL; csi = csi->next) {
 | 
						|
					if (!memcmp(csi->saAmfCSTypeName.value, cs_type->value,
 | 
						|
							cs_type->length)) {
 | 
						|
						comp_assign_csi (comp, csi, si_assignment, ha_state);
 | 
						|
						no_of_assignments++;
 | 
						|
					}
 | 
						|
				}
 | 
						|
				if (no_of_assignments == 0) {
 | 
						|
					log_printf (
 | 
						|
						LOG_WARNING, "\t   No CSIs of type %s configured?!!\n",
 | 
						|
						getSaNameT (cs_type));
 | 
						|
				}
 | 
						|
			}
 | 
						|
			if (no_of_cs_types == 0) {
 | 
						|
				log_printf (LOG_LEVEL_ERROR,
 | 
						|
					"\t   No CS types configured for comp %s ?!!\n",
 | 
						|
					getSaNameT (&comp->name));
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return si_assignment;
 | 
						|
}
 | 
						|
 | 
						|
struct amf_si_assignment *amf_su_get_next_si_assignment (
 | 
						|
	struct amf_su *su, const struct amf_si_assignment *si_assignment)
 | 
						|
{
 | 
						|
	struct amf_si *si;
 | 
						|
	struct amf_si_assignment *tmp_si_assignment;
 | 
						|
	SaNameT dn;
 | 
						|
 | 
						|
	amf_su_dn_make (su, &dn);
 | 
						|
 | 
						|
	if (si_assignment == NULL) {
 | 
						|
		assert (su->sg);
 | 
						|
		assert (su->sg->application);
 | 
						|
		assert (su->sg->application->si_head);
 | 
						|
		si = su->sg->application->si_head;
 | 
						|
		tmp_si_assignment = si->assigned_sis;
 | 
						|
	} else {
 | 
						|
		tmp_si_assignment = si_assignment->next;
 | 
						|
		if (tmp_si_assignment == NULL) {
 | 
						|
			si = si_assignment->si->next;
 | 
						|
			if (si == NULL) {
 | 
						|
				return NULL;
 | 
						|
			} else {
 | 
						|
				tmp_si_assignment = si->assigned_sis;
 | 
						|
			}
 | 
						|
		} else {
 | 
						|
			si = tmp_si_assignment->si;
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	for (; si != NULL; si = si->next) {
 | 
						|
		if (tmp_si_assignment == NULL && si != NULL) {
 | 
						|
			tmp_si_assignment = si->assigned_sis;
 | 
						|
		}
 | 
						|
		for (; tmp_si_assignment != NULL;
 | 
						|
			tmp_si_assignment = tmp_si_assignment->next) {
 | 
						|
 | 
						|
			if (name_match (&tmp_si_assignment->name, &dn)) {
 | 
						|
				return tmp_si_assignment;
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	return NULL;
 | 
						|
}
 | 
						|
 | 
						|
void amf_su_foreach_si_assignment (
 | 
						|
	struct amf_su *su,
 | 
						|
	void (*foreach_fn)(struct amf_su *su,
 | 
						|
	struct amf_si_assignment *si_assignment))
 | 
						|
{
 | 
						|
	struct amf_si_assignment *si_assignment;
 | 
						|
 | 
						|
	assert (foreach_fn != NULL);
 | 
						|
	si_assignment = amf_su_get_next_si_assignment (su, NULL);
 | 
						|
	while (si_assignment != NULL) {
 | 
						|
		foreach_fn (su, si_assignment);
 | 
						|
		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function calculates the number of si-assignments with active HA-state
 | 
						|
 * which currently are associated with the specified su.
 | 
						|
 * TODO: Split into two functions and remove dependency to sg's avail_state
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return int 
 | 
						|
 */
 | 
						|
int amf_su_get_saAmfSUNumCurrActiveSIs(struct amf_su *su)
 | 
						|
{
 | 
						|
	int cnt = 0;
 | 
						|
	struct amf_si_assignment *si_assignment;
 | 
						|
 | 
						|
	si_assignment = amf_su_get_next_si_assignment (su, NULL); 
 | 
						|
	while (si_assignment != NULL) {
 | 
						|
		if (su->sg->avail_state == SG_AC_AssigningOnRequest &&
 | 
						|
			si_assignment->requested_ha_state == SA_AMF_HA_ACTIVE) {
 | 
						|
			cnt++;
 | 
						|
		} else {
 | 
						|
			if (si_assignment->saAmfSISUHAState == SA_AMF_HA_ACTIVE) {
 | 
						|
				cnt++;
 | 
						|
			}
 | 
						|
		}
 | 
						|
		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
 | 
						|
	}
 | 
						|
 | 
						|
	return cnt;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function calculates the number of si-assignments with standby HA-state
 | 
						|
 * which currently are associated with the specified su.
 | 
						|
 * TODO: Split into two functions and remove dependency to sg's avail_state
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return int 
 | 
						|
 */
 | 
						|
int amf_su_get_saAmfSUNumCurrStandbySIs(struct amf_su *su)
 | 
						|
{
 | 
						|
	int cnt = 0;
 | 
						|
	struct amf_si_assignment *si_assignment;
 | 
						|
 | 
						|
	si_assignment = amf_su_get_next_si_assignment (su, NULL); 
 | 
						|
	while (si_assignment != NULL) {
 | 
						|
		if (su->sg->avail_state == SG_AC_AssigningOnRequest &&
 | 
						|
			si_assignment->requested_ha_state == SA_AMF_HA_STANDBY) {
 | 
						|
			cnt++;
 | 
						|
		} else {
 | 
						|
			if (si_assignment->saAmfSISUHAState == SA_AMF_HA_STANDBY) {
 | 
						|
				cnt++;
 | 
						|
			}
 | 
						|
		}
 | 
						|
		si_assignment = amf_su_get_next_si_assignment (su, si_assignment);
 | 
						|
	}
 | 
						|
 | 
						|
	return cnt;
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * This function calculates the readiness state for specified su
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return SaAmfReadinessStateT 
 | 
						|
 */
 | 
						|
SaAmfReadinessStateT amf_su_get_saAmfSUReadinessState (struct amf_su *su)
 | 
						|
{
 | 
						|
	if ((su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) &&
 | 
						|
		((su->saAmfSUPresenceState == SA_AMF_PRESENCE_INSTANTIATED) ||
 | 
						|
		(su->saAmfSUPresenceState == SA_AMF_PRESENCE_RESTARTING))) {
 | 
						|
		return SA_AMF_READINESS_IN_SERVICE;
 | 
						|
	} else if (su->saAmfSUOperState == SA_AMF_OPERATIONAL_ENABLED) {
 | 
						|
		return SA_AMF_READINESS_STOPPING;
 | 
						|
	} else {
 | 
						|
		return SA_AMF_READINESS_OUT_OF_SERVICE;
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Determine if the SU is hosted on the local node.
 | 
						|
 * @param su
 | 
						|
 * 
 | 
						|
 * @return int
 | 
						|
 */
 | 
						|
int amf_su_is_local (struct amf_su *su)
 | 
						|
{
 | 
						|
	if (name_match (&this_amf_node->name, &su->saAmfSUHostedByNode)) {
 | 
						|
		return 1;
 | 
						|
	} else {
 | 
						|
		return 0;
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
/**
 | 
						|
 * Determine if all components have the specified HA-state.
 | 
						|
 * @param su
 | 
						|
 * @param state -specified HA-state
 | 
						|
 * 
 | 
						|
 * @return int - return 0 if not all components have the specified HA-state
 | 
						|
 */
 | 
						|
int amf_su_are_all_comps_in_su (struct amf_su *su,
 | 
						|
	SaAmfPresenceStateT state)
 | 
						|
{
 | 
						|
	int all_comps_in_su_are_set = 1;
 | 
						|
	amf_comp_t *component;
 | 
						|
	for (component = su->comp_head; component != NULL; 
 | 
						|
		  component = component->next) {
 | 
						|
 | 
						|
		if (component->saAmfCompPresenceState != state) {
 | 
						|
			all_comps_in_su_are_set = 0;
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return all_comps_in_su_are_set;
 | 
						|
}
 | 
						|
 |