Merge pull request #4742 from opensourcerouting/ospfd-default-originate

ospfd: fix & clean up default origination

ospfd/ospf_flood.c

@@ -83,6 +83,9 @@ struct external_info *ospf_external_info_check(struct ospf *ospf,
 	struct as_external_lsa *al;
 	struct prefix_ipv4 p;
 	struct route_node *rn;
+	struct list *ext_list;
+	struct listnode *node;
+	struct ospf_external *ext;
 	int type;
 
 	al = (struct as_external_lsa *)lsa->data;
@@ -91,7 +94,7 @@ struct external_info *ospf_external_info_check(struct ospf *ospf,
 	p.prefix = lsa->data->id;
 	p.prefixlen = ip_masklen(al->mask);
 
-	for (type = 0; type <= ZEBRA_ROUTE_MAX; type++) {
+	for (type = 0; type < ZEBRA_ROUTE_MAX; type++) {
 		int redist_on = 0;
 
 		redist_on =
@@ -105,10 +108,6 @@ struct external_info *ospf_external_info_check(struct ospf *ospf,
 					      ospf->vrf_id));
 		// Pending: check for MI above.
 		if (redist_on) {
-			struct list *ext_list;
-			struct listnode *node;
-			struct ospf_external *ext;
-
 			ext_list = ospf->external[type];
 			if (!ext_list)
 				continue;
@@ -129,6 +128,22 @@ struct external_info *ospf_external_info_check(struct ospf *ospf,
 		}
 	}
 
+	if (is_prefix_default(&p) && ospf->external[DEFAULT_ROUTE]) {
+		ext_list = ospf->external[DEFAULT_ROUTE];
+
+		for (ALL_LIST_ELEMENTS_RO(ext_list, node, ext)) {
+			if (!ext->external_info)
+				continue;
+
+			rn = route_node_lookup(ext->external_info,
+					       (struct prefix *)&p);
+			if (!rn)
+				continue;
+			route_unlock_node(rn);
+			if (rn->info != NULL)
+				return (struct external_info *)rn->info;
+		}
+	}
 	return NULL;
 }
 

ospfd/ospf_lsa.c

@@ -1979,6 +1979,13 @@ struct ospf_lsa *ospf_external_lsa_originate(struct ospf *ospf,
 
 	   */
 
+	if (ospf->router_id.s_addr == 0) {
+		if (IS_DEBUG_OSPF_EVENT)
+			zlog_debug("LSA[Type5:%pI4]: deferring AS-external-LSA origination, router ID is zero",
+				   &ei->p.prefix);
+		return NULL;
+	}
+
 	/* Check the AS-external-LSA should be originated. */
 	if (!ospf_redistribute_check(ospf, ei, NULL))
 		return NULL;
@@ -2019,50 +2026,6 @@ struct ospf_lsa *ospf_external_lsa_originate(struct ospf *ospf,
 	return new;
 }
 
-/* Originate AS-external-LSA from external info with initial flag. */
-int ospf_external_lsa_originate_timer(struct thread *thread)
-{
-	struct ospf *ospf = THREAD_ARG(thread);
-	struct route_node *rn;
-	struct external_info *ei;
-	struct route_table *rt;
-	int type = THREAD_VAL(thread);
-	struct list *ext_list;
-	struct listnode *node;
-	struct ospf_external *ext;
-
-	ospf->t_external_lsa = NULL;
-
-	ext_list = ospf->external[type];
-	if (!ext_list)
-		return 0;
-
-	for (ALL_LIST_ELEMENTS_RO(ext_list, node, ext)) {
-		/* Originate As-external-LSA from all type of distribute source.
-		 */
-		rt = ext->external_info;
-		if (!rt)
-			continue;
-
-		for (rn = route_top(rt); rn; rn = route_next(rn)) {
-			ei = rn->info;
-
-			if (!ei)
-				continue;
-
-			if (is_prefix_default((struct prefix_ipv4 *)&ei->p))
-				continue;
-
-			if (!ospf_external_lsa_originate(ospf, ei))
-				flog_warn(
-					EC_OSPF_LSA_INSTALL_FAILURE,
-					"LSA: AS-external-LSA was not originated.");
-		}
-	}
-
-	return 0;
-}
-
 static struct external_info *ospf_default_external_info(struct ospf *ospf)
 {
 	int type;
@@ -2102,31 +2065,52 @@ static struct external_info *ospf_default_external_info(struct ospf *ospf)
 	return NULL;
 }
 
-int ospf_default_originate_timer(struct thread *thread)
+void ospf_external_lsa_rid_change(struct ospf *ospf)
 {
-	struct prefix_ipv4 p;
-	struct in_addr nexthop;
 	struct external_info *ei;
-	struct ospf *ospf;
+	int type;
 
-	ospf = THREAD_ARG(thread);
+	for (type = 0; type < ZEBRA_ROUTE_MAX; type++) {
+		struct route_node *rn;
+		struct route_table *rt;
+		struct list *ext_list;
+		struct listnode *node;
+		struct ospf_external *ext;
 
-	p.family = AF_INET;
+		ext_list = ospf->external[type];
-	p.prefix.s_addr = 0;
+		if (!ext_list)
-	p.prefixlen = 0;
+			continue;
 
-	if (ospf->default_originate == DEFAULT_ORIGINATE_ALWAYS) {
+		for (ALL_LIST_ELEMENTS_RO(ext_list, node, ext)) {
-		/* If there is no default route via redistribute,
+			/* Originate As-external-LSA from all type of
-		   then originate AS-external-LSA with nexthop 0 (self). */
+			 * distribute source.
-		nexthop.s_addr = 0;
+			 */
-		ospf_external_info_add(ospf, DEFAULT_ROUTE, 0, p, 0, nexthop,
+			rt = ext->external_info;
-				       0);
+			if (!rt)
+				continue;
+
+			for (rn = route_top(rt); rn; rn = route_next(rn)) {
+				ei = rn->info;
+
+				if (!ei)
+					continue;
+
+				if (is_prefix_default(
+						(struct prefix_ipv4 *)&ei->p))
+					continue;
+
+				if (!ospf_external_lsa_originate(ospf, ei))
+					flog_warn(EC_OSPF_LSA_INSTALL_FAILURE,
+						"LSA: AS-external-LSA was not originated.");
+			}
+		}
 	}
 
-	if ((ei = ospf_default_external_info(ospf)))
+	ei = ospf_default_external_info(ospf);
-		ospf_external_lsa_originate(ospf, ei);
+	if (ei && !ospf_external_lsa_originate(ospf, ei)) {
-
+		flog_warn(EC_OSPF_LSA_INSTALL_FAILURE,
-	return 0;
+			"LSA: AS-external-LSA for default route was not originated.");
+	}
 }
 
 /* Flush any NSSA LSAs for given prefix */
@@ -2218,44 +2202,20 @@ void ospf_external_lsa_refresh_default(struct ospf *ospf)
 	ei = ospf_default_external_info(ospf);
 	lsa = ospf_external_info_find_lsa(ospf, &p);
 
-	if (ei) {
+	if (ei && lsa) {
-		if (lsa) {
+		if (IS_DEBUG_OSPF_EVENT)
-			if (IS_DEBUG_OSPF_EVENT)
+			zlog_debug("LSA[Type5:0.0.0.0]: Refresh AS-external-LSA %p",
-				zlog_debug(
+				   (void *)lsa);
-					"LSA[Type5:0.0.0.0]: Refresh AS-external-LSA %p",
+		ospf_external_lsa_refresh(ospf, lsa, ei, LSA_REFRESH_FORCE);
-					(void *)lsa);
+	} else if (ei && !lsa) {
-			ospf_external_lsa_refresh(ospf, lsa, ei,
+		if (IS_DEBUG_OSPF_EVENT)
-						  LSA_REFRESH_FORCE);
+			zlog_debug(
-		} else {
+				"LSA[Type5:0.0.0.0]: Originate AS-external-LSA");
-			if (IS_DEBUG_OSPF_EVENT)
+		ospf_external_lsa_originate(ospf, ei);
-				zlog_debug(
+	} else if (lsa) {
-					"LSA[Type5:0.0.0.0]: Originate AS-external-LSA");
+		if (IS_DEBUG_OSPF_EVENT)
-			ospf_external_lsa_originate(ospf, ei);
+			zlog_debug("LSA[Type5:0.0.0.0]: Flush AS-external-LSA");
-		}
+		ospf_external_lsa_flush(ospf, DEFAULT_ROUTE, &p, 0);
-	} else {
-		if (lsa) {
-			if (IS_DEBUG_OSPF_EVENT)
-				zlog_debug(
-					"LSA[Type5:0.0.0.0]: Flush AS-external-LSA");
-			ospf_refresher_unregister_lsa(ospf, lsa);
-			ospf_lsa_flush_as(ospf, lsa);
-		}
-	}
-}
-
-void ospf_default_originate_lsa_update(struct ospf *ospf)
-{
-	struct prefix_ipv4 p;
-	struct ospf_lsa *lsa;
-
-	p.family = AF_INET;
-	p.prefixlen = 0;
-	p.prefix.s_addr = 0;
-
-	lsa = ospf_external_info_find_lsa(ospf, &p);
-	if (lsa && IS_LSA_MAXAGE(lsa)) {
-		ospf_discard_from_db(ospf, lsa->lsdb, lsa);
-		ospf_lsdb_delete(lsa->lsdb, lsa);
 	}
 }
 

ospfd/ospf_lsa.h

@@ -275,8 +275,7 @@ extern struct in_addr ospf_get_ip_from_ifp(struct ospf_interface *);
 
 extern struct ospf_lsa *ospf_external_lsa_originate(struct ospf *,
 						    struct external_info *);
-extern int ospf_external_lsa_originate_timer(struct thread *);
+extern void ospf_external_lsa_rid_change(struct ospf *ospf);
-extern int ospf_default_originate_timer(struct thread *);
 extern struct ospf_lsa *ospf_lsa_lookup(struct ospf *ospf, struct ospf_area *,
 					uint32_t, struct in_addr,
 					struct in_addr);
@@ -301,7 +300,6 @@ extern int ospf_lsa_maxage_walker(struct thread *);
 extern struct ospf_lsa *ospf_lsa_refresh(struct ospf *, struct ospf_lsa *);
 
 extern void ospf_external_lsa_refresh_default(struct ospf *);
-extern void ospf_default_originate_lsa_update(struct ospf *ospf);
 
 extern void ospf_external_lsa_refresh_type(struct ospf *, uint8_t,
 					   unsigned short, int);

ospfd/ospf_nsm.c

@@ -616,8 +616,6 @@ static void nsm_change_state(struct ospf_neighbor *nbr, int state)
 	struct ospf_interface *oi = nbr->oi;
 	struct ospf_area *vl_area = NULL;
 	uint8_t old_state;
-	int x;
-	int force = 1;
 
 	/* Preserve old status. */
 	old_state = nbr->state;
@@ -664,32 +662,6 @@ static void nsm_change_state(struct ospf_neighbor *nbr, int state)
 			if (oi->type == OSPF_IFTYPE_VIRTUALLINK && vl_area)
 				if (++vl_area->full_vls == 1)
 					ospf_schedule_abr_task(oi->ospf);
-
-			/* kevinm: refresh any redistributions */
-			for (x = ZEBRA_ROUTE_SYSTEM; x < ZEBRA_ROUTE_MAX; x++) {
-				struct list *red_list;
-				struct listnode *node;
-				struct ospf_redist *red;
-
-				if (x == ZEBRA_ROUTE_OSPF6)
-					continue;
-
-				red_list = oi->ospf->redist[x];
-				if (!red_list)
-					continue;
-
-				for (ALL_LIST_ELEMENTS_RO(red_list, node, red))
-					ospf_external_lsa_refresh_type(
-						oi->ospf, x, red->instance,
-						force);
-			}
-			/* XXX: Clearly some thing is wrong with refresh of
-			 * external LSAs
-			 * this added to hack around defaults not refreshing
-			 * after a timer
-			 * jump.
-			 */
-			ospf_external_lsa_refresh_default(oi->ospf);
 		} else {
 			oi->full_nbrs--;
 			oi->area->full_nbrs--;

ospfd/ospf_vty.c

@@ -8487,22 +8487,8 @@ DEFUN (no_ospf_default_information_originate,
        "Pointer to route-map entries\n")
 {
 	VTY_DECLVAR_INSTANCE_CONTEXT(ospf, ospf);
-	struct prefix_ipv4 p;
-	struct ospf_external *ext;
 	struct ospf_redist *red;
 
-	p.family = AF_INET;
-	p.prefix.s_addr = 0;
-	p.prefixlen = 0;
-
-	ospf_external_lsa_flush(ospf, DEFAULT_ROUTE, &p, 0);
-
-	ext = ospf_external_lookup(ospf, DEFAULT_ROUTE, 0);
-	if (ext && EXTERNAL_INFO(ext)) {
-		ospf_external_info_delete(ospf, DEFAULT_ROUTE, 0, p);
-		ospf_external_del(ospf, DEFAULT_ROUTE, 0);
-	}
-
 	red = ospf_redist_lookup(ospf, DEFAULT_ROUTE, 0);
 	if (!red)
 		return CMD_SUCCESS;
@@ -8510,7 +8496,8 @@ DEFUN (no_ospf_default_information_originate,
 	ospf_routemap_unset(red);
 	ospf_redist_del(ospf, DEFAULT_ROUTE, 0);
 
-	return ospf_redistribute_default_unset(ospf);
+	return ospf_redistribute_default_set(ospf, DEFAULT_ORIGINATE_NONE,
+					     0, 0);
 }
 
 DEFUN (ospf_default_metric,

ospfd/ospf_zebra.c

@@ -744,10 +744,10 @@ int ospf_redistribute_unset(struct ospf *ospf, int type,
 int ospf_redistribute_default_set(struct ospf *ospf, int originate, int mtype,
 				  int mvalue)
 {
-	struct ospf_external *ext;
 	struct prefix_ipv4 p;
 	struct in_addr nexthop;
 	int cur_originate = ospf->default_originate;
+	const char *type_str = NULL;
 
 	nexthop.s_addr = 0;
 	p.family = AF_INET;
@@ -756,41 +756,7 @@ int ospf_redistribute_default_set(struct ospf *ospf, int originate, int mtype,
 
 	ospf->default_originate = originate;
 
-	ospf_external_add(ospf, DEFAULT_ROUTE, 0);
+	if (cur_originate == originate) {
-
-	if (cur_originate == DEFAULT_ORIGINATE_NONE) {
-		/* First time configuration */
-		if (IS_DEBUG_OSPF(zebra, ZEBRA_REDISTRIBUTE))
-			zlog_debug("Redistribute[DEFAULT]: Start Type[%d], Metric[%d]",
-			metric_type(ospf, DEFAULT_ROUTE, 0),
-			metric_value(ospf, DEFAULT_ROUTE, 0));
-
-		if (ospf->router_id.s_addr == 0)
-			ospf->external_origin |= (1 << DEFAULT_ROUTE);
-		if ((originate == DEFAULT_ORIGINATE_ALWAYS)
-			  && (ospf->router_id.s_addr)) {
-
-			/* always , so originate lsa even it doesn't
-			 * exist in RIB.
-			 */
-			ospf_external_info_add(ospf, DEFAULT_ROUTE, 0,
-						 p, 0, nexthop, 0);
-			ospf_external_lsa_refresh_default(ospf);
-
-		} else if (originate == DEFAULT_ORIGINATE_ZEBRA) {
-			/* Send msg to Zebra to validate default route
-			 * existance.
-			 */
-			zclient_redistribute_default(
-				ZEBRA_REDISTRIBUTE_DEFAULT_ADD, zclient, AFI_IP,
-				ospf->vrf_id);
-		}
-
-		ospf_asbr_status_update(ospf, ++ospf->redistribute);
-		return CMD_SUCCESS;
-
-
-	} else if (originate == cur_originate) {
 		/* Refresh the lsa since metric might different */
 		if (IS_DEBUG_OSPF(zebra, ZEBRA_REDISTRIBUTE))
 			zlog_debug(
@@ -800,67 +766,51 @@ int ospf_redistribute_default_set(struct ospf *ospf, int originate, int mtype,
 				metric_value(ospf, DEFAULT_ROUTE, 0));
 
 		ospf_external_lsa_refresh_default(ospf);
-
+		return CMD_SUCCESS;
-	} else {
-		/* "default-info originate always" configured now,
-		 * where "default-info originate" configured previoulsly.
-		 */
-		if (originate == DEFAULT_ORIGINATE_ALWAYS) {
-
-			zclient_redistribute_default(
-					ZEBRA_REDISTRIBUTE_DEFAULT_DELETE,
-					zclient, AFI_IP, ospf->vrf_id);
-			/* here , ex-info should be added since ex-info might
-			 * have not updated earlier if def route is not exist.
-			 * If ex-iinfo ex-info already exist , it will return
-			 * smoothly.
-			 */
-			ospf_external_info_add(ospf, DEFAULT_ROUTE, 0,
-						 p, 0, nexthop, 0);
-			ospf_external_lsa_refresh_default(ospf);
-
-		} else {
-			/* "default-info originate" configured now,where
-			 * "default-info originate always" configured
-			 * previoulsy.
-			 */
-
-			ospf_external_lsa_flush(ospf, DEFAULT_ROUTE, &p, 0);
-
-			ext = ospf_external_lookup(ospf, DEFAULT_ROUTE, 0);
-			if (ext && EXTERNAL_INFO(ext))
-				ospf_external_info_delete(ospf,
-						 DEFAULT_ROUTE, 0, p);
-
-			zclient_redistribute_default(
-					ZEBRA_REDISTRIBUTE_DEFAULT_ADD,
-					zclient, AFI_IP, ospf->vrf_id);
-		}
 	}
 
-	return CMD_SUCCESS;
+	switch (cur_originate) {
-}
+	case DEFAULT_ORIGINATE_NONE:
-int ospf_redistribute_default_unset(struct ospf *ospf)
+		break;
-{
+	case DEFAULT_ORIGINATE_ZEBRA:
-	if (ospf->default_originate == DEFAULT_ORIGINATE_ZEBRA) {
-		if (!ospf_is_type_redistributed(ospf, DEFAULT_ROUTE, 0))
-			return CMD_SUCCESS;
 		zclient_redistribute_default(ZEBRA_REDISTRIBUTE_DEFAULT_DELETE,
 				 zclient, AFI_IP, ospf->vrf_id);
+		ospf->redistribute--;
+		break;
+	case DEFAULT_ORIGINATE_ALWAYS:
+		ospf_external_info_delete(ospf, DEFAULT_ROUTE, 0, p);
+		ospf_external_del(ospf, DEFAULT_ROUTE, 0);
+		ospf->redistribute--;
+		break;
 	}
 
-	ospf->default_originate = DEFAULT_ORIGINATE_NONE;
+	switch (originate) {
+	case DEFAULT_ORIGINATE_NONE:
+		type_str = "none";
+		break;
+	case DEFAULT_ORIGINATE_ZEBRA:
+		type_str = "normal";
+		ospf->redistribute++;
+		zclient_redistribute_default(ZEBRA_REDISTRIBUTE_DEFAULT_ADD,
+					     zclient, AFI_IP, ospf->vrf_id);
+		break;
+	case DEFAULT_ORIGINATE_ALWAYS:
+		type_str = "always";
+		ospf->redistribute++;
+		ospf_external_add(ospf, DEFAULT_ROUTE, 0);
+		ospf_external_info_add(ospf, DEFAULT_ROUTE, 0, p, 0, nexthop,
+				       0);
+		break;
+	}
 
 	if (IS_DEBUG_OSPF(zebra, ZEBRA_REDISTRIBUTE))
-		zlog_debug("Redistribute[DEFAULT]: Stop");
+		zlog_debug("Redistribute[DEFAULT]: %s Type[%d], Metric[%d]",
-
+		type_str,
-	// Pending: how does the external_info cleanup work in this case?
+		metric_type(ospf, DEFAULT_ROUTE, 0),
-
+		metric_value(ospf, DEFAULT_ROUTE, 0));
-	ospf_asbr_status_update(ospf, --ospf->redistribute);
-
-	/* clean up maxage default originate external lsa */
-	ospf_default_originate_lsa_update(ospf);
 
+	ospf_external_lsa_refresh_default(ospf);
+	ospf_asbr_status_update(ospf, ospf->redistribute);
 	return CMD_SUCCESS;
 }
 
@@ -1067,11 +1017,7 @@ static int ospf_zebra_read_route(ZAPI_CALLBACK_ARGS)
 			/* Nothing has changed, so nothing to do; return */
 			return 0;
 		}
-		if (ospf->router_id.s_addr == 0)
+		if (ospf->router_id.s_addr != 0) {
-			/* Set flags to generate AS-external-LSA originate event
-			   for each redistributed protocols later. */
-			ospf->external_origin |= (1 << rt_type);
-		else {
 			if (ei) {
 				if (is_prefix_default(&p))
 					ospf_external_lsa_refresh_default(ospf);

ospfd/ospfd.c

@@ -91,7 +91,6 @@ void ospf_router_id_update(struct ospf *ospf)
 	struct ospf_interface *oi;
 	struct interface *ifp;
 	struct listnode *node;
-	int type;
 
 	if (!ospf->oi_running) {
 		if (IS_DEBUG_OSPF_EVENT)
@@ -135,24 +134,6 @@ void ospf_router_id_update(struct ospf *ospf)
 			ospf_nbr_self_reset(oi, router_id);
 		}
 
-		/* If AS-external-LSA is queued, then flush those LSAs. */
-		if (router_id_old.s_addr == 0 && ospf->external_origin) {
-			/* Originate each redistributed external route. */
-			for (type = 0; type < ZEBRA_ROUTE_MAX; type++)
-				if (ospf->external_origin & (1 << type))
-					thread_add_event(
-						master,
-						ospf_external_lsa_originate_timer,
-						ospf, type, NULL);
-			/* Originate Deafult. */
-			if (ospf->external_origin & (1 << ZEBRA_ROUTE_MAX))
-				thread_add_event(master,
-						 ospf_default_originate_timer,
-						 ospf, 0, NULL);
-
-			ospf->external_origin = 0;
-		}
-
 		/* Flush (inline) all external LSAs based on the OSPF_LSA_SELF
 		 * flag */
 		if (ospf->lsdb) {
@@ -196,20 +177,14 @@ void ospf_router_id_update(struct ospf *ospf)
 			}
 		}
 
-		/* Originate each redistributed external route. */
-		for (type = 0; type < ZEBRA_ROUTE_MAX; type++)
-			thread_add_event(master,
-					 ospf_external_lsa_originate_timer,
-					 ospf, type, NULL);
-		thread_add_event(master, ospf_default_originate_timer, ospf, 0,
-				 NULL);
-
 		/* update router-lsa's for each area */
 		ospf_router_lsa_update(ospf);
 
 		/* update ospf_interface's */
 		FOR_ALL_INTERFACES (vrf, ifp)
 			ospf_if_update(ospf, ifp);
+
+		ospf_external_lsa_rid_change(ospf);
 	}
 }
 
@@ -633,7 +608,7 @@ static void ospf_finish_final(struct ospf *ospf)
 			ospf_redist_del(ospf, i, red->instance);
 		}
 	}
-	ospf_redistribute_default_unset(ospf);
+	ospf_redistribute_default_set(ospf, DEFAULT_ORIGINATE_NONE, 0, 0);
 
 	for (ALL_LIST_ELEMENTS(ospf->areas, node, nnode, area))
 		ospf_remove_vls_through_area(ospf, area);
@@ -702,7 +677,6 @@ static void ospf_finish_final(struct ospf *ospf)
 	/* Cancel all timers. */
 	OSPF_TIMER_OFF(ospf->t_read);
 	OSPF_TIMER_OFF(ospf->t_write);
-	OSPF_TIMER_OFF(ospf->t_external_lsa);
 	OSPF_TIMER_OFF(ospf->t_spf_calc);
 	OSPF_TIMER_OFF(ospf->t_ase_calc);
 	OSPF_TIMER_OFF(ospf->t_maxage);

ospfd/ospfd.h

@@ -202,7 +202,6 @@ struct ospf {
 	struct ospf_lsdb *lsdb;
 
 	/* Flags. */
-	int external_origin; /* AS-external-LSA origin flag. */
 	int ase_calc;	/* ASE calculation flag. */
 
 	struct list *opaque_lsa_self; /* Type-11 Opaque-LSAs */
@@ -233,7 +232,6 @@ struct ospf {
 	struct thread *t_distribute_update; /* Distirbute list update timer. */
 	struct thread *t_spf_calc;	  /* SPF calculation timer. */
 	struct thread *t_ase_calc;	  /* ASE calculation timer. */
-	struct thread *t_external_lsa;      /* AS-external-LSA origin timer. */
 	struct thread
 		*t_opaque_lsa_self; /* Type-11 Opaque-LSAs origin event. */
 	struct thread *t_sr_update; /* Segment Routing update timer */