From 20fe60f40d80634dd22dd2b85b4b14c86b7dc927 Mon Sep 17 00:00:00 2001 From: Igor Ryzhov Date: Sat, 20 Jan 2024 03:43:18 +0200 Subject: [PATCH] lib: fix conversion of yang decimal64 to double Current denominators are not integers and some of them lose precision because of that, for example, 1e-6 is actually stored as 9.9999999999999995e-07 and 1-e12 is stored as 9.9999999999999998e-13. When multiplying by such denominators, we receive incorrect values. Changing denominators to integers and using division instead of multiplication improves precision and solves the problem. Signed-off-by: Igor Ryzhov --- lib/yang_wrappers.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/yang_wrappers.c b/lib/yang_wrappers.c index 53f40e89b7..237913880a 100644 --- a/lib/yang_wrappers.c +++ b/lib/yang_wrappers.c @@ -168,10 +168,9 @@ struct yang_data *yang_data_new_dec64(const char *xpath, double value) double yang_dnode_get_dec64(const struct lyd_node *dnode, const char *xpath_fmt, ...) { - const double denom[19] = {1e0, 1e-1, 1e-2, 1e-3, 1e-4, - 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, - 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, - 1e-15, 1e-16, 1e-17, 1e-18}; + const double denom[19] = { 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, + 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, + 1e14, 1e15, 1e16, 1e17, 1e18 }; const struct lysc_type_dec *dectype; const struct lyd_value *dvalue; @@ -179,7 +178,7 @@ double yang_dnode_get_dec64(const struct lyd_node *dnode, const char *xpath_fmt, dectype = (const struct lysc_type_dec *)dvalue->realtype; assert(dectype->basetype == LY_TYPE_DEC64); assert(dectype->fraction_digits < sizeof(denom) / sizeof(*denom)); - return (double)dvalue->dec64 * denom[dectype->fraction_digits]; + return (double)dvalue->dec64 / denom[dectype->fraction_digits]; } double yang_get_default_dec64(const char *xpath_fmt, ...)