From 3e725e7bf7916657ba3c50d66a276f627bda93c8 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Fri, 25 Oct 2024 15:44:13 +0500 Subject: [PATCH] =?UTF-8?q?*=20Added=20Jitter=20of=20=C2=B15%=20to=20cache?= =?UTF-8?q?=5Fttl=20to=20increase=20probabilty=20of=20spreading=20out=20dn?= =?UTF-8?q?s=20resolution=20to=20mulitple=20intervals=20and=20avoid=20havi?= =?UTF-8?q?ng=20chances=20of=20having=20similar=20expiry=20time=20*=20Adde?= =?UTF-8?q?d=20a=20small=20delay=20after=20queuing=20DNS=20records,=20allo?= =?UTF-8?q?wing=20DNS=20resolver=20=20=20thread=20time=20to=20process=20as?= =?UTF-8?q?=20many=20hostnames=20as=20possible=20before=20assessing=20=20?= =?UTF-8?q?=20the=20need=20for=20additional=20helper=20threads.=20This=20h?= =?UTF-8?q?elps=20prevent=20'DNS=20=20=20resolver=20queue=20too=20big'=20w?= =?UTF-8?q?arnings=20in=20the=20logs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/MySQL_Monitor.hpp | 3 ++- lib/MySQL_Monitor.cpp | 30 ++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 513d043dc..f4dd87771 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -438,7 +438,8 @@ struct DNS_Resolve_Data { std::shared_ptr dns_cache; std::string hostname; std::set cached_ips; - unsigned int ttl; + unsigned int ttl = 0; + unsigned int refresh_intv = 0; }; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 3caf199e0..a776aa595 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "prometheus/counter.h" #include "MySQL_Protocol.h" #include "MySQL_HostGroups_Manager.h" @@ -4668,6 +4669,14 @@ void* monitor_dns_resolver_thread(void* args) { if (!ips.empty()) { bool to_update_cache = false; + int cache_ttl = dns_resolve_data->ttl; + if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + std::random_device rd; + std::mt19937 gen(rd()); + const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + std::uniform_int_distribution dis(-jitter, jitter); + cache_ttl += dis(gen); + } if (!dns_resolve_data->cached_ips.empty()) { @@ -4686,14 +4695,14 @@ void* monitor_dns_resolver_thread(void* args) { // only update dns_records_bookkeeping if (!to_update_cache) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl)))); } } else to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl)))); dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); } @@ -4841,7 +4850,18 @@ void* MySQL_Monitor::monitor_dns_cache() { std::list>> dns_resolve_result; + int delay_us = 100; + if (hostnames.empty() == false) { + delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size(); + delay_us *= 40; + if (delay_us > 1000000 || delay_us <= 0) { + delay_us = 10000; + } + delay_us = delay_us + rand() % delay_us; + } + if (dns_records_bookkeeping.empty() == false) { + unsigned long long current_time = monotonic_time(); for (auto itr = dns_records_bookkeeping.begin(); @@ -4861,12 +4881,14 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->hostname = std::move(itr->hostname_); dns_resolve_data->cached_ips = std::move(itr->ips_); dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); itr = dns_records_bookkeeping.erase(itr); + usleep(delay_us); continue; } @@ -4881,7 +4903,6 @@ void* MySQL_Monitor::monitor_dns_cache() { if (qsize > (static_cast(mysql_thread___monitor_local_dns_resolver_queue_maxsize) / 8)) { proxy_warning("DNS resolver queue too big: %d. Please refer to https://proxysql.com/documentation/dns-cache/ for further information.\n", qsize); - unsigned int threads_max = num_dns_resolver_max_threads; if (threads_max > num_threads) { @@ -4906,14 +4927,15 @@ void* MySQL_Monitor::monitor_dns_cache() { } if (hostnames.empty() == false) { - for (const std::string& hostname : hostnames) { std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); dns_resolve_data->hostname = hostname; dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); + usleep(delay_us); } }