Add HTTP option to control the size of streaming responses (DataDog#1…

…0183) * Add HTTP option to control the size of streaming responses * wrap `requests.Response` objects * sync config * fix * update example config
kkourtchikov · Sep 28, 2021 · b1d2b0f · b1d2b0f
1 parent 27c1070
commit b1d2b0f
Show file tree

Hide file tree

Showing 138 changed files with 564 additions and 40 deletions.
diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
@@ -92,3 +92,4 @@ typing,PyPI,PSF,"Guido van Rossum, Jukka Lehtosalo, Łukasz Langa, Ivan Levkivsk
 uptime,PyPI,BSD-2-Clause,Koen Crolla
 vertica-python,PyPI,Apache-2.0,"Justin Berka, Alex Kim, Siting Ren"
 win-inet-pton,PyPI,Unlicense,Ryan Vennell
+wrapt,PyPI,BSD-3-Clause,Graham Dumpleton
diff --git a/activemq_xml/datadog_checks/activemq_xml/data/conf.yaml.example b/activemq_xml/datadog_checks/activemq_xml/data/conf.yaml.example
@@ -347,6 +347,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/airflow/datadog_checks/airflow/data/conf.yaml.example b/airflow/datadog_checks/airflow/data/conf.yaml.example
@@ -318,6 +318,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example b/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example
@@ -250,11 +250,6 @@ instances:
     #
     # cache_shared_labels: true
 
-    ## @param request_size - number - optional - default: 16
-    ## The number of kibibytes (KiB) to read from the `openmetrics_endpoint` at a time.
-    #
-    # request_size: 16
-
     ## @param raw_line_filters - list of strings - optional
     ## A list of regular expressions used to exclude lines read from the `openmetrics_endpoint`
     ## from being parsed.
@@ -553,6 +548,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/ambari/datadog_checks/ambari/data/conf.yaml.example b/ambari/datadog_checks/ambari/data/conf.yaml.example
@@ -347,6 +347,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/apache/datadog_checks/apache/config_models/defaults.py b/apache/datadog_checks/apache/config_models/defaults.py
@@ -120,6 +120,10 @@ def instance_read_timeout(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_request_size(field, value):
+    return 16
+
+
 def instance_service(field, value):
     return get_default_field_value(field, value)
 

diff --git a/apache/datadog_checks/apache/config_models/instance.py b/apache/datadog_checks/apache/config_models/instance.py
@@ -60,6 +60,7 @@ class Config:
     persist_connections: Optional[bool]
     proxy: Optional[Proxy]
     read_timeout: Optional[float]
+    request_size: Optional[float]
     service: Optional[str]
     skip_proxy: Optional[bool]
     tags: Optional[Sequence[str]]

diff --git a/apache/datadog_checks/apache/data/conf.yaml.example b/apache/datadog_checks/apache/data/conf.yaml.example
@@ -318,6 +318,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/avi_vantage/datadog_checks/avi_vantage/data/conf.yaml.example b/avi_vantage/datadog_checks/avi_vantage/data/conf.yaml.example
@@ -226,11 +226,6 @@ instances:
     #
     # cache_shared_labels: true
 
-    ## @param request_size - number - optional - default: 16
-    ## The number of kibibytes (KiB) to read from the `openmetrics_endpoint` at a time.
-    #
-    # request_size: 16
-
     ## @param raw_line_filters - list of strings - optional
     ## A list of regular expressions used to exclude lines read from the `openmetrics_endpoint`
     ## from being parsed.
@@ -529,6 +524,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example b/azure_iot_edge/datadog_checks/azure_iot_edge/data/conf.yaml.example
@@ -437,6 +437,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 10
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 10
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/cilium/datadog_checks/cilium/data/conf.yaml.example b/cilium/datadog_checks/cilium/data/conf.yaml.example
@@ -440,6 +440,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 10
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 10
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example
@@ -259,6 +259,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/citrix_hypervisor/datadog_checks/citrix_hypervisor/config_models/defaults.py b/citrix_hypervisor/datadog_checks/citrix_hypervisor/config_models/defaults.py
@@ -120,6 +120,10 @@ def instance_read_timeout(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_request_size(field, value):
+    return 16
+
+
 def instance_service(field, value):
     return get_default_field_value(field, value)
 

diff --git a/citrix_hypervisor/datadog_checks/citrix_hypervisor/config_models/instance.py b/citrix_hypervisor/datadog_checks/citrix_hypervisor/config_models/instance.py
@@ -59,6 +59,7 @@ class Config:
     persist_connections: Optional[bool]
     proxy: Optional[Proxy]
     read_timeout: Optional[float]
+    request_size: Optional[float]
     service: Optional[str]
     skip_proxy: Optional[bool]
     tags: Optional[Sequence[str]]

diff --git a/citrix_hypervisor/datadog_checks/citrix_hypervisor/data/conf.yaml.example b/citrix_hypervisor/datadog_checks/citrix_hypervisor/data/conf.yaml.example
@@ -318,6 +318,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/cloud_foundry_api/datadog_checks/cloud_foundry_api/data/conf.yaml.example b/cloud_foundry_api/datadog_checks/cloud_foundry_api/data/conf.yaml.example
@@ -203,6 +203,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example b/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example
@@ -432,6 +432,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 10
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 10
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/consul/datadog_checks/consul/config_models/defaults.py b/consul/datadog_checks/consul/config_models/defaults.py
@@ -144,6 +144,10 @@ def instance_read_timeout(field, value):
     return get_default_field_value(field, value)
 
 
+def instance_request_size(field, value):
+    return 16
+
+
 def instance_self_leader_check(field, value):
     return False
 

diff --git a/consul/datadog_checks/consul/config_models/instance.py b/consul/datadog_checks/consul/config_models/instance.py
@@ -65,6 +65,7 @@ class Config:
     persist_connections: Optional[bool]
     proxy: Optional[Proxy]
     read_timeout: Optional[float]
+    request_size: Optional[float]
     self_leader_check: Optional[bool]
     service: Optional[str]
     services_exclude: Optional[Sequence[str]]

diff --git a/consul/datadog_checks/consul/data/conf.yaml.example b/consul/datadog_checks/consul/data/conf.yaml.example
@@ -401,6 +401,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/coredns/datadog_checks/coredns/data/conf.yaml.example b/coredns/datadog_checks/coredns/data/conf.yaml.example
@@ -451,6 +451,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 10
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 10
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/couch/datadog_checks/couch/data/conf.yaml.example b/couch/datadog_checks/couch/data/conf.yaml.example
@@ -383,6 +383,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/couchbase/datadog_checks/couchbase/data/conf.yaml.example b/couchbase/datadog_checks/couchbase/data/conf.yaml.example
@@ -330,6 +330,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 16
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 16
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/crio/datadog_checks/crio/data/conf.yaml.example b/crio/datadog_checks/crio/data/conf.yaml.example
@@ -432,6 +432,11 @@ instances:
     #
     # read_timeout: <READ_TIMEOUT>
 
+    ## @param request_size - number - optional - default: 10
+    ## The number of kibibytes (KiB) to read from streaming HTTP responses at a time.
+    #
+    # request_size: 10
+
     ## @param log_requests - boolean - optional - default: false
     ## Whether or not to debug log the HTTP(S) requests made, including the method and URL.
     #

diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/base_check.py
@@ -61,6 +61,7 @@ class OpenMetricsBaseCheck(OpenMetricsScraperMixin, AgentCheck):
         'ssl_private_key': {'name': 'tls_private_key'},
         'ssl_ca_cert': {'name': 'tls_ca_cert'},
         'prometheus_timeout': {'name': 'timeout'},
+        'request_size': {'name': 'request_size', 'default': 10},
     }
 
     def __init__(self, *args, **kwargs):

diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py
@@ -29,7 +29,6 @@ class OpenMetricsScraperMixin(object):
     # This class is not supposed to be used by itself, it provides scraping behavior but
     # need to be within a check in the end
 
-    REQUESTS_CHUNK_SIZE = 1024 * 10  # use 10kb as chunk size when using the Stream feature in requests.get
     # indexes in the sample tuple of core.Metric
     SAMPLE_NAME = 0
     SAMPLE_LABELS = 1
@@ -421,7 +420,7 @@ def parse_metric_family(self, response, scraper_config):
         """
         if response.encoding is None:
             response.encoding = 'utf-8'
-        input_gen = response.iter_lines(chunk_size=self.REQUESTS_CHUNK_SIZE, decode_unicode=True)
+        input_gen = response.iter_lines(decode_unicode=True)
         if scraper_config['_text_filter_blacklist']:
             input_gen = self._text_filter_input(input_gen, scraper_config)
 

diff --git a/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py b/datadog_checks_base/datadog_checks/base/checks/openmetrics/v2/scraper.py
@@ -8,7 +8,6 @@
 from itertools import chain
 from math import isinf, isnan
 
-from binary import KIBIBYTE
 from prometheus_client.openmetrics.parser import text_fd_to_metric_families as parse_metric_families_strict
 from prometheus_client.parser import text_fd_to_metric_families as parse_metric_families
 
@@ -154,10 +153,6 @@ def __init__(self, check, config):
             ignored_tags_re = re.compile('|'.join(set(ignore_tags)))
             custom_tags = [tag for tag in custom_tags if not ignored_tags_re.search(tag)]
 
-        # 16 KiB seems optimal, and is also the standard chunk size of the Bittorrent protocol:
-        # https://www.bittorrent.org/beps/bep_0003.html
-        self.request_size = int(float(config.get('request_size') or 16) * KIBIBYTE)
-
         # These will be applied only to service checks
         self.static_tags = [f'endpoint:{self.endpoint}']
         self.static_tags.extend(custom_tags)
@@ -276,7 +271,7 @@ def generate_sample_data(self, metric):
 
     def stream_connection_lines(self):
         with self.get_connection() as connection:
-            for line in connection.iter_lines(chunk_size=self.request_size, decode_unicode=True):
+            for line in connection.iter_lines(decode_unicode=True):
                 yield line
 
     def filter_connection_lines(self, line_streamer):

diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in
@@ -92,3 +92,4 @@ typing==3.7.4.1; python_version < "3.0"
 uptime==3.0.1
 vertica-python==0.10.1
 win-inet-pton==1.1.0; sys_platform == "win32" and python_version < "3.0"
+wrapt==1.12.1