Add RequestsWrapper option to support UTF-8 for basic auth (DataDog#7441

) * Add RequestsWrapper option to support UTF-8 for basic auth * Update test_load.py
kkourtchikov · Aug 28, 2020 · a666c42 · a666c42
1 parent b3fb456
commit a666c42
Show file tree

Hide file tree

Showing 40 changed files with 235 additions and 2 deletions.
diff --git a/activemq_xml/datadog_checks/activemq_xml/data/conf.yaml.example b/activemq_xml/datadog_checks/activemq_xml/data/conf.yaml.example
@@ -116,6 +116,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -140,6 +141,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/airflow/datadog_checks/airflow/data/conf.yaml.example b/airflow/datadog_checks/airflow/data/conf.yaml.example
@@ -87,6 +87,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -111,6 +112,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example b/amazon_msk/datadog_checks/amazon_msk/data/conf.yaml.example
@@ -205,6 +205,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -229,6 +230,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/ambari/datadog_checks/ambari/data/conf.yaml.example b/ambari/datadog_checks/ambari/data/conf.yaml.example
@@ -95,6 +95,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -150,6 +151,11 @@ instances:
     #
     # skip_proxy: false
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param ntlm_domain - string - optional
     ## If your services use NTLM authentication, specify
     ## the domain used in the check. For NTLM Auth, append

diff --git a/apache/datadog_checks/apache/data/conf.yaml.example b/apache/datadog_checks/apache/data/conf.yaml.example
@@ -87,6 +87,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -111,6 +112,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/cilium/datadog_checks/cilium/data/conf.yaml.example b/cilium/datadog_checks/cilium/data/conf.yaml.example
@@ -159,6 +159,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -183,6 +184,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example
@@ -150,6 +150,11 @@ instances:
     #
     # skip_proxy: false
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param tls_verify - boolean - optional - default: true
     ## Instructs the check to validate the TLS certificate of services.
     #

diff --git a/cloud_foundry_api/datadog_checks/cloud_foundry_api/data/conf.yaml.example b/cloud_foundry_api/datadog_checks/cloud_foundry_api/data/conf.yaml.example
@@ -94,6 +94,11 @@ instances:
     #
     # skip_proxy: false
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param tls_verify - boolean - optional - default: true
     ## Instructs the check to validate the TLS certificate of services.
     #

diff --git a/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example b/cockroachdb/datadog_checks/cockroachdb/data/conf.yaml.example
@@ -183,6 +183,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -207,6 +208,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/consul/datadog_checks/consul/data/conf.yaml.example b/consul/datadog_checks/consul/data/conf.yaml.example
@@ -156,6 +156,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -180,6 +181,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/couch/datadog_checks/couch/data/conf.yaml.example b/couch/datadog_checks/couch/data/conf.yaml.example
@@ -162,6 +162,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -186,6 +187,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param ntlm_domain - string - optional
     ## If your services use NTLM authentication, specify
     ## the domain used in the check. For NTLM Auth, append

diff --git a/datadog_checks_base/datadog_checks/base/utils/http.py b/datadog_checks_base/datadog_checks/base/utils/http.py
@@ -15,7 +15,7 @@
 
 from ..config import is_affirmative
 from ..errors import ConfigurationError
-from .common import ensure_unicode
+from .common import ensure_bytes, ensure_unicode
 from .headers import get_default_headers, update_headers
 from .warnings_util import disable_warnings_ctx
 
@@ -70,6 +70,7 @@
     'tls_private_key': None,
     'tls_verify': True,
     'timeout': DEFAULT_TIMEOUT,
+    'use_legacy_auth_encoding': True,
     'username': None,
 }
 # For any known legacy fields that may be widespread
@@ -428,7 +429,10 @@ def should_bypass_proxy(url, no_proxy_uris):
 def create_basic_auth(config):
     # Since this is the default case, only activate when all fields are explicitly set
     if config['username'] and config['password']:
-        return (config['username'], config['password'])
+        if config['use_legacy_auth_encoding']:
+            return config['username'], config['password']
+        else:
+            return ensure_bytes(config['username']), ensure_bytes(config['password'])
 
 
 def create_digest_auth(config):

diff --git a/datadog_checks_base/tests/test_http.py b/datadog_checks_base/tests/test_http.py
@@ -219,6 +219,13 @@ def test_config_basic_authtype(self):
 
         assert http.options['auth'] == ('user', 'pass')
 
+    def test_config_basic_no_legacy_encoding(self):
+        instance = {'username': 'user', 'password': 'pass', 'use_legacy_auth_encoding': False}
+        init_config = {}
+        http = RequestsWrapper(instance, init_config)
+
+        assert http.options['auth'] == (b'user', b'pass')
+
     def test_config_digest_authtype(self):
         instance = {'username': 'user', 'password': 'pass', 'auth_type': 'digest'}
         init_config = {}

diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/http.yaml b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/http.yaml
@@ -50,6 +50,7 @@
       - basic
         |__ username
         |__ password
+        |__ use_legacy_auth_encoding
       - digest
         |__ username
         |__ password
@@ -72,6 +73,12 @@
     The `aws` auth type relies on boto3 to automatically gather AWS credentials, for example: from `.aws/credentials`. /noqa
     Details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials /noqa
 
+- name: use_legacy_auth_encoding
+  value:
+    example: true
+    type: boolean
+  description: |
+    When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
 - name: username
   value:
     type: string

diff --git a/datadog_checks_dev/tests/tooling/configuration/test_load.py b/datadog_checks_dev/tests/tooling/configuration/test_load.py
@@ -2718,6 +2718,7 @@ def test_template_array():
         'proxy',
         'skip_proxy',
         'auth_type',
+        'use_legacy_auth_encoding',
         'username',
         'password',
         'ntlm_domain',

diff --git a/druid/datadog_checks/druid/data/conf.yaml.example b/druid/datadog_checks/druid/data/conf.yaml.example
@@ -87,6 +87,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -111,6 +112,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/elastic/datadog_checks/elastic/data/conf.yaml.example b/elastic/datadog_checks/elastic/data/conf.yaml.example
@@ -170,6 +170,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -194,6 +195,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/envoy/datadog_checks/envoy/data/conf.yaml.example b/envoy/datadog_checks/envoy/data/conf.yaml.example
@@ -175,6 +175,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -199,6 +200,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param ntlm_domain - string - optional
     ## If your services use NTLM authentication, specify
     ## the domain used in the check. For NTLM Auth, append

diff --git a/fluentd/datadog_checks/fluentd/data/conf.yaml.example b/fluentd/datadog_checks/fluentd/data/conf.yaml.example
@@ -140,6 +140,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -164,6 +165,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/gitlab/datadog_checks/gitlab/data/conf.yaml.example b/gitlab/datadog_checks/gitlab/data/conf.yaml.example
@@ -153,6 +153,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -177,6 +178,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #

diff --git a/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example b/gitlab_runner/datadog_checks/gitlab_runner/data/conf.yaml.example
@@ -237,6 +237,7 @@ instances:
     ##   - basic
     ##     |__ username
     ##     |__ password
+    ##     |__ use_legacy_auth_encoding
     ##   - digest
     ##     |__ username
     ##     |__ password
@@ -261,6 +262,11 @@ instances:
     #
     # auth_type: basic
 
+    ## @param use_legacy_auth_encoding - boolean - optional - default: true
+    ## When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`.
+    #
+    # use_legacy_auth_encoding: true
+
     ## @param username - string - optional
     ## The username to use if services are behind basic or digest auth.
     #