github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/internal/http_client.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  """Utility functions used for creating a common Http client from httplib2.
    19  
    20  For internal use only. No backwards compatibility guarantees.
    21  """
    22  # pytype: skip-file
    23  
    24  import logging
    25  import os
    26  import re
    27  
    28  import httplib2
    29  
    30  # This is the number of seconds the library will wait for GCS operations to
    31  # complete.
    32  DEFAULT_HTTP_TIMEOUT_SECONDS = 60
    33  
    34  
    35  def proxy_info_from_environment_var(proxy_env_var):
    36    """Reads proxy info from the environment and converts to httplib2.ProxyInfo.
    37  
    38    Args:
    39      proxy_env_var: environment variable string to read, http_proxy or
    40         https_proxy (in lower case).
    41         Example: http://myproxy.domain.com:8080
    42  
    43    Returns:
    44      httplib2.ProxyInfo constructed from the environment string.
    45    """
    46    proxy_url = os.environ.get(proxy_env_var)
    47    if not proxy_url:
    48      return None
    49    proxy_protocol = proxy_env_var.lower().split('_')[0]
    50    if not re.match('^https?://', proxy_url, flags=re.IGNORECASE):
    51      logging.warning(
    52          "proxy_info_from_url requires a protocol, which is always "
    53          "http or https.")
    54      proxy_url = proxy_protocol + '://' + proxy_url
    55    return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol)
    56  
    57  
    58  def get_new_http(timeout_secs=DEFAULT_HTTP_TIMEOUT_SECONDS):
    59    """Creates and returns a new httplib2.Http instance.
    60  
    61    Returns:
    62      An initialized httplib2.Http instance.
    63    """
    64    proxy_info = None
    65    for proxy_env_var in ['http_proxy', 'https_proxy']:
    66      if os.environ.get(proxy_env_var):
    67        proxy_info = proxy_info_from_environment_var(proxy_env_var)
    68        break
    69    # Use a non-infinite SSL timeout to avoid hangs during network flakiness.
    70    return httplib2.Http(proxy_info=proxy_info, timeout=timeout_secs)