edq.net.request

View Source

  1import http
  2import logging
  3import os
  4import time
  5import typing
  6import urllib.parse
  7import urllib3
  8
  9import requests
 10
 11import edq.core.errors
 12import edq.net.exchange
 13import edq.net.exchangeserver
 14import edq.net.settings
 15import edq.util.dirent
 16import edq.util.encoding
 17import edq.util.json
 18import edq.util.pyimport
 19
 20_logger = logging.getLogger(__name__)
 21
 22RETRY_BACKOFF_SECS: float = 0.5
 23""" A back-off factor between failed network requests. """
 24
 25@typing.runtime_checkable
 26class ResponseModifierFunction(typing.Protocol):
 27    """
 28    A function that can be used to modify an exchange's response.
 29    Exchanges can use these functions to normalize their responses before saving.
 30    """
 31
 32    def __call__(self,
 33            response: requests.Response,
 34            body: str,
 35            ) -> str:
 36        """
 37        Modify the http response.
 38        Headers may be modified in the response directly,
 39        while the modified (or same) body must be returned.
 40        """
 41
 42def make_request(method: str, url: str,
 43        headers: typing.Union[typing.Dict[str, typing.Any], None] = None,
 44        data: typing.Union[typing.Dict[str, typing.Any], None] = None,
 45        files: typing.Union[typing.List[typing.Any], None] = None,
 46        raise_for_status: bool = True,
 47        timeout_secs: typing.Union[float, typing.Tuple[float, float], None] = None,
 48        output_dir: typing.Union[str, None] = None,
 49        send_anchor_header: bool = True,
 50        headers_to_skip: typing.Union[typing.List[str], None] = None,
 51        params_to_skip: typing.Union[typing.List[str], None] = None,
 52        http_exchange_extension: str = edq.net.exchange.DEFAULT_HTTP_EXCHANGE_EXTENSION,
 53        add_http_prefix: bool = True,
 54        additional_requests_options: typing.Union[typing.Dict[str, typing.Any], None] = None,
 55        allow_redirects: typing.Union[bool, None] = True,
 56        retries: int = 0,
 57        https_verification: typing.Union[bool, None] = None,
 58        request_complete_callback: typing.Union[edq.net.exchange.HTTPExchangeComplete, None] = None,
 59        **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
 60    """
 61    Make an HTTP request and return the response object and text body.
 62
 63    For `timeout_secs`, see: https://docs.python-requests.org/en/latest/user/advanced/#timeouts
 64    """
 65
 66    if (add_http_prefix and (not url.lower().startswith('http'))):
 67        url = 'http://' + url
 68
 69    retries = max(0, retries)
 70
 71    if (output_dir is None):
 72        output_dir = edq.net.settings.get_exchanges_out_dir()
 73
 74    if (headers is None):
 75        headers = {}
 76
 77    if (data is None):
 78        data = {}
 79
 80    if (files is None):
 81        files = []
 82
 83    if (additional_requests_options is None):
 84        additional_requests_options = {}
 85
 86    if (request_complete_callback is None):
 87        raw_callback = edq.net.settings.get_request_complete_callback()
 88        if (raw_callback is not None):
 89            request_complete_callback = typing.cast(edq.net.exchange.HTTPExchangeComplete, raw_callback)
 90
 91    # Add in the anchor as a header (since it is not traditionally sent in an HTTP request).
 92    if (send_anchor_header):
 93        headers = headers.copy()
 94
 95        parts = urllib.parse.urlparse(url)
 96        headers[edq.net.settings.ANCHOR_HEADER_KEY] = parts.fragment.lstrip('#')
 97
 98    # Compute the full connection/read timeout.
 99    if (timeout_secs is None):
100        timeout_secs = (edq.net.settings.get_connection_timeout_secs(), edq.net.settings.get_read_timeout_secs())
101    elif (isinstance(timeout_secs, float)):
102        timeout_secs = (timeout_secs, timeout_secs)
103
104    options: typing.Dict[str, typing.Any] = {
105        'timeout': timeout_secs,
106    }
107
108    # Check for HTTPS verification.
109    if ((https_verification is False) or ((https_verification is None) and (not edq.net.settings.get_https_verification()))):
110        options['verify'] = False
111        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
112
113    options.update(additional_requests_options)
114
115    options.update({
116        'headers': headers,
117        'files': files,
118    })
119
120    if (allow_redirects is False):
121        options['allow_redirects'] = False
122
123    if (method == 'GET'):
124        options['params'] = data
125    else:
126        options['data'] = data
127
128    _logger.debug("Making %s request: '%s' (options = %s).", method, url, options)
129    response = _make_request_with_retry(method, url, options, retries)
130
131    body = response.text
132    if (_logger.level <= logging.DEBUG):
133        log_body = body
134        if (response.encoding is None):
135            log_body = f"<hash> {edq.util.hash.sha256_hex(response.content)}"
136
137        _logger.debug("Response:\n%s", log_body)
138
139    if (raise_for_status):
140        # Handle 404s a little special, as their body may contain useful information.
141        if ((response.status_code == http.HTTPStatus.NOT_FOUND) and (body is not None) and (body.strip() != '')):
142            response.reason += f" (Body: '{body.strip()}')"
143
144        response.raise_for_status()
145
146    exchange = None
147    if ((output_dir is not None) or (request_complete_callback is not None)):
148        exchange = edq.net.exchange.HTTPExchange.from_response(
149            response,
150            headers_to_skip = headers_to_skip,
151            params_to_skip = params_to_skip,
152            allow_redirects = options.get('allow_redirects', True),
153        )
154
155        if (output_dir is not None):
156            _write_exchange(exchange, output_dir, http_exchange_extension)
157
158            # Also write any redirects.
159            for redirect_response in response.history:
160                redirect_exchange = edq.net.exchange.HTTPExchange.from_response(
161                    redirect_response,
162                    headers_to_skip = headers_to_skip,
163                    params_to_skip = params_to_skip,
164                    allow_redirects = options.get('allow_redirects', True),
165                )
166                _write_exchange(redirect_exchange, output_dir, http_exchange_extension)
167
168        if (request_complete_callback is not None):
169            request_complete_callback(exchange)
170
171    return response, body
172
173def _write_exchange(exchange: edq.net.exchange.HTTPExchange, output_dir: str, http_exchange_extension: str) -> None:
174    """ Write an exchange to disk in the computed path. """
175
176    relpath = exchange.compute_relpath(http_exchange_extension = http_exchange_extension)
177    path = os.path.abspath(os.path.join(output_dir, relpath))
178
179    edq.util.dirent.mkdir(os.path.dirname(path))
180    edq.util.json.dump_path(exchange, path, indent = 4, sort_keys = False)
181
182def make_with_exchange(
183        exchange: edq.net.exchange.HTTPExchange,
184        base_url: str,
185        raise_for_status: bool = True,
186        **kwargs: typing.Any,
187        ) -> typing.Tuple[requests.Response, str]:
188    """ Perform the HTTP request described by the given exchange. """
189
190    files = []
191    for file_info in exchange.files:
192        content = file_info.content
193
194        # Content is base64 encoded.
195        if (file_info.b64_encoded and isinstance(content, str)):
196            content = edq.util.encoding.from_base64(content)
197
198        # Content is missing and must be in a file.
199        if (content is None):
200            content = open(file_info.path, 'rb')  # type: ignore[assignment,arg-type]  # pylint: disable=consider-using-with
201
202        files.append((file_info.name, content))
203
204    url = f"{base_url}/{exchange.get_url()}"
205
206    response, body = make_request(exchange.method, url,
207            headers = exchange.headers,
208            data = exchange.parameters,
209            files = files,
210            raise_for_status = raise_for_status,
211            allow_redirects = exchange.allow_redirects,
212            **kwargs,
213    )
214
215    if (exchange.response_modifier is not None):
216        modify_func = edq.util.pyimport.fetch(exchange.response_modifier)
217        body = modify_func(response, body)
218
219    return response, body
220
221
222def make_get(url: str, **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
223    """
224    Make a GET request and return the response object and text body.
225    """
226
227    return make_request('GET', url, **kwargs)
228
229def make_post(url: str, **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
230    """
231    Make a POST request and return the response object and text body.
232    """
233
234    return make_request('POST', url, **kwargs)
235
236def _make_request_with_retry(
237        method: str,
238        url: str,
239        options: typing.Dict[str, typing.Any],
240        retries: int,
241        ) -> requests.Response:
242    """ Make a request, retrying on failure. """
243
244    # Try once and then the number of allowed retries.
245    attempt_count = 1 + retries
246
247    errors = []
248    for attempt_index in range(attempt_count):
249        if (attempt_index > 0):
250            # Wait before the next retry.
251            time.sleep(attempt_index * RETRY_BACKOFF_SECS)
252
253        try:
254            response = requests.request(method, url, **options)  # pylint: disable=missing-timeout
255            break
256        except Exception as ex:
257            errors.append(ex)
258
259    if (len(errors) == attempt_count):
260        raise edq.core.errors.RetryError(f"HTTP {method} for '{url}'", attempt_count, retry_errors = errors)
261
262    return response

RETRY_BACKOFF_SECS: float = 0.5

A back-off factor between failed network requests.

@typing.runtime_checkable

class ResponseModifierFunction(typing.Protocol): View Source

26@typing.runtime_checkable
27class ResponseModifierFunction(typing.Protocol):
28    """
29    A function that can be used to modify an exchange's response.
30    Exchanges can use these functions to normalize their responses before saving.
31    """
32
33    def __call__(self,
34            response: requests.Response,
35            body: str,
36            ) -> str:
37        """
38        Modify the http response.
39        Headers may be modified in the response directly,
40        while the modified (or same) body must be returned.
41        """

A function that can be used to modify an exchange's response. Exchanges can use these functions to normalize their responses before saving.

ResponseModifierFunction(*args, **kwargs) View Source

1953def _no_init_or_replace_init(self, *args, **kwargs):
1954    cls = type(self)
1955
1956    if cls._is_protocol:
1957        raise TypeError('Protocols cannot be instantiated')
1958
1959    # Already using a custom `__init__`. No need to calculate correct
1960    # `__init__` to call. This can lead to RecursionError. See bpo-45121.
1961    if cls.__init__ is not _no_init_or_replace_init:
1962        return
1963
1964    # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`.
1965    # The first instantiation of the subclass will call `_no_init_or_replace_init` which
1966    # searches for a proper new `__init__` in the MRO. The new `__init__`
1967    # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent
1968    # instantiation of the protocol subclass will thus use the new
1969    # `__init__` and no longer call `_no_init_or_replace_init`.
1970    for base in cls.__mro__:
1971        init = base.__dict__.get('__init__', _no_init_or_replace_init)
1972        if init is not _no_init_or_replace_init:
1973            cls.__init__ = init
1974            break
1975    else:
1976        # should not happen
1977        cls.__init__ = object.__init__
1978
1979    cls.__init__(self, *args, **kwargs)

def make_request( method: str, url: str, headers: Optional[Dict[str, Any]] = None, data: Optional[Dict[str, Any]] = None, files: Optional[List[Any]] = None, raise_for_status: bool = True, timeout_secs: Union[float, Tuple[float, float], NoneType] = None, output_dir: Optional[str] = None, send_anchor_header: bool = True, headers_to_skip: Optional[List[str]] = None, params_to_skip: Optional[List[str]] = None, http_exchange_extension: str = '.httpex.json', add_http_prefix: bool = True, additional_requests_options: Optional[Dict[str, Any]] = None, allow_redirects: Optional[bool] = True, retries: int = 0, https_verification: Optional[bool] = None, request_complete_callback: Optional[edq.net.exchange.HTTPExchangeComplete] = None, **kwargs: Any) -> Tuple[requests.models.Response, str]: View Source

 43def make_request(method: str, url: str,
 44        headers: typing.Union[typing.Dict[str, typing.Any], None] = None,
 45        data: typing.Union[typing.Dict[str, typing.Any], None] = None,
 46        files: typing.Union[typing.List[typing.Any], None] = None,
 47        raise_for_status: bool = True,
 48        timeout_secs: typing.Union[float, typing.Tuple[float, float], None] = None,
 49        output_dir: typing.Union[str, None] = None,
 50        send_anchor_header: bool = True,
 51        headers_to_skip: typing.Union[typing.List[str], None] = None,
 52        params_to_skip: typing.Union[typing.List[str], None] = None,
 53        http_exchange_extension: str = edq.net.exchange.DEFAULT_HTTP_EXCHANGE_EXTENSION,
 54        add_http_prefix: bool = True,
 55        additional_requests_options: typing.Union[typing.Dict[str, typing.Any], None] = None,
 56        allow_redirects: typing.Union[bool, None] = True,
 57        retries: int = 0,
 58        https_verification: typing.Union[bool, None] = None,
 59        request_complete_callback: typing.Union[edq.net.exchange.HTTPExchangeComplete, None] = None,
 60        **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
 61    """
 62    Make an HTTP request and return the response object and text body.
 63
 64    For `timeout_secs`, see: https://docs.python-requests.org/en/latest/user/advanced/#timeouts
 65    """
 66
 67    if (add_http_prefix and (not url.lower().startswith('http'))):
 68        url = 'http://' + url
 69
 70    retries = max(0, retries)
 71
 72    if (output_dir is None):
 73        output_dir = edq.net.settings.get_exchanges_out_dir()
 74
 75    if (headers is None):
 76        headers = {}
 77
 78    if (data is None):
 79        data = {}
 80
 81    if (files is None):
 82        files = []
 83
 84    if (additional_requests_options is None):
 85        additional_requests_options = {}
 86
 87    if (request_complete_callback is None):
 88        raw_callback = edq.net.settings.get_request_complete_callback()
 89        if (raw_callback is not None):
 90            request_complete_callback = typing.cast(edq.net.exchange.HTTPExchangeComplete, raw_callback)
 91
 92    # Add in the anchor as a header (since it is not traditionally sent in an HTTP request).
 93    if (send_anchor_header):
 94        headers = headers.copy()
 95
 96        parts = urllib.parse.urlparse(url)
 97        headers[edq.net.settings.ANCHOR_HEADER_KEY] = parts.fragment.lstrip('#')
 98
 99    # Compute the full connection/read timeout.
100    if (timeout_secs is None):
101        timeout_secs = (edq.net.settings.get_connection_timeout_secs(), edq.net.settings.get_read_timeout_secs())
102    elif (isinstance(timeout_secs, float)):
103        timeout_secs = (timeout_secs, timeout_secs)
104
105    options: typing.Dict[str, typing.Any] = {
106        'timeout': timeout_secs,
107    }
108
109    # Check for HTTPS verification.
110    if ((https_verification is False) or ((https_verification is None) and (not edq.net.settings.get_https_verification()))):
111        options['verify'] = False
112        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
113
114    options.update(additional_requests_options)
115
116    options.update({
117        'headers': headers,
118        'files': files,
119    })
120
121    if (allow_redirects is False):
122        options['allow_redirects'] = False
123
124    if (method == 'GET'):
125        options['params'] = data
126    else:
127        options['data'] = data
128
129    _logger.debug("Making %s request: '%s' (options = %s).", method, url, options)
130    response = _make_request_with_retry(method, url, options, retries)
131
132    body = response.text
133    if (_logger.level <= logging.DEBUG):
134        log_body = body
135        if (response.encoding is None):
136            log_body = f"<hash> {edq.util.hash.sha256_hex(response.content)}"
137
138        _logger.debug("Response:\n%s", log_body)
139
140    if (raise_for_status):
141        # Handle 404s a little special, as their body may contain useful information.
142        if ((response.status_code == http.HTTPStatus.NOT_FOUND) and (body is not None) and (body.strip() != '')):
143            response.reason += f" (Body: '{body.strip()}')"
144
145        response.raise_for_status()
146
147    exchange = None
148    if ((output_dir is not None) or (request_complete_callback is not None)):
149        exchange = edq.net.exchange.HTTPExchange.from_response(
150            response,
151            headers_to_skip = headers_to_skip,
152            params_to_skip = params_to_skip,
153            allow_redirects = options.get('allow_redirects', True),
154        )
155
156        if (output_dir is not None):
157            _write_exchange(exchange, output_dir, http_exchange_extension)
158
159            # Also write any redirects.
160            for redirect_response in response.history:
161                redirect_exchange = edq.net.exchange.HTTPExchange.from_response(
162                    redirect_response,
163                    headers_to_skip = headers_to_skip,
164                    params_to_skip = params_to_skip,
165                    allow_redirects = options.get('allow_redirects', True),
166                )
167                _write_exchange(redirect_exchange, output_dir, http_exchange_extension)
168
169        if (request_complete_callback is not None):
170            request_complete_callback(exchange)
171
172    return response, body

Make an HTTP request and return the response object and text body.

For timeout_secs, see: https://docs.python-requests.org/en/latest/user/advanced/#timeouts

def make_with_exchange( exchange: edq.net.exchange.HTTPExchange, base_url: str, raise_for_status: bool = True, **kwargs: Any) -> Tuple[requests.models.Response, str]: View Source

183def make_with_exchange(
184        exchange: edq.net.exchange.HTTPExchange,
185        base_url: str,
186        raise_for_status: bool = True,
187        **kwargs: typing.Any,
188        ) -> typing.Tuple[requests.Response, str]:
189    """ Perform the HTTP request described by the given exchange. """
190
191    files = []
192    for file_info in exchange.files:
193        content = file_info.content
194
195        # Content is base64 encoded.
196        if (file_info.b64_encoded and isinstance(content, str)):
197            content = edq.util.encoding.from_base64(content)
198
199        # Content is missing and must be in a file.
200        if (content is None):
201            content = open(file_info.path, 'rb')  # type: ignore[assignment,arg-type]  # pylint: disable=consider-using-with
202
203        files.append((file_info.name, content))
204
205    url = f"{base_url}/{exchange.get_url()}"
206
207    response, body = make_request(exchange.method, url,
208            headers = exchange.headers,
209            data = exchange.parameters,
210            files = files,
211            raise_for_status = raise_for_status,
212            allow_redirects = exchange.allow_redirects,
213            **kwargs,
214    )
215
216    if (exchange.response_modifier is not None):
217        modify_func = edq.util.pyimport.fetch(exchange.response_modifier)
218        body = modify_func(response, body)
219
220    return response, body

Perform the HTTP request described by the given exchange.

def make_get(url: str, **kwargs: Any) -> Tuple[requests.models.Response, str]: View Source

223def make_get(url: str, **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
224    """
225    Make a GET request and return the response object and text body.
226    """
227
228    return make_request('GET', url, **kwargs)

Make a GET request and return the response object and text body.

def make_post(url: str, **kwargs: Any) -> Tuple[requests.models.Response, str]: View Source

230def make_post(url: str, **kwargs: typing.Any) -> typing.Tuple[requests.Response, str]:
231    """
232    Make a POST request and return the response object and text body.
233    """
234
235    return make_request('POST', url, **kwargs)

Make a POST request and return the response object and text body.