import datetime import logging import mimetypes import os import re import tempfile import time from typing import Dict import omagent_core.engine.http.models as http_models import six import urllib3 from omagent_core.engine.configuration.configuration import Configuration from omagent_core.engine.http import rest from omagent_core.engine.http.rest import AuthorizationException from omagent_core.engine.http.thread import AwaitableThread from omagent_core.utils.container import container from requests.structures import CaseInsensitiveDict from six.moves.urllib.parse import quote logger = logging.getLogger(Configuration.get_logging_formatted_name(__name__)) class ApiClient(object): PRIMITIVE_TYPES = (float, bool, bytes, six.text_type) + six.integer_types NATIVE_TYPES_MAPPING = { "int": int, "long": int if six.PY3 else long, # noqa: F821 "float": float, "str": str, "bool": bool, "date": datetime.date, "datetime": datetime.datetime, "object": object, } def __init__( self, configuration=None, header_name=None, header_value=None, cookie=None ): self.rest_client = rest.RESTClientObject( connection=container.conductor_config.http_connection ) self.default_headers = self.__get_default_headers(header_name, header_value) self.cookie = cookie self.__refresh_auth_token() def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, ): try: return self.__call_api_no_retry( resource_path=resource_path, method=method, path_params=path_params, query_params=query_params, header_params=header_params, body=body, post_params=post_params, files=files, response_type=response_type, auth_settings=auth_settings, _return_http_data_only=_return_http_data_only, collection_formats=collection_formats, _preload_content=_preload_content, _request_timeout=_request_timeout, ) except AuthorizationException as ae: if ae.token_expired: logger.error( f"authentication token has expired, refreshing the token. request= {method} {resource_path}" ) # if the token has expired, lets refresh the token self.__force_refresh_auth_token() # and now retry the same request return self.__call_api_no_retry( resource_path=resource_path, method=method, path_params=path_params, query_params=query_params, header_params=header_params, body=body, post_params=post_params, files=files, response_type=response_type, auth_settings=auth_settings, _return_http_data_only=_return_http_data_only, collection_formats=collection_formats, _preload_content=_preload_content, _request_timeout=_request_timeout, ) raise ae def __call_api_no_retry( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, ): config = container.conductor_config # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params["Cookie"] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict( self.parameters_to_tuples(header_params, collection_formats) ) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( "{%s}" % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = self.prepare_post_parameters(post_params, files) post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) # auth setting auth_headers = None if ( container.conductor_config.authentication_settings is not None and resource_path != "/token" ): auth_headers = self.__get_authentication_headers() self.update_params_for_auth(header_params, query_params, auth_headers) # body if body: body = self.sanitize_for_serialization(body) # request url url = container.conductor_config.host + resource_path # perform request and return response response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout, ) self.last_response = response_data return_data = response_data if _preload_content: # deserialize response data if response_type: return_data = self.deserialize(response_data, response_type) else: return_data = None if _return_http_data_only: return return_data else: return (return_data, response_data.status, response_data.getheaders()) def sanitize_for_serialization(self, obj): """Builds a JSON POST object. If obj is None, return None. If obj is str, int, long, float, bool, return directly. If obj is datetime.datetime, datetime.date convert to string in iso8601 format. If obj is list, sanitize each element in the list. If obj is dict, return the dict. If obj is swagger model, return the properties dict. :param obj: The data to serialize. :return: The serialized form of data. """ if obj is None: return None elif isinstance(obj, self.PRIMITIVE_TYPES): return obj elif isinstance(obj, list): return [self.sanitize_for_serialization(sub_obj) for sub_obj in obj] elif isinstance(obj, tuple): return tuple(self.sanitize_for_serialization(sub_obj) for sub_obj in obj) elif isinstance(obj, (datetime.datetime, datetime.date)): return obj.isoformat() if isinstance(obj, dict) or isinstance(obj, CaseInsensitiveDict): obj_dict = obj else: # Convert model obj to dict except # attributes `swagger_types`, `attribute_map` # and attributes which value is not None. # Convert attribute name to json key in # model definition for request. if hasattr(obj, "attribute_map") and hasattr(obj, "swagger_types"): obj_dict = { obj.attribute_map[attr]: getattr(obj, attr) for attr, _ in six.iteritems(obj.swagger_types) if getattr(obj, attr) is not None } else: obj_dict = { name: getattr(obj, name) for name in vars(obj) if getattr(obj, name) is not None } return { key: self.sanitize_for_serialization(val) for key, val in six.iteritems(obj_dict) } def deserialize(self, response, response_type): """Deserializes response into an object. :param response: RESTResponse object to be deserialized. :param response_type: class literal for deserialized object, or string of class name. :return: deserialized object. """ # handle file downloading # save response body into a tmp file and return the instance if response_type == "file": return self.__deserialize_file(response) # fetch data from response object try: data = response.resp.json() except Exception: data = response.resp.text try: return self.__deserialize(data, response_type) except ValueError as e: logger.error( f"failed to deserialize data {data} into class {response_type}, reason: {e}" ) return None def deserialize_class(self, data, klass): return self.__deserialize(data, klass) def __deserialize(self, data, klass): """Deserializes dict, list, str into an object. :param data: dict, list or str. :param klass: class literal, or string of class name. :return: object. """ if data is None: return None if type(klass) == str: if klass.startswith("list["): sub_kls = re.match(r"list\[(.*)\]", klass).group(1) return [self.__deserialize(sub_data, sub_kls) for sub_data in data] if klass.startswith("dict("): sub_kls = re.match(r"dict\(([^,]*), (.*)\)", klass).group(2) return { k: self.__deserialize(v, sub_kls) for k, v in six.iteritems(data) } # convert str to class if klass in self.NATIVE_TYPES_MAPPING: klass = self.NATIVE_TYPES_MAPPING[klass] else: klass = getattr(http_models, klass) if klass in self.PRIMITIVE_TYPES: return self.__deserialize_primitive(data, klass) elif klass == object: return self.__deserialize_object(data) elif klass == datetime.date: return self.__deserialize_date(data) elif klass == datetime.datetime: return self.__deserialize_datatime(data) else: return self.__deserialize_model(data, klass) def call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, ): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: return self.__call_api( resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, ) thread = AwaitableThread( target=self.__call_api, args=( resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, ), ) thread.start() return thread def request( self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None, ): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.GET( url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers, ) elif method == "HEAD": return self.rest_client.HEAD( url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers, ) elif method == "OPTIONS": return self.rest_client.OPTIONS( url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body, ) elif method == "POST": return self.rest_client.POST( url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body, ) elif method == "PUT": return self.rest_client.PUT( url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body, ) elif method == "PATCH": return self.rest_client.PATCH( url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body, ) elif method == "DELETE": return self.rest_client.DELETE( url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body, ) else: raise ValueError( "http method must be `GET`, `HEAD`, `OPTIONS`," " `POST`, `PATCH`, `PUT` or `DELETE`." ) def parameters_to_tuples(self, params, collection_formats): """Get parameters as list of tuples, formatting collections. :param params: Parameters as dict or list of two-tuples :param dict collection_formats: Parameter collection formats :return: Parameters as list of tuples, collections formatted """ new_params = [] if collection_formats is None: collection_formats = {} for k, v in ( six.iteritems(params) if isinstance(params, dict) else params ): # noqa: E501 if k in collection_formats: collection_format = collection_formats[k] if collection_format == "multi": new_params.extend((k, value) for value in v) else: if collection_format == "ssv": delimiter = " " elif collection_format == "tsv": delimiter = "\t" elif collection_format == "pipes": delimiter = "|" else: # csv is the default delimiter = "," new_params.append((k, delimiter.join(str(value) for value in v))) else: new_params.append((k, v)) return new_params def prepare_post_parameters(self, post_params=None, files=None): """Builds form parameters. :param post_params: Normal form parameters. :param files: File parameters. :return: Form parameters with files. """ params = [] if post_params: params = post_params if files: for k, v in six.iteritems(files): if not v: continue file_names = v if type(v) is list else [v] for n in file_names: with open(n, "rb") as f: filename = os.path.basename(f.name) filedata = f.read() mimetype = ( mimetypes.guess_type(filename)[0] or "application/octet-stream" ) params.append(tuple([k, tuple([filename, filedata, mimetype])])) return params def select_header_accept(self, accepts): """Returns `Accept` based on an array of accepts provided. :param accepts: List of headers. :return: Accept (e.g. application/json). """ if not accepts: return accepts = [x.lower() for x in accepts] if "application/json" in accepts: return "application/json" else: return ", ".join(accepts) def select_header_content_type(self, content_types): """Returns `Content-Type` based on an array of content_types provided. :param content_types: List of content-types. :return: Content-Type (e.g. application/json). """ if not content_types: return "application/json" content_types = [x.lower() for x in content_types] if "application/json" in content_types or "*/*" in content_types: return "application/json" else: return content_types[0] def update_params_for_auth(self, headers, querys, auth_settings): """Updates header and query params based on authentication setting. :param headers: Header parameters dict to be updated. :param querys: Query parameters tuple list to be updated. :param auth_settings: Authentication setting identifiers list. """ if not auth_settings: return if "header" in auth_settings: for key, value in auth_settings["header"].items(): headers[key] = value if "query" in auth_settings: for key, value in auth_settings["query"].items(): querys[key] = value def __deserialize_file(self, response): """Deserializes body to file Saves response body into a file in a temporary folder, using the filename from the `Content-Disposition` header if provided. :param response: RESTResponse. :return: file path. """ fd, path = tempfile.mkstemp(dir=container.conductor_config.temp_folder_path) os.close(fd) os.remove(path) content_disposition = response.getheader("Content-Disposition") if content_disposition: filename = re.search( r'filename=[\'"]?([^\'"\s]+)[\'"]?', content_disposition ).group(1) path = os.path.join(os.path.dirname(path), filename) response_data = response.data with open(path, "wb") as f: if isinstance(response_data, str): # change str to bytes so we can write it response_data = response_data.encode("utf-8") f.write(response_data) else: f.write(response_data) return path def __deserialize_primitive(self, data, klass): """Deserializes string to primitive type. :param data: str. :param klass: class literal. :return: int, long, float, str, bool. """ try: if klass == str and type(data) == bytes: return self.__deserialize_bytes_to_str(data) return klass(data) except UnicodeEncodeError: return six.text_type(data) except TypeError: return data def __deserialize_bytes_to_str(self, data): return data.decode("utf-8") def __deserialize_object(self, value): """Return a original value. :return: object. """ return value def __deserialize_date(self, string): """Deserializes string to date. :param string: str. :return: date. """ try: from dateutil.parser import parse return parse(string).date() except ImportError: return string except ValueError: raise rest.ApiException( status=0, reason="Failed to parse `{0}` as date object".format(string) ) def __deserialize_datatime(self, string): """Deserializes string to datetime. The string should be in iso8601 datetime format. :param string: str. :return: datetime. """ try: from dateutil.parser import parse return parse(string) except ImportError: return string except ValueError: raise rest.ApiException( status=0, reason=("Failed to parse `{0}` as datetime object".format(string)), ) def __hasattr(self, object, name): return name in object.__class__.__dict__ def __deserialize_model(self, data, klass): """Deserializes list or dict to model. :param data: dict, list. :param klass: class literal. :return: model object. """ if not klass.swagger_types and not self.__hasattr( klass, "get_real_child_model" ): return data kwargs = {} if klass.swagger_types is not None: for attr, attr_type in six.iteritems(klass.swagger_types): if ( data is not None and klass.attribute_map[attr] in data and isinstance(data, (list, dict)) ): value = data[klass.attribute_map[attr]] kwargs[attr] = self.__deserialize(value, attr_type) instance = klass(**kwargs) if ( isinstance(instance, dict) and klass.swagger_types is not None and isinstance(data, dict) ): for key, value in data.items(): if key not in klass.swagger_types: instance[key] = value if self.__hasattr(instance, "get_real_child_model"): klass_name = instance.get_real_child_model(data) if klass_name: instance = self.__deserialize(data, klass_name) return instance def __get_authentication_headers(self): if container.conductor_config.AUTH_TOKEN is None: return None now = round(time.time() * 1000) time_since_last_update = now - container.conductor_config.token_update_time if time_since_last_update > container.conductor_config.auth_token_ttl_msec: # time to refresh the token logger.debug(f"refreshing authentication token") token = self.__get_new_token() container.conductor_config.update_token(token) return {"header": {"X-Authorization": container.conductor_config.AUTH_TOKEN}} def __refresh_auth_token(self) -> None: if container.conductor_config.AUTH_TOKEN is not None: return if container.conductor_config.authentication_settings is None: return token = self.__get_new_token() container.conductor_config.update_token(token) def __force_refresh_auth_token(self) -> None: """ Forces the token refresh. Unlike the __refresh_auth_token method above """ if container.conductor_config.authentication_settings is None: return token = self.__get_new_token() container.conductor_config.update_token(token) def __get_new_token(self) -> str: try: if ( container.conductor_config.authentication_settings.key_id is None or container.conductor_config.authentication_settings.key_secret is None ): logger.error( "Authentication Key or Secret is set. Failed to get the auth token" ) return None response = self.call_api( "/token", "POST", header_params={ "Content-Type": self.select_header_content_type(["*/*"]) }, body={ "keyId": container.conductor_config.authentication_settings.key_id, "keySecret": container.conductor_config.authentication_settings.key_secret, }, _return_http_data_only=True, response_type="Token", ) return response.token except Exception as e: logger.error(f"Failed to get new token, reason: {e.args}") return None def __get_default_headers( self, header_name: str, header_value: object ) -> Dict[str, object]: headers = { "Accept-Encoding": "gzip", } if header_name is not None: headers[header_name] = header_value parsed = urllib3.util.parse_url(container.conductor_config.host) if parsed.auth is not None: encrypted_headers = urllib3.util.make_headers(basic_auth=parsed.auth) for key, value in encrypted_headers.items(): headers[key] = value return headers