github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/clients/python/lakefs_sdk/api_client.py (about) 1 # coding: utf-8 2 3 """ 4 lakeFS API 5 6 lakeFS HTTP API 7 8 The version of the OpenAPI document: 1.0.0 9 Contact: services@treeverse.io 10 Generated by OpenAPI Generator (https://openapi-generator.tech) 11 12 Do not edit the class manually. 13 """ # noqa: E501 14 15 16 import atexit 17 import datetime 18 from dateutil.parser import parse 19 import json 20 import mimetypes 21 from multiprocessing.pool import ThreadPool 22 import os 23 import re 24 import tempfile 25 26 from urllib.parse import quote 27 28 from lakefs_sdk.configuration import Configuration 29 from lakefs_sdk.api_response import ApiResponse 30 import lakefs_sdk.models 31 from lakefs_sdk import rest 32 from lakefs_sdk.exceptions import ApiValueError, ApiException 33 34 35 class ApiClient(object): 36 """Generic API client for OpenAPI client library builds. 37 38 OpenAPI generic API client. This client handles the client- 39 server communication, and is invariant across implementations. Specifics of 40 the methods and models for each application are generated from the OpenAPI 41 templates. 42 43 :param configuration: .Configuration object for this client 44 :param header_name: a header to pass when making calls to the API. 45 :param header_value: a header value to pass when making calls to 46 the API. 47 :param cookie: a cookie to include in the header when making calls 48 to the API 49 :param pool_threads: The number of threads to use for async requests 50 to the API. More threads means more concurrent API requests. 51 """ 52 53 PRIMITIVE_TYPES = (float, bool, bytes, str, int) 54 NATIVE_TYPES_MAPPING = { 55 'int': int, 56 'long': int, # TODO remove as only py3 is supported? 57 'float': float, 58 'str': str, 59 'bool': bool, 60 'date': datetime.date, 61 'datetime': datetime.datetime, 62 'object': object, 63 } 64 _pool = None 65 66 def __init__(self, configuration=None, header_name=None, header_value=None, 67 cookie=None, pool_threads=1): 68 # use default configuration if none is provided 69 if configuration is None: 70 configuration = Configuration.get_default() 71 self.configuration = configuration 72 self.pool_threads = pool_threads 73 74 self.rest_client = rest.RESTClientObject(configuration) 75 self.default_headers = {} 76 if header_name is not None: 77 self.default_headers[header_name] = header_value 78 self.cookie = cookie 79 # Set default User-Agent. 80 self.user_agent = 'lakefs-python-sdk/0.1.0-SNAPSHOT' 81 self.client_side_validation = configuration.client_side_validation 82 83 def __enter__(self): 84 return self 85 86 def __exit__(self, exc_type, exc_value, traceback): 87 self.close() 88 89 def close(self): 90 if self._pool: 91 self._pool.close() 92 self._pool.join() 93 self._pool = None 94 if hasattr(atexit, 'unregister'): 95 atexit.unregister(self.close) 96 97 @property 98 def pool(self): 99 """Create thread pool on first request 100 avoids instantiating unused threadpool for blocking clients. 101 """ 102 if self._pool is None: 103 atexit.register(self.close) 104 self._pool = ThreadPool(self.pool_threads) 105 return self._pool 106 107 @property 108 def user_agent(self): 109 """User agent for this API client""" 110 return self.default_headers['User-Agent'] 111 112 @user_agent.setter 113 def user_agent(self, value): 114 self.default_headers['User-Agent'] = value 115 116 def set_default_header(self, header_name, header_value): 117 self.default_headers[header_name] = header_value 118 119 120 _default = None 121 122 @classmethod 123 def get_default(cls): 124 """Return new instance of ApiClient. 125 126 This method returns newly created, based on default constructor, 127 object of ApiClient class or returns a copy of default 128 ApiClient. 129 130 :return: The ApiClient object. 131 """ 132 if cls._default is None: 133 cls._default = ApiClient() 134 return cls._default 135 136 @classmethod 137 def set_default(cls, default): 138 """Set default instance of ApiClient. 139 140 It stores default ApiClient. 141 142 :param default: object of ApiClient. 143 """ 144 cls._default = default 145 146 def __call_api( 147 self, resource_path, method, path_params=None, 148 query_params=None, header_params=None, body=None, post_params=None, 149 files=None, response_types_map=None, auth_settings=None, 150 _return_http_data_only=None, collection_formats=None, 151 _preload_content=True, _request_timeout=None, _host=None, 152 _request_auth=None): 153 154 config = self.configuration 155 156 # header parameters 157 header_params = header_params or {} 158 header_params.update(self.default_headers) 159 if self.cookie: 160 header_params['Cookie'] = self.cookie 161 if header_params: 162 header_params = self.sanitize_for_serialization(header_params) 163 header_params = dict(self.parameters_to_tuples(header_params, 164 collection_formats)) 165 166 # path parameters 167 if path_params: 168 path_params = self.sanitize_for_serialization(path_params) 169 path_params = self.parameters_to_tuples(path_params, 170 collection_formats) 171 for k, v in path_params: 172 # specified safe chars, encode everything 173 resource_path = resource_path.replace( 174 '{%s}' % k, 175 quote(str(v), safe=config.safe_chars_for_path_param) 176 ) 177 178 # post parameters 179 if post_params or files: 180 post_params = post_params if post_params else [] 181 post_params = self.sanitize_for_serialization(post_params) 182 post_params = self.parameters_to_tuples(post_params, 183 collection_formats) 184 post_params.extend(self.files_parameters(files)) 185 186 # auth setting 187 self.update_params_for_auth( 188 header_params, query_params, auth_settings, 189 resource_path, method, body, 190 request_auth=_request_auth) 191 192 # body 193 if body: 194 body = self.sanitize_for_serialization(body) 195 196 # request url 197 if _host is None: 198 url = self.configuration.host + resource_path 199 else: 200 # use server/host defined in path or operation instead 201 url = _host + resource_path 202 203 # query parameters 204 if query_params: 205 query_params = self.sanitize_for_serialization(query_params) 206 url_query = self.parameters_to_url_query(query_params, 207 collection_formats) 208 url += "?" + url_query 209 210 try: 211 # perform request and return response 212 response_data = self.request( 213 method, url, 214 query_params=query_params, 215 headers=header_params, 216 post_params=post_params, body=body, 217 _preload_content=_preload_content, 218 _request_timeout=_request_timeout) 219 except ApiException as e: 220 if e.body: 221 e.body = e.body.decode('utf-8') 222 raise e 223 224 self.last_response = response_data 225 226 return_data = None # assuming derialization is not needed 227 # data needs deserialization or returns HTTP data (deserialized) only 228 if _preload_content or _return_http_data_only: 229 response_type = response_types_map.get(str(response_data.status), None) 230 231 if response_type == "bytearray": 232 response_data.data = response_data.data 233 else: 234 match = None 235 content_type = response_data.getheader('content-type') 236 if content_type is not None: 237 match = re.search(r"charset=([a-zA-Z\-\d]+)[\s;]?", content_type) 238 encoding = match.group(1) if match else "utf-8" 239 response_data.data = response_data.data.decode(encoding) 240 241 # deserialize response data 242 if response_type == "bytearray": 243 return_data = response_data.data 244 elif response_type: 245 return_data = self.deserialize(response_data, response_type) 246 else: 247 return_data = None 248 249 if _return_http_data_only: 250 return return_data 251 else: 252 return ApiResponse(status_code = response_data.status, 253 data = return_data, 254 headers = response_data.getheaders(), 255 raw_data = response_data.data) 256 257 def sanitize_for_serialization(self, obj): 258 """Builds a JSON POST object. 259 260 If obj is None, return None. 261 If obj is str, int, long, float, bool, return directly. 262 If obj is datetime.datetime, datetime.date 263 convert to string in iso8601 format. 264 If obj is list, sanitize each element in the list. 265 If obj is dict, return the dict. 266 If obj is OpenAPI model, return the properties dict. 267 268 :param obj: The data to serialize. 269 :return: The serialized form of data. 270 """ 271 if obj is None: 272 return None 273 elif isinstance(obj, self.PRIMITIVE_TYPES): 274 return obj 275 elif isinstance(obj, list): 276 return [self.sanitize_for_serialization(sub_obj) 277 for sub_obj in obj] 278 elif isinstance(obj, tuple): 279 return tuple(self.sanitize_for_serialization(sub_obj) 280 for sub_obj in obj) 281 elif isinstance(obj, (datetime.datetime, datetime.date)): 282 return obj.isoformat() 283 284 if isinstance(obj, dict): 285 obj_dict = obj 286 else: 287 # Convert model obj to dict except 288 # attributes `openapi_types`, `attribute_map` 289 # and attributes which value is not None. 290 # Convert attribute name to json key in 291 # model definition for request. 292 obj_dict = obj.to_dict() 293 294 return {key: self.sanitize_for_serialization(val) 295 for key, val in obj_dict.items()} 296 297 def deserialize(self, response, response_type): 298 """Deserializes response into an object. 299 300 :param response: RESTResponse object to be deserialized. 301 :param response_type: class literal for 302 deserialized object, or string of class name. 303 304 :return: deserialized object. 305 """ 306 # handle file downloading 307 # save response body into a tmp file and return the instance 308 if response_type == "file": 309 return self.__deserialize_file(response) 310 311 # fetch data from response object 312 try: 313 data = json.loads(response.data) 314 except ValueError: 315 data = response.data 316 317 return self.__deserialize(data, response_type) 318 319 def __deserialize(self, data, klass): 320 """Deserializes dict, list, str into an object. 321 322 :param data: dict, list or str. 323 :param klass: class literal, or string of class name. 324 325 :return: object. 326 """ 327 if data is None: 328 return None 329 330 if type(klass) == str: 331 if klass.startswith('List['): 332 sub_kls = re.match(r'List\[(.*)]', klass).group(1) 333 return [self.__deserialize(sub_data, sub_kls) 334 for sub_data in data] 335 336 if klass.startswith('Dict['): 337 sub_kls = re.match(r'Dict\[([^,]*), (.*)]', klass).group(2) 338 return {k: self.__deserialize(v, sub_kls) 339 for k, v in data.items()} 340 341 # convert str to class 342 if klass in self.NATIVE_TYPES_MAPPING: 343 klass = self.NATIVE_TYPES_MAPPING[klass] 344 else: 345 klass = getattr(lakefs_sdk.models, klass) 346 347 if klass in self.PRIMITIVE_TYPES: 348 return self.__deserialize_primitive(data, klass) 349 elif klass == object: 350 return self.__deserialize_object(data) 351 elif klass == datetime.date: 352 return self.__deserialize_date(data) 353 elif klass == datetime.datetime: 354 return self.__deserialize_datetime(data) 355 else: 356 return self.__deserialize_model(data, klass) 357 358 def call_api(self, resource_path, method, 359 path_params=None, query_params=None, header_params=None, 360 body=None, post_params=None, files=None, 361 response_types_map=None, auth_settings=None, 362 async_req=None, _return_http_data_only=None, 363 collection_formats=None, _preload_content=True, 364 _request_timeout=None, _host=None, _request_auth=None): 365 """Makes the HTTP request (synchronous) and returns deserialized data. 366 367 To make an async_req request, set the async_req parameter. 368 369 :param resource_path: Path to method endpoint. 370 :param method: Method to call. 371 :param path_params: Path parameters in the url. 372 :param query_params: Query parameters in the url. 373 :param header_params: Header parameters to be 374 placed in the request header. 375 :param body: Request body. 376 :param post_params dict: Request post form parameters, 377 for `application/x-www-form-urlencoded`, `multipart/form-data`. 378 :param auth_settings list: Auth Settings names for the request. 379 :param response: Response data type. 380 :param files dict: key -> filename, value -> filepath, 381 for `multipart/form-data`. 382 :param async_req bool: execute request asynchronously 383 :param _return_http_data_only: response data instead of ApiResponse 384 object with status code, headers, etc 385 :param _preload_content: if False, the ApiResponse.data will 386 be set to none and raw_data will store the 387 HTTP response body without reading/decoding. 388 Default is True. 389 :param collection_formats: dict of collection formats for path, query, 390 header, and post parameters. 391 :param _request_timeout: timeout setting for this request. If one 392 number provided, it will be total request 393 timeout. It can also be a pair (tuple) of 394 (connection, read) timeouts. 395 :param _request_auth: set to override the auth_settings for an a single 396 request; this effectively ignores the authentication 397 in the spec for a single request. 398 :type _request_token: dict, optional 399 :return: 400 If async_req parameter is True, 401 the request will be called asynchronously. 402 The method will return the request thread. 403 If parameter async_req is False or missing, 404 then the method will return the response directly. 405 """ 406 if not async_req: 407 return self.__call_api(resource_path, method, 408 path_params, query_params, header_params, 409 body, post_params, files, 410 response_types_map, auth_settings, 411 _return_http_data_only, collection_formats, 412 _preload_content, _request_timeout, _host, 413 _request_auth) 414 415 return self.pool.apply_async(self.__call_api, (resource_path, 416 method, path_params, 417 query_params, 418 header_params, body, 419 post_params, files, 420 response_types_map, 421 auth_settings, 422 _return_http_data_only, 423 collection_formats, 424 _preload_content, 425 _request_timeout, 426 _host, _request_auth)) 427 428 def request(self, method, url, query_params=None, headers=None, 429 post_params=None, body=None, _preload_content=True, 430 _request_timeout=None): 431 """Makes the HTTP request using RESTClient.""" 432 if method == "GET": 433 return self.rest_client.get_request(url, 434 query_params=query_params, 435 _preload_content=_preload_content, 436 _request_timeout=_request_timeout, 437 headers=headers) 438 elif method == "HEAD": 439 return self.rest_client.head_request(url, 440 query_params=query_params, 441 _preload_content=_preload_content, 442 _request_timeout=_request_timeout, 443 headers=headers) 444 elif method == "OPTIONS": 445 return self.rest_client.options_request(url, 446 query_params=query_params, 447 headers=headers, 448 _preload_content=_preload_content, 449 _request_timeout=_request_timeout) 450 elif method == "POST": 451 return self.rest_client.post_request(url, 452 query_params=query_params, 453 headers=headers, 454 post_params=post_params, 455 _preload_content=_preload_content, 456 _request_timeout=_request_timeout, 457 body=body) 458 elif method == "PUT": 459 return self.rest_client.put_request(url, 460 query_params=query_params, 461 headers=headers, 462 post_params=post_params, 463 _preload_content=_preload_content, 464 _request_timeout=_request_timeout, 465 body=body) 466 elif method == "PATCH": 467 return self.rest_client.patch_request(url, 468 query_params=query_params, 469 headers=headers, 470 post_params=post_params, 471 _preload_content=_preload_content, 472 _request_timeout=_request_timeout, 473 body=body) 474 elif method == "DELETE": 475 return self.rest_client.delete_request(url, 476 query_params=query_params, 477 headers=headers, 478 _preload_content=_preload_content, 479 _request_timeout=_request_timeout, 480 body=body) 481 else: 482 raise ApiValueError( 483 "http method must be `GET`, `HEAD`, `OPTIONS`," 484 " `POST`, `PATCH`, `PUT` or `DELETE`." 485 ) 486 487 def parameters_to_tuples(self, params, collection_formats): 488 """Get parameters as list of tuples, formatting collections. 489 490 :param params: Parameters as dict or list of two-tuples 491 :param dict collection_formats: Parameter collection formats 492 :return: Parameters as list of tuples, collections formatted 493 """ 494 new_params = [] 495 if collection_formats is None: 496 collection_formats = {} 497 for k, v in params.items() if isinstance(params, dict) else params: # noqa: E501 498 if k in collection_formats: 499 collection_format = collection_formats[k] 500 if collection_format == 'multi': 501 new_params.extend((k, value) for value in v) 502 else: 503 if collection_format == 'ssv': 504 delimiter = ' ' 505 elif collection_format == 'tsv': 506 delimiter = '\t' 507 elif collection_format == 'pipes': 508 delimiter = '|' 509 else: # csv is the default 510 delimiter = ',' 511 new_params.append( 512 (k, delimiter.join(str(value) for value in v))) 513 else: 514 new_params.append((k, v)) 515 return new_params 516 517 def parameters_to_url_query(self, params, collection_formats): 518 """Get parameters as list of tuples, formatting collections. 519 520 :param params: Parameters as dict or list of two-tuples 521 :param dict collection_formats: Parameter collection formats 522 :return: URL query string (e.g. a=Hello%20World&b=123) 523 """ 524 new_params = [] 525 if collection_formats is None: 526 collection_formats = {} 527 for k, v in params.items() if isinstance(params, dict) else params: # noqa: E501 528 if isinstance(v, (int, float)): 529 v = str(v) 530 if isinstance(v, bool): 531 v = str(v).lower() 532 if isinstance(v, dict): 533 v = json.dumps(v) 534 535 if k in collection_formats: 536 collection_format = collection_formats[k] 537 if collection_format == 'multi': 538 new_params.extend((k, value) for value in v) 539 else: 540 if collection_format == 'ssv': 541 delimiter = ' ' 542 elif collection_format == 'tsv': 543 delimiter = '\t' 544 elif collection_format == 'pipes': 545 delimiter = '|' 546 else: # csv is the default 547 delimiter = ',' 548 new_params.append( 549 (k, delimiter.join(quote(str(value)) for value in v))) 550 else: 551 new_params.append((k, quote(str(v)))) 552 553 return "&".join(["=".join(item) for item in new_params]) 554 555 def files_parameters(self, files=None): 556 """Builds form parameters. 557 558 :param files: File parameters. 559 :return: Form parameters with files. 560 """ 561 params = [] 562 563 if files: 564 for k, v in files.items(): 565 if not v: 566 continue 567 file_names = v if type(v) is list else [v] 568 for n in file_names: 569 with open(n, 'rb') as f: 570 filename = os.path.basename(f.name) 571 filedata = f.read() 572 mimetype = (mimetypes.guess_type(filename)[0] or 573 'application/octet-stream') 574 params.append( 575 tuple([k, tuple([filename, filedata, mimetype])])) 576 577 return params 578 579 def select_header_accept(self, accepts): 580 """Returns `Accept` based on an array of accepts provided. 581 582 :param accepts: List of headers. 583 :return: Accept (e.g. application/json). 584 """ 585 if not accepts: 586 return 587 588 for accept in accepts: 589 if re.search('json', accept, re.IGNORECASE): 590 return accept 591 592 return accepts[0] 593 594 def select_header_content_type(self, content_types): 595 """Returns `Content-Type` based on an array of content_types provided. 596 597 :param content_types: List of content-types. 598 :return: Content-Type (e.g. application/json). 599 """ 600 if not content_types: 601 return None 602 603 for content_type in content_types: 604 if re.search('json', content_type, re.IGNORECASE): 605 return content_type 606 607 return content_types[0] 608 609 def update_params_for_auth(self, headers, queries, auth_settings, 610 resource_path, method, body, 611 request_auth=None): 612 """Updates header and query params based on authentication setting. 613 614 :param headers: Header parameters dict to be updated. 615 :param queries: Query parameters tuple list to be updated. 616 :param auth_settings: Authentication setting identifiers list. 617 :resource_path: A string representation of the HTTP request resource path. 618 :method: A string representation of the HTTP request method. 619 :body: A object representing the body of the HTTP request. 620 The object type is the return value of sanitize_for_serialization(). 621 :param request_auth: if set, the provided settings will 622 override the token in the configuration. 623 """ 624 if not auth_settings: 625 return 626 627 if request_auth: 628 self._apply_auth_params(headers, queries, 629 resource_path, method, body, 630 request_auth) 631 return 632 633 for auth in auth_settings: 634 auth_setting = self.configuration.auth_settings().get(auth) 635 if auth_setting: 636 self._apply_auth_params(headers, queries, 637 resource_path, method, body, 638 auth_setting) 639 640 def _apply_auth_params(self, headers, queries, 641 resource_path, method, body, 642 auth_setting): 643 """Updates the request parameters based on a single auth_setting 644 645 :param headers: Header parameters dict to be updated. 646 :param queries: Query parameters tuple list to be updated. 647 :resource_path: A string representation of the HTTP request resource path. 648 :method: A string representation of the HTTP request method. 649 :body: A object representing the body of the HTTP request. 650 The object type is the return value of sanitize_for_serialization(). 651 :param auth_setting: auth settings for the endpoint 652 """ 653 if auth_setting['in'] == 'cookie': 654 headers['Cookie'] = auth_setting['value'] 655 elif auth_setting['in'] == 'header': 656 if auth_setting['type'] != 'http-signature': 657 headers[auth_setting['key']] = auth_setting['value'] 658 elif auth_setting['in'] == 'query': 659 queries.append((auth_setting['key'], auth_setting['value'])) 660 else: 661 raise ApiValueError( 662 'Authentication token must be in `query` or `header`' 663 ) 664 665 def __deserialize_file(self, response): 666 """Deserializes body to file 667 668 Saves response body into a file in a temporary folder, 669 using the filename from the `Content-Disposition` header if provided. 670 671 :param response: RESTResponse. 672 :return: file path. 673 """ 674 fd, path = tempfile.mkstemp(dir=self.configuration.temp_folder_path) 675 os.close(fd) 676 os.remove(path) 677 678 content_disposition = response.getheader("Content-Disposition") 679 if content_disposition: 680 filename = re.search(r'filename=[\'"]?([^\'"\s]+)[\'"]?', 681 content_disposition).group(1) 682 path = os.path.join(os.path.dirname(path), filename) 683 684 with open(path, "wb") as f: 685 f.write(response.data) 686 687 return path 688 689 def __deserialize_primitive(self, data, klass): 690 """Deserializes string to primitive type. 691 692 :param data: str. 693 :param klass: class literal. 694 695 :return: int, long, float, str, bool. 696 """ 697 try: 698 return klass(data) 699 except UnicodeEncodeError: 700 return str(data) 701 except TypeError: 702 return data 703 704 def __deserialize_object(self, value): 705 """Return an original value. 706 707 :return: object. 708 """ 709 return value 710 711 def __deserialize_date(self, string): 712 """Deserializes string to date. 713 714 :param string: str. 715 :return: date. 716 """ 717 try: 718 return parse(string).date() 719 except ImportError: 720 return string 721 except ValueError: 722 raise rest.ApiException( 723 status=0, 724 reason="Failed to parse `{0}` as date object".format(string) 725 ) 726 727 def __deserialize_datetime(self, string): 728 """Deserializes string to datetime. 729 730 The string should be in iso8601 datetime format. 731 732 :param string: str. 733 :return: datetime. 734 """ 735 try: 736 return parse(string) 737 except ImportError: 738 return string 739 except ValueError: 740 raise rest.ApiException( 741 status=0, 742 reason=( 743 "Failed to parse `{0}` as datetime object" 744 .format(string) 745 ) 746 ) 747 748 def __deserialize_model(self, data, klass): 749 """Deserializes list or dict to model. 750 751 :param data: dict, list. 752 :param klass: class literal. 753 :return: model object. 754 """ 755 756 return klass.from_dict(data)