From 68585b38cb5c4d2fea3dfe013dcab0efb46e553b Mon Sep 17 00:00:00 2001 From: Leah Date: Sat, 11 Jul 2020 21:57:53 -0400 Subject: [PATCH 1/8] add file utils --- gql/utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/gql/utils.py b/gql/utils.py index 8f47d97d..33318eeb 100644 --- a/gql/utils.py +++ b/gql/utils.py @@ -1,5 +1,8 @@ """Utilities to manipulate several python objects.""" +import io +from typing import Dict, List, Any, Union + # From this response in Stackoverflow # http://stackoverflow.com/a/19053800/1072990 @@ -8,3 +11,27 @@ def to_camel_case(snake_str): # We capitalize the first letter of each component except the first one # with the 'title' method and join them together. return components[0] + "".join(x.title() if x else "_" for x in components[1:]) + + +def is_file_like(value: Any) -> bool: + """Check if a value represents a file like object""" + return isinstance(value, io.IOBase) + + +def is_file_like_list(value: Any) -> bool: + """Check if value is a list and if all items in the list are file-like""" + return isinstance(value, list) and all(is_file_like(item) for item in value) + + +def contains_file_like_values(value: Any) -> bool: + return is_file_like(value) or is_file_like_list(value) + + +def get_file_variables( + variables: Dict[str, Any] +) -> Dict[str, Union[io.IOBase, List[io.IOBase]]]: + return { + variable: value + for variable, value in variables.items() + if contains_file_like_values(value) + } From aabd2de9029355d56e84b3cf2c5870bc84ab5d7e Mon Sep 17 00:00:00 2001 From: Leah Date: Sat, 11 Jul 2020 21:59:49 -0400 Subject: [PATCH 2/8] initial add for file support --- gql/transport/requests.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/gql/transport/requests.py b/gql/transport/requests.py index 8eb4b2f8..bb3feb63 100644 --- a/gql/transport/requests.py +++ b/gql/transport/requests.py @@ -1,4 +1,5 @@ from typing import Any, Dict, Optional, Union +import json import requests from graphql import DocumentNode, ExecutionResult, print_ast @@ -8,6 +9,7 @@ from gql.transport import Transport +from .utils import get_file_variables, is_file_like, is_file_like_list from .exceptions import ( TransportAlreadyConnected, TransportClosed, @@ -120,7 +122,28 @@ def execute( # type: ignore query_str = print_ast(document) payload: Dict[str, Any] = {"query": query_str} if variable_values: - payload["variables"] = variable_values + file_variables = get_file_variables(variable_values) + if file_variables: + map_ = { + file_variable: [f"variables.{file_variable}"] + for file_variable, value in file_variables.items() + } + all_variables = { + **variable_values, + **{ + file_variable: None + for file_variable in file_variables.values() + } + } + file_payload = { + "operations": + json.dumps( + {"query": query_str, "variables": all_variables} + ), + "map": json.dumps(map_) + } + else: + payload["variables"] = variable_values if operation_name: payload["operationName"] = operation_name From f6ffdd382074bad86347765c4fc70abf1f1823ab Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Fri, 17 Jul 2020 04:22:46 +0800 Subject: [PATCH 3/8] Implement file handling in aiohttp.py --- gql/transport/aiohttp.py | 29 +++++++++++++++--- gql/transport/requests.py | 46 ++++++++++++++-------------- gql/utils.py | 63 ++++++++++++++++++++++++++++++--------- 3 files changed, 97 insertions(+), 41 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index fa66e4db..e6da8ec3 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -1,5 +1,6 @@ from ssl import SSLContext from typing import Any, AsyncGenerator, Dict, Optional, Union +import json import aiohttp from aiohttp.client_exceptions import ClientResponseError @@ -15,6 +16,7 @@ TransportProtocolError, TransportServerError, ) +from ..utils import extract_files class AIOHTTPTransport(AsyncTransport): @@ -103,15 +105,34 @@ async def execute( """ query_str = print_ast(document) + + nulled_variable_values, files = extract_files(variable_values) + payload = { "query": query_str, - "variables": variable_values or {}, + "variables": nulled_variable_values or {}, "operationName": operation_name or "", } - post_args = { - "json": payload, - } + if files: + data = aiohttp.FormData() + operations_json = json.dumps(cls.prepare_json_data(query, variables, operation)) + + file_map = {str(i): [path] for i, path in enumerate(files)} # header + # path is nested in a list because the spec allows multiple pointers to the same file. + # But we don't use that. + file_streams = {str(i): files[path] for i, path in enumerate(files)} # payload + + data.add_field('operations', operations_json, content_type='application/json') + data.add_field('map', json.dumps(file_map), content_type='application/json') + data.add_fields(*file_streams.items()) + + post_args = { "data": payload } + + else: + post_args = { "json": payload } + + # Pass post_args to aiohttp post method post_args.update(extra_args) diff --git a/gql/transport/requests.py b/gql/transport/requests.py index bb3feb63..d38e2f5b 100644 --- a/gql/transport/requests.py +++ b/gql/transport/requests.py @@ -9,7 +9,7 @@ from gql.transport import Transport -from .utils import get_file_variables, is_file_like, is_file_like_list +# from ..utils import get_file_variables, is_file_like, is_file_like_list from .exceptions import ( TransportAlreadyConnected, TransportClosed, @@ -122,28 +122,28 @@ def execute( # type: ignore query_str = print_ast(document) payload: Dict[str, Any] = {"query": query_str} if variable_values: - file_variables = get_file_variables(variable_values) - if file_variables: - map_ = { - file_variable: [f"variables.{file_variable}"] - for file_variable, value in file_variables.items() - } - all_variables = { - **variable_values, - **{ - file_variable: None - for file_variable in file_variables.values() - } - } - file_payload = { - "operations": - json.dumps( - {"query": query_str, "variables": all_variables} - ), - "map": json.dumps(map_) - } - else: - payload["variables"] = variable_values + # file_variables = get_file_variables(variable_values) + # if file_variables: + # map_ = { + # file_variable: [f"variables.{file_variable}"] + # for file_variable, value in file_variables.items() + # } + # all_variables = { + # **variable_values, + # **{ + # file_variable: None + # for file_variable in file_variables.values() + # } + # } + # file_payload = { + # "operations": + # json.dumps( + # {"query": query_str, "variables": all_variables} + # ), + # "map": json.dumps(map_) + # } + # else: + payload["variables"] = variable_values if operation_name: payload["operationName"] = operation_name diff --git a/gql/utils.py b/gql/utils.py index 33318eeb..47203f0d 100644 --- a/gql/utils.py +++ b/gql/utils.py @@ -1,7 +1,7 @@ """Utilities to manipulate several python objects.""" import io -from typing import Dict, List, Any, Union +from typing import Dict, List, Any, Union, Tuple # From this response in Stackoverflow @@ -18,20 +18,55 @@ def is_file_like(value: Any) -> bool: return isinstance(value, io.IOBase) -def is_file_like_list(value: Any) -> bool: - """Check if value is a list and if all items in the list are file-like""" - return isinstance(value, list) and all(is_file_like(item) for item in value) +# def is_file_like_list(value: Any) -> bool: +# """Check if value is a list and if all items in the list are file-like""" +# return isinstance(value, list) and all(is_file_like(item) for item in value) -def contains_file_like_values(value: Any) -> bool: - return is_file_like(value) or is_file_like_list(value) +# def contains_file_like_values(value: Any) -> bool: +# return is_file_like(value) or is_file_like_list(value) -def get_file_variables( - variables: Dict[str, Any] -) -> Dict[str, Union[io.IOBase, List[io.IOBase]]]: - return { - variable: value - for variable, value in variables.items() - if contains_file_like_values(value) - } +# def get_file_variables(variables: Dict[str, Any]) -> Dict[str, Union[io.IOBase, List[io.IOBase]]]: +# return { +# variable: value +# for variable, value in variables.items() +# if contains_file_like_values(value) +# } + + + +def extract_files(variables: dict) -> Tuple[dict, dict, list]: + files = {} + + def recurse_extract(path, obj): + """ + recursively traverse obj, doing a deepcopy, but + replacing any file-like objects with nulls and + shunting the originals off to the side. + """ + nonlocal files + if type(obj) is list: + nulled_obj = [] + for key, value in enumerate(obj): + value = recurse_extract(f'{path}.{key}', value) + nulled_obj.append(value) + # TODO: merge this with dict case below. somehow. + return nulled_obj + elif type(obj) is dict: + nulled_obj = {} + for key, value in obj.items(): + value = recurse_extract(f'{path}.{key}', value) + nulled_obj[key] = value + return nulled_obj + elif is_file_like(obj): + # extract obj from its parent and put it into files instead. + files[path] = obj + return None + else: + # base case: pass through unchanged + return obj + + nulled_variables = recurse_extract('variables', variables) + + return nulled_variables, files \ No newline at end of file From e637983e3a408501a1aa16127c12602b13730668 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sat, 1 Aug 2020 05:44:41 +0800 Subject: [PATCH 4/8] Fix JSON serialization, remove comments, conform to double quotes --- gql/transport/aiohttp.py | 10 +++++----- gql/transport/requests.py | 21 --------------------- gql/utils.py | 24 +++--------------------- 3 files changed, 8 insertions(+), 47 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index e6da8ec3..70403f6f 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -110,21 +110,21 @@ async def execute( payload = { "query": query_str, - "variables": nulled_variable_values or {}, + "variables": json.dumps(nulled_variable_values) or "{}", "operationName": operation_name or "", } if files: data = aiohttp.FormData() - operations_json = json.dumps(cls.prepare_json_data(query, variables, operation)) + operations_json = json.dumps(payload) file_map = {str(i): [path] for i, path in enumerate(files)} # header # path is nested in a list because the spec allows multiple pointers to the same file. # But we don't use that. - file_streams = {str(i): files[path] for i, path in enumerate(files)} # payload + file_streams = {i: files[path] for i, path in enumerate(files)} # payload - data.add_field('operations', operations_json, content_type='application/json') - data.add_field('map', json.dumps(file_map), content_type='application/json') + data.add_field("operations", operations_json, content_type="application/json") + data.add_field("map", json.dumps(file_map), content_type="application/json") data.add_fields(*file_streams.items()) post_args = { "data": payload } diff --git a/gql/transport/requests.py b/gql/transport/requests.py index d38e2f5b..59c7ac48 100644 --- a/gql/transport/requests.py +++ b/gql/transport/requests.py @@ -122,27 +122,6 @@ def execute( # type: ignore query_str = print_ast(document) payload: Dict[str, Any] = {"query": query_str} if variable_values: - # file_variables = get_file_variables(variable_values) - # if file_variables: - # map_ = { - # file_variable: [f"variables.{file_variable}"] - # for file_variable, value in file_variables.items() - # } - # all_variables = { - # **variable_values, - # **{ - # file_variable: None - # for file_variable in file_variables.values() - # } - # } - # file_payload = { - # "operations": - # json.dumps( - # {"query": query_str, "variables": all_variables} - # ), - # "map": json.dumps(map_) - # } - # else: payload["variables"] = variable_values if operation_name: payload["operationName"] = operation_name diff --git a/gql/utils.py b/gql/utils.py index 47203f0d..6e6decf5 100644 --- a/gql/utils.py +++ b/gql/utils.py @@ -18,24 +18,6 @@ def is_file_like(value: Any) -> bool: return isinstance(value, io.IOBase) -# def is_file_like_list(value: Any) -> bool: -# """Check if value is a list and if all items in the list are file-like""" -# return isinstance(value, list) and all(is_file_like(item) for item in value) - - -# def contains_file_like_values(value: Any) -> bool: -# return is_file_like(value) or is_file_like_list(value) - - -# def get_file_variables(variables: Dict[str, Any]) -> Dict[str, Union[io.IOBase, List[io.IOBase]]]: -# return { -# variable: value -# for variable, value in variables.items() -# if contains_file_like_values(value) -# } - - - def extract_files(variables: dict) -> Tuple[dict, dict, list]: files = {} @@ -49,14 +31,14 @@ def recurse_extract(path, obj): if type(obj) is list: nulled_obj = [] for key, value in enumerate(obj): - value = recurse_extract(f'{path}.{key}', value) + value = recurse_extract(f"{path}.{key}", value) nulled_obj.append(value) # TODO: merge this with dict case below. somehow. return nulled_obj elif type(obj) is dict: nulled_obj = {} for key, value in obj.items(): - value = recurse_extract(f'{path}.{key}', value) + value = recurse_extract(f"{path}.{key}", value) nulled_obj[key] = value return nulled_obj elif is_file_like(obj): @@ -67,6 +49,6 @@ def recurse_extract(path, obj): # base case: pass through unchanged return obj - nulled_variables = recurse_extract('variables', variables) + nulled_variables = recurse_extract("variables", variables) return nulled_variables, files \ No newline at end of file From 9e86b03d9b4d152e40b7181ae3e316213d5f6d34 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sat, 1 Aug 2020 06:33:37 +0800 Subject: [PATCH 5/8] Fix more JSON serialization --- gql/transport/aiohttp.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index 70403f6f..de8eb19d 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -110,24 +110,23 @@ async def execute( payload = { "query": query_str, - "variables": json.dumps(nulled_variable_values) or "{}", + "variables": nulled_variable_values or {}, "operationName": operation_name or "", } if files: data = aiohttp.FormData() - operations_json = json.dumps(payload) file_map = {str(i): [path] for i, path in enumerate(files)} # header # path is nested in a list because the spec allows multiple pointers to the same file. # But we don't use that. - file_streams = {i: files[path] for i, path in enumerate(files)} # payload + file_streams = {str(i): files[path] for i, path in enumerate(files)} # payload - data.add_field("operations", operations_json, content_type="application/json") + data.add_field("operations", json.dumps(payload), content_type="application/json") data.add_field("map", json.dumps(file_map), content_type="application/json") data.add_fields(*file_streams.items()) - post_args = { "data": payload } + post_args = { "data": data } else: post_args = { "json": payload } From c2e38dc1ad18a95805a0e3d3fa2e646d3c195798 Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sat, 1 Aug 2020 06:58:02 +0800 Subject: [PATCH 6/8] Cleanup --- gql/transport/aiohttp.py | 2 +- gql/transport/requests.py | 2 -- gql/utils.py | 11 +++++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index de8eb19d..374ad358 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -1,6 +1,6 @@ +import json from ssl import SSLContext from typing import Any, AsyncGenerator, Dict, Optional, Union -import json import aiohttp from aiohttp.client_exceptions import ClientResponseError diff --git a/gql/transport/requests.py b/gql/transport/requests.py index 59c7ac48..8eb4b2f8 100644 --- a/gql/transport/requests.py +++ b/gql/transport/requests.py @@ -1,5 +1,4 @@ from typing import Any, Dict, Optional, Union -import json import requests from graphql import DocumentNode, ExecutionResult, print_ast @@ -9,7 +8,6 @@ from gql.transport import Transport -# from ..utils import get_file_variables, is_file_like, is_file_like_list from .exceptions import ( TransportAlreadyConnected, TransportClosed, diff --git a/gql/utils.py b/gql/utils.py index 6e6decf5..591b92a2 100644 --- a/gql/utils.py +++ b/gql/utils.py @@ -1,7 +1,7 @@ """Utilities to manipulate several python objects.""" import io -from typing import Dict, List, Any, Union, Tuple +from typing import Dict, Any, Tuple # From this response in Stackoverflow @@ -18,7 +18,7 @@ def is_file_like(value: Any) -> bool: return isinstance(value, io.IOBase) -def extract_files(variables: dict) -> Tuple[dict, dict, list]: +def extract_files(variables: Dict) -> Tuple[Dict, Dict]: files = {} def recurse_extract(path, obj): @@ -28,14 +28,13 @@ def recurse_extract(path, obj): shunting the originals off to the side. """ nonlocal files - if type(obj) is list: + if isinstance(obj, list): nulled_obj = [] for key, value in enumerate(obj): value = recurse_extract(f"{path}.{key}", value) nulled_obj.append(value) - # TODO: merge this with dict case below. somehow. return nulled_obj - elif type(obj) is dict: + elif isinstance(obj, dict): nulled_obj = {} for key, value in obj.items(): value = recurse_extract(f"{path}.{key}", value) @@ -51,4 +50,4 @@ def recurse_extract(path, obj): nulled_variables = recurse_extract("variables", variables) - return nulled_variables, files \ No newline at end of file + return nulled_variables, files From 8839141d59b0666935ee75658b13ba0520a42fea Mon Sep 17 00:00:00 2001 From: Michael Liu Date: Sat, 1 Aug 2020 07:34:13 +0800 Subject: [PATCH 7/8] Blackened --- gql/transport/aiohttp.py | 2 +- gql/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index f8fee87b..ca394cd2 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -9,6 +9,7 @@ from aiohttp.typedefs import LooseCookies, LooseHeaders from graphql import DocumentNode, ExecutionResult, print_ast +from ..utils import extract_files from .async_transport import AsyncTransport from .exceptions import ( TransportAlreadyConnected, @@ -16,7 +17,6 @@ TransportProtocolError, TransportServerError, ) -from ..utils import extract_files class AIOHTTPTransport(AsyncTransport): diff --git a/gql/utils.py b/gql/utils.py index 591b92a2..ce0318b0 100644 --- a/gql/utils.py +++ b/gql/utils.py @@ -1,7 +1,7 @@ """Utilities to manipulate several python objects.""" import io -from typing import Dict, Any, Tuple +from typing import Any, Dict, Tuple # From this response in Stackoverflow From 194d6457c03bf68a5309b9361467250ef008a969 Mon Sep 17 00:00:00 2001 From: Manuel Bojato <30560560+KingDarBoja@users.noreply.github.com> Date: Sat, 19 Sep 2020 12:07:58 -0500 Subject: [PATCH 8/8] fix: safe check if parameters are none on aiohttp --- gql/transport/aiohttp.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/gql/transport/aiohttp.py b/gql/transport/aiohttp.py index ca394cd2..7d71f3a9 100644 --- a/gql/transport/aiohttp.py +++ b/gql/transport/aiohttp.py @@ -35,7 +35,7 @@ def __init__( auth: Optional[BasicAuth] = None, ssl: Union[SSLContext, bool, Fingerprint] = False, timeout: Optional[int] = None, - client_session_args: Dict[str, Any] = {}, + client_session_args: Optional[Dict[str, Any]] = None, ) -> None: """Initialize the transport with the given aiohttp parameters. @@ -53,7 +53,6 @@ def __init__( self.ssl: Union[SSLContext, bool, Fingerprint] = ssl self.timeout: Optional[int] = timeout self.client_session_args = client_session_args - self.session: Optional[aiohttp.ClientSession] = None async def connect(self) -> None: @@ -78,7 +77,8 @@ async def connect(self) -> None: ) # Adding custom parameters passed from init - client_session_args.update(self.client_session_args) + if self.client_session_args: + client_session_args.update(self.client_session_args) # type: ignore self.session = aiohttp.ClientSession(**client_session_args) @@ -95,7 +95,7 @@ async def execute( document: DocumentNode, variable_values: Optional[Dict[str, str]] = None, operation_name: Optional[str] = None, - extra_args: Dict[str, Any] = {}, + extra_args: Dict[str, Any] = None, ) -> ExecutionResult: """Execute the provided document AST against the configured remote server. This uses the aiohttp library to perform a HTTP POST request asynchronously @@ -106,7 +106,10 @@ async def execute( query_str = print_ast(document) - nulled_variable_values, files = extract_files(variable_values) + nulled_variable_values = None + files = None + if variable_values: + nulled_variable_values, files = extract_files(variable_values) payload: Dict[str, Any] = { "query": query_str, @@ -120,22 +123,28 @@ async def execute( if files: data = aiohttp.FormData() - file_map = {str(i): [path] for i, path in enumerate(files)} # header - # path is nested in a list because the spec allows multiple pointers to the same file. - # But we don't use that. - file_streams = {str(i): files[path] for i, path in enumerate(files)} # payload - - data.add_field("operations", json.dumps(payload), content_type="application/json") + # header + file_map = {str(i): [path] for i, path in enumerate(files)} + # path is nested in a list because the spec allows multiple pointers + # to the same file. But we don't use that. + file_streams = { + str(i): files[path] for i, path in enumerate(files) + } # payload + + data.add_field( + "operations", json.dumps(payload), content_type="application/json" + ) data.add_field("map", json.dumps(file_map), content_type="application/json") data.add_fields(*file_streams.items()) - post_args = { "data": data } + post_args = {"data": data} else: - post_args = { "json": payload } + post_args = {"json": payload} # type: ignore # Pass post_args to aiohttp post method - post_args.update(extra_args) + if extra_args: + post_args.update(extra_args) # type: ignore if self.session is None: raise TransportClosed("Transport is not connected")