44import platform
55import sys
66import requests
7+ from concurrent .futures import ThreadPoolExecutor , as_completed
8+ from threading import Lock
79from .input_file import InputFile
810from .exception import AppwriteException
911from .encoders .value_class_encoder import ValueClassEncoder
@@ -15,11 +17,11 @@ def __init__(self):
1517 self ._endpoint = 'https://cloud.appwrite.io/v1'
1618 self ._global_headers = {
1719 'content-type' : '' ,
18- 'user-agent' : f'AppwritePythonSDK/19.1 .0 ({ platform .uname ().system } ; { platform .uname ().version } ; { platform .uname ().machine } )' ,
20+ 'user-agent' : f'AppwritePythonSDK/20.0 .0 ({ platform .uname ().system } ; { platform .uname ().version } ; { platform .uname ().machine } )' ,
1921 'x-sdk-name' : 'Python' ,
2022 'x-sdk-platform' : 'server' ,
2123 'x-sdk-language' : 'python' ,
22- 'x-sdk-version' : '19.1 .0' ,
24+ 'x-sdk-version' : '20.0 .0' ,
2325 'X-Appwrite-Response-Format' : '1.9.5' ,
2426 }
2527
@@ -186,14 +188,15 @@ def chunked_upload(
186188
187189 if input_file .source_type == 'path' :
188190 size = os .stat (input_file .path ).st_size
189- input = open ( input_file . path , 'rb' )
191+ input = None
190192 elif input_file .source_type == 'bytes' :
191193 size = len (input_file .data )
192194 input = input_file .data
193195
194196 if size < self ._chunk_size :
195197 if input_file .source_type == 'path' :
196- input_file .data = input .read ()
198+ with open (input_file .path , 'rb' ) as input :
199+ input_file .data = input .read ()
197200
198201 params [param_name ] = input_file
199202 return self .call (
@@ -214,46 +217,103 @@ def chunked_upload(
214217
215218 if counter > 0 :
216219 offset = counter * self ._chunk_size
217- input .seek (offset )
218220
221+ total_chunks = (size + self ._chunk_size - 1 ) // self ._chunk_size
222+ chunks = []
219223 while offset < size :
220- if input_file . source_type == 'path' :
221- input_file . data = input . read ( self . _chunk_size ) or input . read ( size - offset )
222- elif input_file . source_type == 'bytes' :
223- if offset + self . _chunk_size < size :
224- end = offset + self . _chunk_size
225- else :
226- end = size
227- input_file . data = input [ offset : end ]
224+ end = min ( offset + self . _chunk_size , size )
225+ chunks . append ({
226+ 'index' : counter ,
227+ 'start' : offset ,
228+ ' end' : end ,
229+ })
230+ offset = end
231+ counter = counter + 1
228232
229- params [param_name ] = input_file
230- headers ["content-range" ] = f'bytes { offset } -{ min ((offset + self ._chunk_size ) - 1 , size - 1 )} /{ size } '
233+ if not chunks :
234+ return result
235+
236+ def read_chunk (start , end ):
237+ if input_file .source_type == 'path' :
238+ with open (input_file .path , 'rb' ) as chunk_file :
239+ chunk_file .seek (start )
240+ return chunk_file .read (end - start )
241+ return input [start :end ]
242+
243+ upload_id_header = upload_id
244+ completed_count = chunks [0 ]['index' ]
245+ uploaded_size = chunks [0 ]['start' ]
246+ progress_lock = Lock ()
247+ last_result = None
248+ final_result = None
249+
250+ def is_upload_complete (chunk_result ):
251+ chunks_uploaded = chunk_result .get ('chunksUploaded' )
252+ if chunks_uploaded is None :
253+ return False
254+ chunks_total = chunk_result .get ('chunksTotal' , total_chunks )
255+ return int (chunks_uploaded ) >= int (chunks_total )
256+
257+ def upload_chunk (chunk , current_upload_id ):
258+ chunk_input = InputFile .from_bytes (
259+ read_chunk (chunk ['start' ], chunk ['end' ]),
260+ input_file .filename ,
261+ getattr (input_file , 'mime_type' , None )
262+ )
263+ chunk_params = {** params , param_name : chunk_input }
264+ chunk_headers = {** headers }
265+ chunk_headers ["content-range" ] = f"bytes { chunk ['start' ]} -{ chunk ['end' ] - 1 } /{ size } "
266+ if current_upload_id :
267+ chunk_headers ["x-appwrite-id" ] = current_upload_id
231268
232- result = self .call (
269+ return self .call (
233270 'post' ,
234271 path ,
235- headers ,
236- params ,
272+ chunk_headers ,
273+ chunk_params ,
237274 )
238275
239- offset = offset + self ._chunk_size
240-
241- if "$id" in result :
242- headers ["x-appwrite-id" ] = result ["$id" ]
243-
244- if on_progress is not None :
245- end = min ((((counter * self ._chunk_size ) + self ._chunk_size ) - 1 ), size - 1 )
246- on_progress ({
247- "$id" : result ["$id" ],
248- "progress" : min (offset , size )/ size * 100 ,
249- "sizeUploaded" : end + 1 ,
250- "chunksTotal" : result ["chunksTotal" ],
251- "chunksUploaded" : result ["chunksUploaded" ],
252- })
253-
254- counter = counter + 1
255-
256- return result
276+ result = upload_chunk (chunks [0 ], upload_id_header )
277+ last_result = result
278+ if "$id" in result :
279+ upload_id_header = result ["$id" ]
280+
281+ completed_count = chunks [0 ]['index' ] + 1
282+ uploaded_size = chunks [0 ]['end' ]
283+
284+ if on_progress is not None :
285+ on_progress ({
286+ "$id" : result .get ("$id" ),
287+ "progress" : uploaded_size / size * 100 ,
288+ "sizeUploaded" : uploaded_size ,
289+ "chunksTotal" : total_chunks ,
290+ "chunksUploaded" : completed_count ,
291+ })
292+
293+ def upload_remaining_chunk (chunk ):
294+ nonlocal completed_count , uploaded_size , last_result , final_result
295+ chunk_result = upload_chunk (chunk , upload_id_header )
296+ with progress_lock :
297+ completed_count = completed_count + 1
298+ uploaded_size = uploaded_size + (chunk ['end' ] - chunk ['start' ])
299+ last_result = chunk_result
300+ if is_upload_complete (chunk_result ):
301+ final_result = chunk_result
302+ if on_progress is not None :
303+ on_progress ({
304+ "$id" : upload_id_header ,
305+ "progress" : uploaded_size / size * 100 ,
306+ "sizeUploaded" : uploaded_size ,
307+ "chunksTotal" : total_chunks ,
308+ "chunksUploaded" : completed_count ,
309+ })
310+
311+ with ThreadPoolExecutor (max_workers = 8 ) as executor :
312+ futures = [executor .submit (upload_remaining_chunk , chunk ) for chunk in chunks [1 :]]
313+ for future in as_completed (futures ):
314+ future .result ()
315+
316+ return final_result or last_result
257317
258318 def flatten (self , data , prefix = '' , stringify = False ):
259319 output = {}
0 commit comments