let fix #3016 as well

Author: Wauplin

src/huggingface_hub/_upload_large_folder.py

@@ -43,8 +43,6 @@
 
 WAITING_TIME_IF_NO_TASKS = 10  # seconds
 MAX_NB_FILES_FETCH_UPLOAD_MODE = 100
-MAX_NB_REGULAR_FILES_PER_COMMIT = 75
-MAX_NB_LFS_FILES_PER_COMMIT = 150
 COMMIT_SIZE_SCALE: List[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000]
 
 
@@ -405,13 +403,13 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
         ):
             status.nb_workers_commit += 1
             logger.debug("Job: commit (more than 5 minutes since last commit attempt)")
-            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
+            return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
 
         # 2. Commit if at least 100 files are ready to commit
         elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 150:
             status.nb_workers_commit += 1
             logger.debug("Job: commit (>100 files ready)")
-            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
+            return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
 
         # 3. Get upload mode if at least 100 files
         elif status.queue_get_upload_mode.qsize() >= MAX_NB_FILES_FETCH_UPLOAD_MODE:
@@ -467,7 +465,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
         ):
             status.nb_workers_commit += 1
             logger.debug("Job: commit (1 min since last commit attempt)")
-            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
+            return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
 
         # 11. Commit if at least 1 file all other queues are empty and all workers are waiting
         #     e.g. when it's the last commit
@@ -483,7 +481,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
         ):
             status.nb_workers_commit += 1
             logger.debug("Job: commit")
-            return (WorkerJob.COMMIT, _get_items_to_commit(status.queue_commit))
+            return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
 
         # 12. If all queues are empty, exit
         elif all(metadata.is_committed or metadata.should_ignore for _, metadata in status.items):
@@ -601,30 +599,6 @@ def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> List[JOB_ITEM_T]:
     return [queue.get() for _ in range(min(queue.qsize(), n))]
 
 
-def _get_items_to_commit(queue: "queue.Queue[JOB_ITEM_T]") -> List[JOB_ITEM_T]:
-    """Special case for commit job: the number of items to commit depends on the type of files."""
-    # Can take at most 50 regular files and/or 100 LFS files in a single commit
-    items: List[JOB_ITEM_T] = []
-    nb_lfs, nb_regular = 0, 0
-    while True:
-        # If empty queue => commit everything
-        if queue.qsize() == 0:
-            return items
-
-        # If we have enough items => commit them
-        if nb_lfs >= MAX_NB_LFS_FILES_PER_COMMIT or nb_regular >= MAX_NB_REGULAR_FILES_PER_COMMIT:
-            return items
-
-        # Else, get a new item and increase counter
-        item = queue.get()
-        items.append(item)
-        _, metadata = item
-        if metadata.upload_mode == "lfs":
-            nb_lfs += 1
-        else:
-            nb_regular += 1
-
-
 def _print_overwrite(report: str) -> None:
     """Print a report, overwriting the previous lines.