Upload folder using huggingface_hub
Browse files
modeling_open_provence_standalone.py
CHANGED
|
@@ -3355,6 +3355,15 @@ class OpenProvenceModel(OpenProvencePreTrainedModel):
|
|
| 3355 |
torch_dataloader_kwargs: Optional mapping forwarded directly to the preprocessing
|
| 3356 |
``DataLoader`` to fine-tune worker behaviour (e.g., setting a custom
|
| 3357 |
``worker_init_fn`` or pinning strategy).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3358 |
"""
|
| 3359 |
|
| 3360 |
progress_restore: Callable[[], None] | None = None
|
|
|
|
| 3355 |
torch_dataloader_kwargs: Optional mapping forwarded directly to the preprocessing
|
| 3356 |
``DataLoader`` to fine-tune worker behaviour (e.g., setting a custom
|
| 3357 |
``worker_init_fn`` or pinning strategy).
|
| 3358 |
+
|
| 3359 |
+
.. caution::
|
| 3360 |
+
Input shape determines how batching behaves. Passing ``question: str`` with
|
| 3361 |
+
``context: List[str]`` is interpreted as *one* query paired with multiple
|
| 3362 |
+
documents. To batch distinct question–context pairs, provide
|
| 3363 |
+
``question: List[str]`` and ``context: List[str]`` of equal length. If you
|
| 3364 |
+
supply ``context: List[List[str]]`` the inner lists are assumed to be
|
| 3365 |
+
pre-split sentences and the sentence splitter is skipped—use this form only
|
| 3366 |
+
when you have already segmented the text yourself.
|
| 3367 |
"""
|
| 3368 |
|
| 3369 |
progress_restore: Callable[[], None] | None = None
|