diff --git a/docs/02_concepts/01_async_support.mdx b/docs/02_concepts/01_async_support.mdx index ddc06f9b..a210fe3f 100644 --- a/docs/02_concepts/01_async_support.mdx +++ b/docs/02_concepts/01_async_support.mdx @@ -7,6 +7,7 @@ description: Use the async client for non-blocking API calls with Python asyncio import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import ApiLink from '@site/src/components/ApiLink'; import AsyncSupportExample from '!!raw-loader!./code/01_async_support.py'; @@ -17,3 +18,5 @@ The following example demonstrates how to run an Actor asynchronously and stream {AsyncSupportExample} + +For the full async client API, see the `ApifyClientAsync` reference. diff --git a/docs/02_concepts/04_error_handling.mdx b/docs/02_concepts/04_error_handling.mdx index 16a96fae..b267e304 100644 --- a/docs/02_concepts/04_error_handling.mdx +++ b/docs/02_concepts/04_error_handling.mdx @@ -7,6 +7,7 @@ description: Handle API errors with the ApifyApiError exception and automatic da import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +import ApiLink from '@site/src/components/ApiLink'; import ErrorAsyncExample from '!!raw-loader!./code/04_error_async.py'; import ErrorSyncExample from '!!raw-loader!./code/04_error_sync.py'; @@ -25,3 +26,5 @@ When you use the Apify client, it automatically extracts all relevant data from + +For a complete list of error classes, see the `ApifyApiError` reference. diff --git a/docs/02_concepts/08_pagination.mdx b/docs/02_concepts/08_pagination.mdx index c79638f2..a7fe1d82 100644 --- a/docs/02_concepts/08_pagination.mdx +++ b/docs/02_concepts/08_pagination.mdx @@ -11,6 +11,9 @@ import CodeBlock from '@theme/CodeBlock'; import PaginationAsyncExample from '!!raw-loader!./code/08_pagination_async.py'; import PaginationSyncExample from '!!raw-loader!./code/08_pagination_sync.py'; +import IterateItemsAsyncExample from '!!raw-loader!./code/08_iterate_items_async.py'; +import IterateItemsSyncExample from '!!raw-loader!./code/08_iterate_items_sync.py'; + Most methods named `list` or `list_something` in the Apify client return a [`ListPage`](/reference/class/ListPage) object. This object provides a consistent interface for working with paginated data and includes the following properties: - `items` - The main results you're looking for. @@ -37,3 +40,24 @@ The following example demonstrates how to fetch all items from a dataset using p The [`ListPage`](/reference/class/ListPage) interface offers several key benefits. Its consistent structure ensures predictable results for most `list` methods, providing a uniform way to work with paginated data. It also offers flexibility, allowing you to customize the `limit` and `offset` parameters to control data fetching according to your needs. Additionally, it provides scalability, enabling you to efficiently handle large datasets through pagination. This approach ensures efficient data retrieval while keeping memory usage under control, making it ideal for managing and processing large collections. + +## Generator-based iteration + +For most use cases, `iterate_items()` is the recommended way to process all items in a dataset. It handles pagination automatically using a Python generator, fetching items in batches behind the scenes so you don't need to manage offsets or limits yourself. + + + + + {IterateItemsAsyncExample} + + + + + {IterateItemsSyncExample} + + + + +`iterate_items()` accepts the same filtering parameters as `list_items()` (`clean`, `fields`, `omit`, `unwind`, `skip_empty`, `skip_hidden`), so you can combine automatic pagination with data filtering. + +Similarly, `KeyValueStoreClient` provides an `iterate_keys()` method for iterating over all keys in a key-value store without manual pagination. diff --git a/docs/02_concepts/11_timeouts.mdx b/docs/02_concepts/11_timeouts.mdx new file mode 100644 index 00000000..2d40bc63 --- /dev/null +++ b/docs/02_concepts/11_timeouts.mdx @@ -0,0 +1,62 @@ +--- +id: timeouts +title: Timeouts +description: Configure the tiered timeout system for controlling how long API requests can take. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import ApiLink from '@site/src/components/ApiLink'; + +import TimeoutsAsyncExample from '!!raw-loader!./code/11_timeouts_async.py'; +import TimeoutsSyncExample from '!!raw-loader!./code/11_timeouts_sync.py'; + +The Apify client uses a tiered timeout system to set appropriate time limits for different types of API requests. Each tier has a default value suited to its use case: + +| Tier | Default | Purpose | +|---|---|---| +| `short` | 5 seconds | Fast CRUD operations (get, update, delete) | +| `medium` | 30 seconds | Batch, list, and data transfer operations | +| `long` | 360 seconds | Long-polling, streaming, and heavy operations | +| `no_timeout` | — | Disables the timeout entirely | + +Every client method has a pre-assigned tier that matches the expected duration of the underlying API call. You generally don't need to change these unless you're working with unusually large payloads or slow network conditions. + +## Configuring default timeouts + +You can override the default values for each tier in the `ApifyClient` or `ApifyClientAsync` constructor. The `timeout_max` parameter sets an upper cap on the timeout for any individual API request, limiting exponential growth during retries. + + + + + {TimeoutsAsyncExample} + + + + + {TimeoutsSyncExample} + + + + +## Per-call overrides + +Most client methods accept a `timeout` parameter that overrides the default tier for that specific call. You can pass either a `timedelta` for an exact duration or a tier literal (`'short'`, `'medium'`, `'long'`, `'no_timeout'`) to switch tiers. + +```python +from datetime import timedelta + +# Use an exact timeout for this call. +client.dataset('id').list_items(timeout=timedelta(seconds=120)) + +# Switch to a different tier. +client.dataset('id').list_items(timeout='long') + +# Disable the timeout entirely. +client.dataset('id').list_items(timeout='no_timeout') +``` + +## Interaction with retries + +Timeouts work together with the [retry system](/api/client/python/docs/concepts/retries). When a request times out, it counts as a failed attempt and triggers a retry (up to `max_retries`). The timeout applies to each individual attempt, not the total time across all retries. diff --git a/docs/02_concepts/code/08_iterate_items_async.py b/docs/02_concepts/code/08_iterate_items_async.py new file mode 100644 index 00000000..fba9b5b0 --- /dev/null +++ b/docs/02_concepts/code/08_iterate_items_async.py @@ -0,0 +1,12 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + dataset_client = apify_client.dataset('dataset-id') + + # Iterate through all items automatically. + async for item in dataset_client.iterate_items(): + print(item) diff --git a/docs/02_concepts/code/08_iterate_items_sync.py b/docs/02_concepts/code/08_iterate_items_sync.py new file mode 100644 index 00000000..005c899f --- /dev/null +++ b/docs/02_concepts/code/08_iterate_items_sync.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + dataset_client = apify_client.dataset('dataset-id') + + # Iterate through all items automatically. + for item in dataset_client.iterate_items(): + print(item) + + +if __name__ == '__main__': + main() diff --git a/docs/02_concepts/code/11_timeouts_async.py b/docs/02_concepts/code/11_timeouts_async.py new file mode 100644 index 00000000..c402a44b --- /dev/null +++ b/docs/02_concepts/code/11_timeouts_async.py @@ -0,0 +1,24 @@ +from datetime import timedelta + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + # Configure default timeout tiers globally. + apify_client = ApifyClientAsync( + token=TOKEN, + timeout_short=timedelta(seconds=10), + timeout_medium=timedelta(seconds=60), + timeout_long=timedelta(seconds=600), + timeout_max=timedelta(seconds=600), + ) + + dataset_client = apify_client.dataset('dataset-id') + + # Override the timeout for a single call using a timedelta. + items = await dataset_client.list_items(timeout=timedelta(seconds=120)) + + # Or use a tier literal to select a predefined timeout. + items = await dataset_client.list_items(timeout='long') diff --git a/docs/02_concepts/code/11_timeouts_sync.py b/docs/02_concepts/code/11_timeouts_sync.py new file mode 100644 index 00000000..fc027706 --- /dev/null +++ b/docs/02_concepts/code/11_timeouts_sync.py @@ -0,0 +1,24 @@ +from datetime import timedelta + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + # Configure default timeout tiers globally. + apify_client = ApifyClient( + token=TOKEN, + timeout_short=timedelta(seconds=10), + timeout_medium=timedelta(seconds=60), + timeout_long=timedelta(seconds=600), + timeout_max=timedelta(seconds=600), + ) + + dataset_client = apify_client.dataset('dataset-id') + + # Override the timeout for a single call using a timedelta. + items = dataset_client.list_items(timeout=timedelta(seconds=120)) + + # Or use a tier literal to select a predefined timeout. + items = dataset_client.list_items(timeout='long')