pyproject.toml

[build-system]
requires = ["uv_build"]
build-backend = "uv_build"

[project]
name = "crawlee"
version = "1.11.2"
description = "Crawlee for Python"
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
license = "Apache-2.0"
license-files = ["LICENSE"]
readme = "README.md"
requires-python = ">=3.10"
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Programming Language :: Python :: 3.15",
    "Topic :: Software Development :: Libraries",
]
keywords = [
    "apify",
    "automation",
    "chrome",
    "crawlee",
    "crawler",
    "headless",
    "scraper",
    "scraping",
]
dependencies = [
    "async-timeout>=5.0.1",
    "cachetools>=5.5.0",
    "colorama>=0.4.0",
    "impit>=0.13.2",
    "more-itertools>=10.2.0",
    "proclimits>=0.2.0",
    "protego>=0.5.0",
    "psutil>=6.0.0",
    "pydantic-settings>=2.12.0",
    "pydantic>=2.11.0",
    "typing-extensions>=4.10.0",
    "yarl>=1.18.0",
]

[project.optional-dependencies]
all = ["crawlee[adaptive-crawler,pydantic-ai,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres,sql_mysql,stagehand,redis]"]
adaptive-crawler = [
    "crawlee[beautifulsoup,parsel]",
    "playwright>=1.27.0",
    "rapidfuzz>=3.0.0",
    "scikit-learn>=1.6.0",
    "apify_fingerprint_datapoints>=0.0.3",
    "browserforge>=1.2.4"
]
pydantic-ai = ["pydantic-ai-slim[openai]>=2.1.0", "parsel>=1.10.0", "lxml[html_clean]>=5.2.0"]
beautifulsoup = ["beautifulsoup4[lxml]>=4.12.0", "html5lib>=1.0"]
cli = [
    "cookiecutter>=2.6.0",
    "inquirer>=3.3.0",
    "rich>=13.9.0",
    "typer>=0.26.0",
]
curl-impersonate = ["curl-cffi>=0.9.0"]
httpx = ["httpx2[brotli,http2,zstd]>=2.12.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"]
parsel = ["parsel>=1.10.0"]
playwright = ["playwright>=1.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"]
otel = [
    "opentelemetry-api>=1.34.1",
    "opentelemetry-distro[otlp]>=0.54b0",
    "opentelemetry-instrumentation>=0.54b0",
    "opentelemetry-sdk>=1.34.1",
    "opentelemetry-semantic-conventions>=0.54b0",
    "wrapt>=1.17.0",
]
sql_postgres = [
    "sqlalchemy[asyncio]>=2.0.0",
    "asyncpg>=0.24.0"
]
stagehand = [
    "stagehand>=3.19.5,<4.0.0",
    "playwright>=1.27.0",
    "apify_fingerprint_datapoints>=0.0.2",
    "browserforge>=1.2.3",
]
sql_sqlite = [
    "sqlalchemy[asyncio]>=2.0.0",
    "aiosqlite>=0.21.0",
]
sql_mysql = [
    "sqlalchemy[asyncio]>=2.0.0",
    "aiomysql>=0.3.2",
    "cryptography>=46.0.5",
]
redis = ["redis[hiredis] >= 7.0.0"]

[project.scripts]
crawlee = "crawlee._cli:cli"

[project.urls]
"Apify Homepage" = "https://apify.com"
"Changelog" = "https://crawlee.dev/python/docs/changelog"
"Discord" = "https://discord.com/invite/jyEM2PRvMU"
"Documentation" = "https://crawlee.dev/python/docs/quick-start"
"Homepage" = "https://crawlee.dev/python"
"Issue Tracker" = "https://github.com/apify/crawlee-python/issues"
"Release Notes" = "https://crawlee.dev/python/docs/upgrading"
"Source Code" = "https://github.com/apify/crawlee-python"

[dependency-groups]
dev = [
    # TODO: Remove this constraint once pydoc-markdown updates its dependencies.
    # Package pydoc-markdown is unmaintained and pins old docspec-python with vulnerable black.
    # See https://github.com/apify/apify-client-python/pull/582/ for more details.
    # We explicitly constrain black>=24.3.0 to override the transitive dependency.
    "black>=24.3.0",
    "anyio<5.0.0",
    # Pin below 3.0, which drops Python 3.10 support and introduces a breaking API. Once Crawlee
    # drops Python 3.10 (https://github.com/apify/crawlee-python/issues/1721), remove this constraint.
    "apify_client<3.0.0", # For e2e tests.
    "build<2.0.0", # For e2e tests.
    "fakeredis[probabilistic,json,lua]<3.0.0; python_version != '3.15' or sys_platform != 'win32'",
    "fakeredis[probabilistic,json]<3.0.0; python_version == '3.15' and sys_platform == 'win32'",
    "filelock<5.0.0", # For e2e tests.
    "poethepoet<1.0.0",
    "pre-commit<5.0.0",
    "proxy-py<3.0.0",
    "pydoc-markdown<5.0.0",
    # pytest-asyncio 1.1+ runs tests through `backports.asyncio.runner` on Python 3.10, whose process-wide
    # `contextvars.copy_context` patch can leak the fixture context into the test server thread and hang the test.
    "pytest-asyncio<2.0.0; python_version >= '3.11'",
    "pytest-asyncio>=1.0.0,<1.1.0; python_version < '3.11'",
    "pytest-cov<8.0.0",
    "pytest-rerunfailures<17.0.0",
    "pytest-timeout<3.0.0",
    "pytest-xdist<4.0.0",
    "pytest<10.0.0",
    "ruff~=0.17.0",
    "setuptools", # setuptools are used by pytest, but not explicitly required
    "ty~=0.0.0",
    "types-beautifulsoup4<5.0.0",
    "types-cachetools<8.0.0",
    "types-colorama<1.0.0",
    "types-psutil<8.0.0",
    "types-python-dateutil<3.0.0",
    "uvicorn[standard]<1.0.0",
]

[tool.uv.build-backend]
# The module, pyproject.toml, README, and LICENSE are included in the sdist by default.
source-include = ["CHANGELOG.md", "CONTRIBUTING.md"]

[tool.ruff]
line-length = 120
include = [
    "**/*.py",
    # Ruff formats Python code blocks embedded in Markdown files.
    "**/*.md",
    "**/*.mdx",
]
exclude = [
    "website/versioned_docs/**",
]
extend-exclude = ["src/crawlee/project_template"]
# MDX is Markdown; without this mapping Ruff would try to parse `.mdx` files as Python.
extension = { mdx = "markdown" }

[tool.ruff.lint]
select = ["ALL"]
ignore = [
    "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {filename}
    "ASYNC109", # Async function definition with a `timeout` parameter
    "BLE001",   # Do not catch blind exception
    "C901",     # `{name}` is too complex
    "COM812",   # This rule may cause conflicts when used with the formatter
    "CPY001",   # Missing copyright notice at top of file
    "D100",     # Missing docstring in public module
    "D104",     # Missing docstring in public package
    "D107",     # Missing docstring in `__init__`
    "D203",     # One blank line required before class docstring
    "D213",     # Multi-line docstring summary should start at the second line
    "D413",     # Missing blank line after last section
    "EM",       # flake8-errmsg
    "G004",     # Logging statement uses f-string
    "ISC001",   # This rule may cause conflicts when used with the formatter
    "FIX",      # flake8-fixme
    "PLR0911",  # Too many return statements
    "PLR0912",  # Too many branches
    "PLR0913",  # Too many arguments in function definition
    "PLR0915",  # Too many statements
    "PYI034",   # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
    "PYI036",   # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
    "S102",     # Use of `exec` detected
    "S105",     # Possible hardcoded password assigned to
    "S106",     # Possible hardcoded password assigned to argument: "{name}"
    "S301",     # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
    "S303",     # Use of insecure MD2, MD4, MD5, or SHA1 hash function
    "S311",     # Standard pseudo-random generators are not suitable for cryptographic purposes
    "TD002",    # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
    "TRY003",   # Avoid specifying long messages outside the exception class
]

[tool.ruff.format]
quote-style = "single"
indent-style = "space"

[tool.ruff.lint.per-file-ignores]
"**/__init__.py" = [
    "F401", # Unused imports
]
"**/{scripts}/*" = [
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
    "T20",     # flake8-print
]
"**/{tests}/*" = [
    "ASYNC230", # Async functions should not open files with blocking methods like `open`
    "D",       # Everything from the pydocstyle
    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
    "S101",    # Use of assert detected
    "SLF001",  # Private member accessed: `{name}`
    "T20",     # flake8-print
    "TRY301",  # Abstract `raise` to an inner function
]
"**/{docs,website}/**" = [
    "D",      # Everything from the pydocstyle
    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
    "F841",   # Local variable {variable} is assigned to but never used
    "N999",   # Invalid module name
    "T201",   # `print` found
]
"**/docs/03_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [
    "PLR2004", # Magic value used in comparison. Ignored for simplicity and readability of example code.
]
"**/docs/04_guides/code_examples/running_in_web_server/server.py" = [
    "TC002", # ruff false positive. Import actually needed during runtime.
]
"**/docs/04_guides/code_examples/creating_web_archive/*.*" = [
    "ASYNC230", # Ignore for simplicity of the example.
]
"**/docs/04_guides/code_examples/scrapy_migration/scrapy_*.py" = [
    "ANN",    # Idiomatic Scrapy tutorial snippets are untyped; shown for comparison, not executed.
    "RUF012", # Scrapy declares `start_urls` as a plain class-level list.
]

[tool.ruff.lint.flake8-quotes]
docstring-quotes = "double"
inline-quotes = "single"

[tool.ruff.lint.flake8-type-checking]
runtime-evaluated-base-classes = [
    "pydantic.BaseModel",
    "pydantic_settings.BaseSettings",
]

[tool.ruff.lint.flake8-builtins]
builtins-ignorelist = ["id"]

[tool.ruff.lint.isort]
known-first-party = ["crawlee"]

[tool.pytest.ini_options]
addopts = "-r a --verbose --dist worksteal"
asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
timeout = 1800
# A thread timer kills the stuck worker even when its main thread is blocked and a SIGALRM handler can never run.
timeout_method = "thread"
# pytest-timeout writes its stack dump to the xdist worker's stdout, which never reaches the console. faulthandler
# writes straight to the stderr file descriptor, so a test stuck this long leaves its stacks in the log.
faulthandler_timeout = 600
markers = [
    "run_alone: marks tests that must run in isolation",
]
# Ignore DeprecationWarnings coming from Uvicorn's internal imports. Uvicorn relies on deprecated
# modules from `websockets`, which triggers warnings during tests. These are safe to ignore until
# Uvicorn updates its internals.
filterwarnings = [
    "ignore:websockets.legacy is deprecated:DeprecationWarning",
    "ignore:websockets.server.WebSocketServerProtocol is deprecated:DeprecationWarning",
]

[tool.ty.environment]
python-version = "3.10"

[tool.ty.src]
include = ["src", "tests", "scripts", "docs", "website"]
exclude = [
    "src/crawlee/project_template",
    "docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py",
    "website/versioned_docs",
]

[tool.ty.rules]
unused-ignore-comment = "error"

[[tool.ty.overrides]]
include = [
    "docs/**/*.py",
    "website/**/*.py",
]

[tool.ty.overrides.rules]
import-not-found = "ignore"

[tool.coverage.report]
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]

[tool.ipdb]
context = 7

[tool.uv]
# Minimal defense against supply-chain attacks.
exclude-newer = "24 hours"

[tool.uv.exclude-newer-package]
# Allow internal Apify packages to install immediately.
apify-client = false
apify-shared = false
apify_fingerprint_datapoints = false
crawlee = false
impit = false

# Run tasks with: uv run poe <task>
[tool.poe.tasks]
clean = { cmd = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache .uv-cache build coverage-unit.xml coverage-integration.xml dist htmlcov website/.docusaurus website/module_shortcuts.json website/node_modules ", help = "Remove build artifacts and clean caches" }
install-sync = { cmd = "uv sync --all-extras", help = "Sync dependencies with all extras" }
build = { cmd = "uv build --verbose", help = "Build the package" }
publish-to-pypi = { cmd = "uv publish --verbose --token ${APIFY_PYPI_TOKEN_CRAWLEE}", help = "Publish the package to PyPI" }
type-check = { cmd = "uv run ty check", help = "Run the type checker" }
check-code = { sequence = ["lint", "type-check", "unit-tests", "integration-tests"], help = "Run lint, type check, unit tests, and integration tests" }

[tool.poe.tasks.install-dev]
shell = "uv sync --all-extras && uv run pre-commit install && uv run playwright install"
help = "Install development dependencies, pre-commit hooks, and Playwright browsers"

[tool.poe.tasks.lint]
shell = "uv run ruff format --check && uv run ruff check"
help = "Run the linter and check formatting"

[tool.poe.tasks.format]
shell = "uv run ruff check --fix && uv run ruff format"
help = "Fix lint issues and format code"

[tool.poe.tasks.unit-tests]
shell = """
uv run pytest \
    -m "run_alone" \
    tests/unit && \
uv run pytest \
    --numprocesses=${TESTS_CONCURRENCY:-auto} \
    -m "not run_alone" \
    tests/unit
"""
help = "Run unit tests (`run_alone` tests first, then the rest in parallel)"

[tool.poe.tasks.unit-tests-cov]
# Falls back to the default core with a warning on Python < 3.12.
env = { COVERAGE_CORE = "sysmon" }
shell = """
uv run pytest \
    -m "run_alone" \
    --cov=src/crawlee \
    --cov-report=xml:coverage-unit.xml \
    tests/unit && \
uv run pytest \
    --numprocesses=${TESTS_CONCURRENCY:-auto} \
    -m "not run_alone" \
    --cov=src/crawlee \
    --cov-report=xml:coverage-unit.xml \
    --cov-append \
    tests/unit
"""
help = "Run unit tests with an XML coverage report"

[tool.poe.tasks.integration-tests]
shell = """
uv run pytest \
    -m "run_alone" \
    tests/integration && \
uv run pytest \
    --numprocesses=${TESTS_CONCURRENCY:-auto} \
    -m "not run_alone" \
    tests/integration
"""
help = "Run integration tests (`run_alone` tests first, then the rest in parallel)"

[tool.poe.tasks.integration-tests-cov]
# Falls back to the default core with a warning on Python < 3.12.
env = { COVERAGE_CORE = "sysmon" }
shell = """
uv run pytest \
    -m "run_alone" \
    --cov=src/crawlee \
    --cov-report=xml:coverage-integration.xml \
    tests/integration && \
uv run pytest \
    --numprocesses=${TESTS_CONCURRENCY:-auto} \
    -m "not run_alone" \
    --cov=src/crawlee \
    --cov-report=xml:coverage-integration.xml \
    --cov-append \
    tests/integration
"""
help = "Run integration tests with an XML coverage report"

[tool.poe.tasks.e2e-templates-tests]
cmd = """
uv run pytest \
    --numprocesses=${TESTS_CONCURRENCY:-auto} \
    --basetemp=./.pytest-tmp \
    tests/e2e/project_template
"""
help = "Run E2E tests of the project templates"

[tool.poe.tasks.build-docs]
shell = "./build_api_reference.sh && pnpm install && uv run pnpm build"
cwd = "website"
help = "Build the documentation website"

[tool.poe.tasks.run-docs]
shell = "./build_api_reference.sh && pnpm install && uv run pnpm start"
cwd = "website"
help = "Run the documentation website locally"