# PySpark dependencies (required)
py4j>=0.10.9.9

# PySpark dependencies (optional)
numpy>=1.23.2
pyarrow>=18.0.0
pandas>=2.2.0,<3.0.0
scipy>=1.8.0
plotly<6.0.0
mlflow>=2.3.1
scikit-learn
matplotlib
memory-profiler>=0.61.0
pyyaml>=3.11

# PySpark test dependencies
unittest-xml-reporting
openpyxl

# PySpark test dependencies (optional)
asv
coverage
tabulate
# Used by the opt-in UDF transpile hypothesis test suite (gated on
# RUN_HYPOTHESIS in the environment). See SPARK-54783.
hypothesis>=6,<7

# Kafka streaming test dependencies (optional)
# Required for running Kafka integration tests with Docker test containers
testcontainers[kafka]>=3.7.0
kafka-python-ng>=2.0.2

# Linter
ruff==0.14.8
mypy==1.19.1
pytest-mypy-plugins==3.2.0
# See SPARK-38680.
pandas-stubs>=2.2.0
scipy-stubs
types-PyYAML

# Documentation (SQL)
mkdocs

# Documentation (Python)
pydata_sphinx_theme>=0.13
ipython
nbsphinx
numpydoc
jinja2
sphinx==8.2.3
sphinx-plotly-directive
sphinx-copybutton
docutils
markupsafe

# Development scripts
jira>=3.5.2
PyGithub

# Spark Connect (required)
grpcio>=1.76.0
grpcio-status>=1.76.0
googleapis-common-protos>=1.71.0
protobuf==6.33.5
zstandard>=0.25.0

# Spark Connect python proto generation plugin (optional)
mypy-protobuf==3.3.0
googleapis-common-protos-stubs==2.2.0
grpc-stubs==1.24.11

# Debug for Spark and Spark Connect
graphviz>=0.20
flameprof==0.4
viztracer
debugpy
pystack>=1.6.0; sys_platform=='linux'
psutil

# TorchDistributor dependencies
torch
torchvision
torcheval

# DeepspeedTorchDistributor dependencies
deepspeed; sys_platform != 'darwin'
