github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/python/Makefile (about)

     1  
     2  SHELL := /bin/bash
     3  PYTHON = python3
     4  PIP = pip3
     5  DOCFILE := ../docs/python_sdk.md
     6  PYAISLOADER_TEST_TYPE ?= short
     7  define PREFIX =
     8  ---
     9  layout: post
    10  title: PYTHON SDK
    11  permalink: /docs/python-sdk
    12  redirect_from:
    13   - /python_sdk.md/
    14   - /docs/python_sdk.md/
    15  ---
    16  
    17  AIStore Python SDK is a growing set of client-side objects and methods to access and utilize AIS clusters.
    18  
    19  > For PyTorch integration and usage examples, please refer to [AIS Python SDK](https://pypi.org/project/aistore) available via Python Package Index (PyPI), or see [https://github.com/NVIDIA/aistore/tree/main/python/aistore](https://github.com/NVIDIA/aistore/tree/main/python/aistore).
    20  
    21  endef
    22  export PREFIX
    23  BOTO_UNIT_TEST_COUNT := $(shell ls -1 tests/unit/botocore_patch/test*py | wc -l)
    24  export AWS_SESSION_TOKEN := testing
    25  # will be fetched from CI pipeline secrets
    26  # export AWS_REGION := us-east-1
    27  # export AWS_SECRET_ACCESS_KEY := testing
    28  # export AWS_ACCESS_KEY_ID := testing
    29  export AWS_SECURITY_TOKEN := testing
    30  BOTO_VERSION_ARGS := ""
    31  ifdef BOTO3_VERSION
    32  	BOTO_VERSION_ARGS := " boto3==$(BOTO3_VERSION) "
    33  endif
    34  ifdef BOTOCORE_VERSION
    35  	BOTO_VERSION_ARGS := " $(BOTO_VERSION_ARGS) botocore==$(BOTOCORE_VERSION) "
    36  endif
    37  BOTO_VERSION_ARGS := $(patsubst "%",%,$(BOTO_VERSION_ARGS))
    38  
    39  .PHONY: common_deps
    40  common_deps:
    41  	$(PIP) install -r aistore/common_requirements --quiet
    42  
    43  .PHONY: dev_deps
    44  dev_deps:
    45  	$(PIP) install -r aistore/pytorch/dev_requirements --quiet
    46  
    47  .PHONY: botocore_deps
    48  botocore_deps:
    49  	$(PIP) install --upgrade -r aistore/botocore_patch/botocore_requirements $(BOTO_VERSION_ARGS) --quiet
    50  
    51  .PHONY: s3_test_deps
    52  s3_test_deps:
    53  	$(PIP) install --upgrade -r tests/s3compat/requirements --quiet
    54  
    55  .PHONY: python_tests
    56  python_tests: common_deps dev_deps botocore_deps python_sdk_tests python_etl_tests python_botocore_tests
    57  
    58  # Tests for aistore.sdk
    59  .PHONY: python_sdk_tests
    60  python_sdk_tests: common_deps python_sdk_unit_tests python_sdk_integration_tests
    61  
    62  .PHONY: python_sdk_integration_tests
    63  python_sdk_integration_tests: common_deps
    64  	pytest -v tests/integration/sdk -m "not etl"
    65  
    66  .PHONY: python_sdk_unit_tests
    67  python_sdk_unit_tests: common_deps
    68  	pytest -v tests/unit/sdk
    69  
    70  .PHONY: python_etl_tests
    71  python_etl_tests: common_deps
    72  	pytest -v -s tests/integration/sdk/ -m etl
    73  
    74  .PHONY: python_s3_compat_test
    75  python_s3_compat_test: s3_test_deps
    76  	python3 tests/s3compat/run_tests.py
    77  
    78  # Tests for aistore.botocore_patch
    79  .PHONY: python_botocore_tests
    80  python_botocore_tests: common_deps botocore_deps python_botocore_unit_tests python_botocore_integration_tests
    81  
    82  .PHONY: python_botocore_unit_tests
    83  python_botocore_unit_tests: common_deps botocore_deps
    84  	pytest -v -n $(BOTO_UNIT_TEST_COUNT) --dist loadfile  tests/unit/botocore_patch
    85  
    86  .PHONY: python_botocore_integration_tests
    87  python_botocore_integration_tests: common_deps botocore_deps
    88  	pytest -v tests/integration/botocore_patch tests/integration/boto3
    89  
    90  # Tests for aistore.pytorch
    91  .PHONY: python_pytorch_unit_tests
    92  python_pytorch_unit_tests: common_deps dev_deps
    93  	pytest -v tests/unit/pytorch
    94  
    95  .PHONY: lint
    96  lint: common_deps
    97  	pylint --rcfile=.pylintrc --recursive=y aistore
    98  
    99  .PHONY: lint-tests
   100  lint-tests: common_deps botocore_deps
   101  	pylint --rcfile=.pylintrc-tests --recursive=y tests
   102  
   103  .PHONY: generate-docs
   104  generate-docs:
   105  	@ set -e
   106  	@ [[ $$(pipx --help) ]] || $(PIP) install pipx; $(PYTHON) -m pipx ensurepath --force
   107  	@ [[ $$(pydoc-markdown --help) ]] || pipx install --force pydoc-markdown[novella]
   108  	@ echo "$$PREFIX" > $(DOCFILE)
   109  	@ pydoc-markdown -I ./aistore/sdk -m client -m cluster -m bucket -m object -m multiobj.object_group -m multiobj.object_names -m multiobj.object_range -m multiobj.object_template -m job -m object_reader -m object_iterator -m etl '{ renderer: { type: markdown, descriptive_class_title: "Class: ", render_toc: true, render_toc_title: "", render_module_header: false, classdef_with_decorators: true } }' >> $(DOCFILE)
   110  	@ sed -i -e 's/####/###/g' $(DOCFILE)
   111  
   112  .PHONY:
   113  test-pyaisloader:
   114  	@./pyaisloader/ci-test.sh $(PYAISLOADER_TEST_TYPE)