Николай Сасковец, инженер-программист, iTechArt
Николай Сасковец
March 1, 2019
echo celery > requirements.txt
./.venv/bin/python -m pip install -r requirements.txt
source ./.venv/bin/activate
pip freeze
amqp==2.4.0
billiard==3.5.0.5
celery==4.2.1
kombu==4.2.2.post1
pytz==2018.9
vine==1.2.0
↖ зависимости тянут за собой свои зависимости
echo celery > requirements.txt
pip install -r requirements.txt
echo celery > requirements.txt
pip install -r requirements.txt
amqp==2.3.2 | amqp==2.4.0 billiard==3.5.0.5 | billiard==3.5.0.5 celery==4.2.1 | celery==4.2.1 kombu==4.2.2 | kombu==4.2.2.post1 pytz==2018.7 | pytz==2018.9 vine==1.1.4 | vine==1.2.0
amqp==2.3.2 | amqp==2.4.0 billiard==3.5.0.5 | billiard==3.5.0.5 celery==4.2.1 | celery==4.2.1 kombu==4.2.2 | kombu==4.2.2.post1 pytz==2018.7 | pytz==2018.9 vine==1.1.4 | vine==1.2.0
amqp==2.4.0 | amqp==2.4.1 billiard==3.5.0.5 | billiard==3.5.0.5 celery==4.2.1 | celery==4.2.1 kombu==4.2.2.post1 | kombu==4.3.0 pytz==2018.9 | pytz==2018.9 vine==1.2.0 | vine==1.2.0
cat ~/celery/requirements/default.txt
pytz>dev billiard>=3.5.0.2,<3.6.0 kombu>=4.2.0,<5.0
1 +-- 15 lines: amqp==2.4.0 + 1 +-- 15 lines: amqp==2.4.0 16 coreschema==0.0.4 16 coreschema==0.0.4 17 coverage==4.5.2 17 coverage==4.5.2 18 cryptography==2.4.2 18 cryptography==2.5 19 decorator==4.3.2 19 decorator==4.3.2 20 Django==2.1.5 20 Django==2.1.5 21 +-- 7 lines: django-bulk-up + 21 +-- 7 lines: django-bulk-u 28 django-timezone-field==3.0 28 django-timezone-field==3.0 29 djangorestframework==3.9.1 29 djangorestframework==3.9.1 30 docker==3.6.0 30 docker==3.7.0 31 docker-pycreds==0.4.0 31 docker-pycreds==0.4.0 32 docutils==0.14 32 docutils==0.14 33 ecdsa==0.13 33 ecdsa==0.13 ---------------------------- 34 entrypoints==0.3 34 factory-boy==2.11.1 35 factory-boy==2.11.1 35 Faker==1.0.1 36 Faker==1.0.2 36 flake8==3.6.0 37 flake8==3.7.3 37 flake8-quotes==1.0.0 38 flake8-quotes==1.0.0 38 +-- 12 lines: flower==0.9.2 + 39 +-- 12 lines: flower==0.9.2 50 Jinja2==2.10 51 Jinja2==2.10 51 jmespath==0.9.3 52 jmespath==0.9.3 52 jsondiff==1.1.1 53 jsondiff==1.1.1 53 jsonpickle==1.0 54 jsonpickle==1.1 54 kombu==4.2.2.post1 55 kombu==4.2.2.post1 55 MarkupSafe==1.1.0 56 MarkupSafe==1.1.0 56 mccabe==0.6.1 57 mccabe==0.6.1 57 mock==2.0.0 58 mock==2.0.0 58 moto==1.3.7 59 moto==1.3.7 59 openapi-codec==1.3.2 60 openapi-codec==1.3.2 60 opentracing==1.3.0 61 opentracing==1.3.0 61 opentracing-instrumentation> 62 opentracing-instrumentatio> 62 parameterized==0.6.3 63 parameterized==0.6.3 63 parso==0.3.2 64 parso==0.3.2 64 pbr==5.1.1 65 pbr==5.1.2 65 pexpect==4.6.0 66 pexpect==4.6.0 66 pickleshare==0.7.5 67 pickleshare==0.7.5 67 prompt-toolkit==2.0.8 68 prompt-toolkit==2.0.8 68 psycopg2==2.7.7 69 psycopg2==2.7.7 69 ptyprocess==0.6.0 70 ptyprocess==0.6.0 70 pyaml==18.11.0 71 pyaml==18.11.0 71 pycodestyle==2.4.0 72 pycodestyle==2.5.0 72 pycountry==18.12.8 73 pycountry==18.12.8 73 pycparser==2.19 74 pycparser==2.19 74 pycryptodome==3.7.2 75 pycryptodome==3.7.3 75 pyflakes==2.0.0 76 pyflakes==2.1.0
Пихаем выхлоп pip freeze
в requirements.txt
!
pip install celery
pip freeze > requirements.txt
...
pip install -r requirements.txt
Назовём это pip-freeze решения.
requirements.in
requirements.txt
Назовём это Решение requirement.in.
echo 'celery'>requirements.in
pip install -U -r requirements.in
pip freeze > requirements.txt
pip install -r requirements.txt
pip freeze | diff requirements.txt - && echo 'Идентичны!'
Идентичны!Последовательность команд для обновления «запиненных» зависимостей
cat requirements.in
celery>=4.2.1,<4.3
virtualenv ./temp-venv
./temp-venv/bin/python -m pip install -r requirements.in
./temp-venv/bin/python -m pip freeze > requirements.txt
rm -rf ./temp-venv
./.venv/bin/python -m pip install -r requirements.txt
pip-tools — это набор консольных утилит, которые автоматизируют ваш процесс управления установленными с помощью pip пакетов:
source /path/to/venv/bin/activate
pip install pip-tools
pip freeze
echo celery > requirements.in
pip-compile requirements.in --generate-hashes --output-file requirements.txt
Параметр --generate-hashes указывает pip-compile добавить в requirements.txt хэши, которые будут использоваться в дальнейшем алгоритмом проверки хэшей, встроенным в pip (>=8.0) на этапе pip-sync.
amqp==2.4.0 \ --hash=sha256:9f181e4aef6562e6f9f45660578fc1556150ca06e836ecb9e733e6ea10b48464 \ --hash=sha256:c3d7126bfbc640d076a01f1f4f6e609c0e4348508150c1f61336b0d83c738d2b \ # via kombu billiard==3.5.0.5 \ --hash=sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e \ # via celery celery==4.2.1 \ --hash=sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678 \ --hash=sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13 kombu==4.2.2.post1 \ --hash=sha256:1ef049243aa05f29e988ab33444ec7f514375540eaa8e0b2e1f5255e81c5e56d \ --hash=sha256:3c9dca2338c5d893f30c151f5d29bfb81196748ab426d33c362ab51f1e8dbf78 \ # via celery pytz==2018.9 \ --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \ --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c \ # via celery vine==1.2.0 \ --hash=sha256:3cd505dcf980223cfaf13423d371f2e7ff99247e38d5985a01ec8264e4f2aca1 \ --hash=sha256:ee4813e915d0e1a54e5c1963fde0855337f82655678540a6bc5996bca4165f76 \ # via amqp
amqp==2.4.0 \ --hash=sha256:9f181e4aef6562e6f9f45660578fc1556150ca06e836ecb9e733e6ea10b48464 \ --hash=sha256:c3d7126bfbc640d076a01f1f4f6e609c0e4348508150c1f61336b0d83c738d2b \ # via kombu billiard==3.5.0.5 \ --hash=sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e \ # via celery celery==4.2.1 \ --hash=sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678 \ --hash=sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13 kombu==4.2.2.post1 \ --hash=sha256:1ef049243aa05f29e988ab33444ec7f514375540eaa8e0b2e1f5255e81c5e56d \ --hash=sha256:3c9dca2338c5d893f30c151f5d29bfb81196748ab426d33c362ab51f1e8dbf78 \ # via celery pytz==2018.9 \ --hash=sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9 \ --hash=sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c \ # via celery vine==1.2.0 \ --hash=sha256:3cd505dcf980223cfaf13423d371f2e7ff99247e38d5985a01ec8264e4f2aca1 \ --hash=sha256:ee4813e915d0e1a54e5c1963fde0855337f82655678540a6bc5996bca4165f76 \ # via amqp
pip-compile --generate-hashes --output-file requirements.txt
Путём процесса установки, удаленеия и обновления пакетов в виртуальном окружении доводит состояние пакетов до точно того состояния, которое описано в requirements.txt, если это возможно.
pip-sync
pip-sync custom-requirements.txt another-one.txt
* pyp-sync не трогает пакеты инструментов пакетирования (setuptools, pip, pip-tools).
Pipenv is a tool that aims to bring the best of all packaging worlds to the Python world.
Pipfile.lock
— содержит «запиненные», все зависимости, необходимые для запуска и работы проекта. Генерируется на основе Pipfile.Pipfile
— содержит только зависимости проекта и мета-информацию о зависимостях.Список необходимых для установки пакетов декларируется в файле Pipfile, вместо привычного всем requirements.txt
.
Pipfile:
[[source]] url = "https://pypi.org/simple" verify_ssl = true name = "pypi" [packages] celery = "*" [dev-packages] "flake8" = "*" [requires] python_version = "3.6"
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
celery = "*"
[dev-packages]
"flake8" = "*"
[requires]
python_version = "3.6"
...
[dev-packages]
"flake8" = "*"
[requires]
python_version = "3.6"
... [packages] django = "*" celery = "*" elasticsearch = "<6.0.0,>=5.0.0" [dev-packages] "flake8" = "*" moto = "==1.3.4" [requires] python_version = "3.6"
...
[packages]
django = "*"
celery = "*"
elasticsearch = "<6.0.0,>=5.0.0"
api_client = {git =
"ssh://git@gitlab.com/something/api-client.git",
ref = "v1.0.2"}
[dev-packages]
"flake8" = "*"
moto = "==1.3.4"
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
celery = "*"
[dev-packages]
"flake8" = "*"
[requires]
python_version = "3.6"
[[source]] url = "https://pypi.org/simple" verify_ssl = true name = "pypi" [[source]] url = "http://pypi.famous.org/simple" verify_ssl = false name = "famous"
[packages] celery = { version="*", index="pypi"} api_base_client = { version="*", index="famous"} api_client = { git="ssh://git@gitlab.com/something/api-client.git", ref = "v1.0.2"}
{ "_meta": { "hash": { "sha256": "415dfdcb118dd9bdfef17671cb7dcd78dbd69b6ae7d4f39e8b44e71d60ca72e7" }, "pipfile-spec": 6, "requires": { "python_version": "3.6" }, "sources": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true } ] }, "default": {}, "develop": {} }
{ "_meta": {
"hash": { "sha256": "415dfdcb118dd9bdfef17671cb7dcd78dbd69b6ae7d4f39e8b44e71d60ca72e7" },
"pipfile-spec": 6,
"requires": { "python_version": "3.6" },
"sources": [
{ "name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {},
"develop": {}
}
{ "_meta": {
"hash": { "sha256": "415dfdcb118dd9bdfef17671cb7dcd78dbd69b6ae7d4f39e8b44e71d60ca72e7" },
"pipfile-spec": 6,
"requires": { "python_version": "3.6" },
"sources": [
{ "name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {},
"develop": {}
}
{ "_meta": {
"hash": { "sha256": "415dfdcb118dd9bdfef17671cb7dcd78dbd69b6ae7d4f39e8b44e71d60ca72e7" },
"pipfile-spec": 6,
"requires": { "python_version": "3.6" },
"sources": [
{ "name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {},
"develop": {}
}
"default": { "amqp": { "hashes": [ "sha256:9f181e4aef6562e6f9f45660578fc1556150ca06e836ecb9e733e6ea10b48464", "sha256:c3d7126bfbc640d076a01f1f4f6e609c0e4348508150c1f61336b0d83c738d2b" ], "version": "==2.4.0" }, "billiard": { "hashes": [ "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" ], "version": "==3.5.0.5" }, "celery": { "hashes": [ "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13" ], "index": "pypi", "version": "==4.2.1" }, "kombu": { "hashes": [ "sha256:1ef049243aa05f29e988ab33444ec7f514375540eaa8e0b2e1f5255e81c5e56d", "sha256:3c9dca2338c5d893f30c151f5d29bfb81196748ab426d33c362ab51f1e8dbf78" ], "version": "==4.2.2.post1" }, "pytz": { "hashes": [ "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" ], "version": "==2018.9" }, "vine": { "hashes": [ "sha256:3cd505dcf980223cfaf13423d371f2e7ff99247e38d5985a01ec8264e4f2aca1", "sha256:ee4813e915d0e1a54e5c1963fde0855337f82655678540a6bc5996bca4165f76" ], "version": "==1.2.0" } },
"default": { "amqp": { "hashes": [ "sha256:9f181e4aef6562e6f9f45660578fc1556150ca06e836ecb9e733e6ea10b48464", "sha256:c3d7126bfbc640d076a01f1f4f6e609c0e4348508150c1f61336b0d83c738d2b" ], "version": "==2.4.0" }, "billiard": { "hashes": [ "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" ], "version": "==3.5.0.5" }, "celery": { "hashes": [ "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13" ], "index": "pypi", "version": "==4.2.1" }, "kombu": { "hashes": [ "sha256:1ef049243aa05f29e988ab33444ec7f514375540eaa8e0b2e1f5255e81c5e56d", "sha256:3c9dca2338c5d893f30c151f5d29bfb81196748ab426d33c362ab51f1e8dbf78" ], "version": "==4.2.2.post1" }, "pytz": { "hashes": [ "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" ], "version": "==2018.9" }, "vine": { "hashes": [ "sha256:3cd505dcf980223cfaf13423d371f2e7ff99247e38d5985a01ec8264e4f2aca1", "sha256:ee4813e915d0e1a54e5c1963fde0855337f82655678540a6bc5996bca4165f76" ], "version": "==1.2.0" } },
pipenv --help | только-интересные
install Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages. graph Displays currently-installed dependency graph information. uninstall Un-installs a provided package and removes it from Pipfile. clean Uninstalls all packages not specified in Pipfile.lock. lock Generates Pipfile.lock. update Runs lock, then sync. sync Installs all packages specified in Pipfile.lock.
pipenv install celery
... Installing celery… ✔ Installation Succeeded Pipfile.lock (c07081) out of date, updating to (ca72e7)… Locking [dev-packages] dependencies… Locking [packages] dependencies… ✔ Success! Updated Pipfile.lock (c07081)! Installing dependencies from Pipfile.lock (c07081)… 🐍 ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 6/6 — 00:00:01
pipenv install celery
Pipfile
Pipfile.lock
pipenv install
Pipfile
Pipfile.lock
pipenv uninstall celery
Uninstalling celery… Uninstalling celery-4.2.1: Successfully uninstalled celery-4.2.1
pipenv run pip freeze
amqp==2.4.0 billiard==3.5.0.5 kombu==4.2.2.post1 pytz==2018.9 vine==1.2.0
pipenv uninstall celery
Uninstalling celery… Uninstalling celery-4.2.1: Successfully uninstalled celery-4.2.1
pipenv clean
Uninstalling vine… Uninstalling pytz… Uninstalling kombu… Uninstalling billiard… Uninstalling amqp…
pipenv run pip freeze
pipenv graph
celery==4.2.1 - billiard [required: >=3.5.0.2,<3.6.0, installed: 3.5.0.5] - kombu [required: >=4.2.0,<5.0, installed: 4.2.2.post1] - amqp [required: >=2.1.4,<3.0, installed: 2.4.0] - vine [required: >=1.1.3, installed: 1.2.0] - pytz [required: >dev, installed: 2018.9]
pipenv install factory-boy python-slugify==2.0.0
pipenv install factory-boy python-slugify==2.0.0
pipenv graph | grep -iE 'unidecode|'
factory-boy==2.11.1
- Faker [required: >=0.7.0, installed: 1.0.2]
- python-dateutil [required: >=2.4, installed: 2.7.5]
- six [required: >=1.5, installed: 1.12.0]
- six [required: >=1.10, installed: 1.12.0]
- text-unidecode [required: ==1.2, installed: 1.2]
python-slugify==2.0.0
pipenv lock
— генерация Pipfile.lockpipenv update
— обновление версий запиненных пакетов в Pipfile.lockpipenv sync
— установка пакетов, указанных в Pipfile.lock
pipenv sync
—
(игнорирует Pipfile, может быть полезно для прода)~/.virtualenvs/
.
export WORKON_HOME=~/.venvs
export PIPENV_VENV_IN_PROJECT=True
.venv
внутри каталога проекта)
run
🤔
shell
Единоразовое выполнение: pipenv run %SOMETHING%
pipenv run какой-то--скрипт.py
pipenv run pip freeze
pipenv run env
Интерактивный режим: pipenv shell
pipenv shell
какой-то--скрипт.py
pip freeze
env
pipenv run one.sh | pipenv run two.py | pipenv run ./three.py
pipenv shell
one.sh | two.py | ./three.py
$VIRTUAL_ENV
, соответствующая пути к виртуальному окружениюvirtualenvwrapper
, autoenv
и т.д.pyproject.toml
Подробнее о poetry: https://github.com/sdispater/poetry/
pyproject.toml is created to replace setup.py
, requirements.txt
, setup.cfg
, MANIFEST.in
Уже поддерживают: poetry, flit, black.
В планах: pytest, flake8, coverage.
Не собираются тратить на это время: mypy.
requirements.txt | pip-tools | pipenv | poetry | |
проверка хэшей | да | да | да | да |
генерация хэшей | нет | да | да | да |
разные PyPI | нет | нет | да | да |
разные версии Python | нет | нет | нет | да |
встроенная работа с виртуальным окружением | нет | нет | да | да |
подходит для управления зависимостями библиотек | нет | нет | нет | да |
публикация вашего проекта на PyPI | нет | нет | нет | да |